analysis_interns_groups.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "from statsmodels.stats.multicomp import pairwise_tukeyhsd\n",
    "from statsmodels.stats.multicomp import MultiComparison\n",
    "from statsmodels.formula.api import ols\n",
    "from scipy import stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"playlists.csv\", sep=\";\", encoding = \"ISO-8859-1\") "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Negatives and positives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "companies = data['company'].unique()\n",
    "by_company = [data[data.company == company] for company in companies]\n",
    "positives = []\n",
    "negatives = []\n",
    "for data_com in by_company:\n",
    "    data_com = data_com.sort_values('playlist_sample')\n",
    "    last_pl = int(data_com.tail(1).playlist_sample)\n",
    "    #pls = pd.DataFrame({'pl':range (1,last_pl+1), 'old':[0]*last_pl, 'new':[0]*last_pl})\n",
    "    #curr_pl = data_com.query('playlist_sample == '+str(1))\n",
    "    #pls.new[0]=(curr_pl.shape[0])/3\n",
    "    #sum_pls = [curr_pl.shape[0]/3]\n",
    "    #for i in range(2,last_pl+1):\n",
    "    #    curr_pl = data_com.query('playlist_sample == '+str(i))\n",
    "    #    pre_pl = data_com.query('playlist_sample == '+str(i-1))\n",
    "    #    olds = curr_pl['song'].map(pre_pl['song'].value_counts()).sum(axis = 0)/3\n",
    "    #    pls.old[i-1]= olds/3 \n",
    "    #    pls.new[i-1]=(curr_pl.shape[0]-olds)/3\n",
    "    #    sum_pls.append(curr_pl.shape[0]/3)\n",
    "    #sum_pls = pd.DataFrame(sum_pls)\n",
    "    #pls_rel = (pls[['old']].div(sum_pls.values, axis=0)*100).join(pls[['new']].div(sum_pls.values, axis=0)*100)\n",
    "    #pls[['old','new']].plot(kind='bar', stacked=True, title=data_com.iloc[0,0])\n",
    "    #for n in pls_rel:\n",
    "    #    for i, (cs, ab, pc) in enumerate(zip(pls.iloc[:, 1:].cumsum(1)[n].values, pls[n].values, pls_rel[n].values)):\n",
    "    #        if(pc>0):\n",
    "    #            plt.text(i, cs - ab/2, str(np.round(pc, 1)) + '%', va='center', ha='center')\n",
    "    df_last_pl= data_com.query('playlist_sample == '+str(last_pl))\n",
    "    positives.append(df_last_pl)\n",
    "    pos_loc = pd.DataFrame({}, columns=data_com.columns)\n",
    "    rep=0\n",
    "    for index, row in data_com[data_com.playlist_sample<last_pl].iterrows(): \n",
    "        if not ((df_last_pl['artist'] == row['artist']) & (df_last_pl['song'] ==  row['song'])).any() and pos_loc[(pos_loc['artist']== row['artist']) & (pos_loc['song'] ==  row['song'])].shape[0]<3:\n",
    "            pos_loc= pos_loc.append(row, ignore_index=True)\n",
    "        else:\n",
    "            rep+=1\n",
    "    #n_vs_p = pd.DataFrame({'sam':['pos', 'neg'],'num':[df_last_pl.shape[0]/3,pos_loc.shape[0]/3]})\n",
    "    # n_vs_p.plot.bar(x='sam', y='num', rot=0, title=data_com.iloc[0,0])\n",
    "    negatives.append(pos_loc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Count of positives negatives and total by company."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Arte Francés\n",
      "Negatives:  76.0\n",
      "Positives:  29.0\n",
      "Total:  147.0\n",
      "Club De Banqueros y Empresarios\n",
      "Negatives:  61.0\n",
      "Positives:  24.0\n",
      "Total:  88.0\n",
      "Gramma\n",
      "Negatives:  51.0\n",
      "Positives:  21.0\n",
      "Total:  103.0\n",
      "Hotel Marrakech\n",
      "Negatives:  28.0\n",
      "Positives:  20.0\n",
      "Total:  68.0\n",
      "Specialized\n",
      "Negatives:  38.0\n",
      "Positives:  35.0\n",
      "Total:  103.0\n",
      "Urban Place\n",
      "Negatives:  48.0\n",
      "Positives:  24.0\n",
      "Total:  85.0\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(companies)):\n",
    "    print(companies[i])\n",
    "    print(\"Negatives: \",negatives[i].shape[0]/3)\n",
    "    print(\"Positives: \",positives[i].shape[0]/3)\n",
    "    print(\"Total: \", by_company[i].shape[0]/3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Append the positives and negatives records with a new column \"chosen\" that takes value of 1 if is chosen."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Usuarios\\1144084318\\AppData\\Roaming\\Python\\Python37\\site-packages\\ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "df_n_ps = []\n",
    "for i in range(len(negatives)):\n",
    "    negatives[i]['chosen']=0\n",
    "    positives[i]['chosen']=1\n",
    "    df_n_ps.append(negatives[i].append(positives[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score, silhouette_samples, silhouette_score, calinski_harabaz_score\n",
    "from sklearn import preprocessing\n",
    "from sklearn.decomposition import PCA\n",
    "import math\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 315 entries, 0 to 179\n",
      "Data columns (total 65 columns):\n",
      "company                 315 non-null object\n",
      "playlist_sample         315 non-null object\n",
      "namesfiles              315 non-null object\n",
      "no                      315 non-null object\n",
      "artist                  315 non-null object\n",
      "song                    315 non-null object\n",
      "sampleratefiles         315 non-null object\n",
      "totalsamplesfiles       315 non-null object\n",
      "durationfiles           315 non-null float64\n",
      "bitratefiles            315 non-null float64\n",
      "rmsfiles                315 non-null float64\n",
      "rmsmedianfiles          315 non-null float64\n",
      "lowenergyfiles          315 non-null float64\n",
      "ASRfiles                315 non-null float64\n",
      "beatspectrumfiles       315 non-null float64\n",
      "eventdensityfiles       315 non-null float64\n",
      "tempofiles              315 non-null float64\n",
      "pulseclarityfiles       315 non-null float64\n",
      "zerocrossfiles          315 non-null float64\n",
      "rolloffsfiles           315 non-null float64\n",
      "brightnessfiles         315 non-null float64\n",
      "spreadfiles             315 non-null float64\n",
      "centroidfiles           314 non-null float64\n",
      "kurtosisfiles           315 non-null float64\n",
      "flatnessfiles           315 non-null float64\n",
      "entropyfiles            315 non-null float64\n",
      "mfccfiles_1             315 non-null float64\n",
      "mfccfiles_2             315 non-null float64\n",
      "mfccfiles_3             315 non-null float64\n",
      "mfccfiles_4             315 non-null float64\n",
      "mfccfiles_5             315 non-null float64\n",
      "mfccfiles_6             315 non-null float64\n",
      "mfccfiles_7             315 non-null float64\n",
      "mfccfiles_8             315 non-null float64\n",
      "mfccfiles_9             315 non-null float64\n",
      "mfccfiles_10            315 non-null float64\n",
      "mfccfiles_11            315 non-null float64\n",
      "mfccfiles_12            315 non-null float64\n",
      "mfccfiles_13            315 non-null float64\n",
      "pitchfiles              315 non-null float64\n",
      "inharmonicityfiles      315 non-null float64\n",
      "bestkeyfiles            315 non-null float64\n",
      "keyclarityfiles         315 non-null float64\n",
      "modalityfiles           315 non-null float64\n",
      "tonalcentroidfiles_1    315 non-null float64\n",
      "tonalcentroidfiles_2    315 non-null float64\n",
      "tonalcentroidfiles_3    315 non-null float64\n",
      "tonalcentroidfiles_4    315 non-null float64\n",
      "tonalcentroidfiles_5    315 non-null float64\n",
      "tonalcentroidfiles_6    315 non-null float64\n",
      "chromagramfiles_1       315 non-null float64\n",
      "chromagramfiles_2       315 non-null float64\n",
      "chromagramfiles_3       315 non-null float64\n",
      "chromagramfiles_4       315 non-null float64\n",
      "chromagramfiles_5       315 non-null float64\n",
      "chromagramfiles_6       315 non-null float64\n",
      "chromagramfiles_7       315 non-null float64\n",
      "chromagramfiles_8       315 non-null float64\n",
      "chromagramfiles_9       315 non-null float64\n",
      "chromagramfiles_10      315 non-null float64\n",
      "chromagramfiles_11      315 non-null float64\n",
      "chromagramfiles_12      315 non-null float64\n",
      "attackslopefiles        315 non-null float64\n",
      "attackleapfiles         315 non-null float64\n",
      "chosen                  315 non-null int64\n",
      "dtypes: float64(56), int64(1), object(8)\n",
      "memory usage: 162.4+ KB\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(companies)):\n",
    "    df_n_ps[i].bitratefiles = df_n_ps[i].bitratefiles.astype('float64')\n",
    "    df_n_ps[i].pitchfiles = df_n_ps[i].pitchfiles.astype('float64')\n",
    "    df_n_ps[i].bestkeyfiles = df_n_ps[i].bestkeyfiles.astype('float64')\n",
    "df_n_ps[0].info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Vamos a reemplazar los NaN y entonces a normalizar los datos para que todas las variables tengan la misma importancia. Solo vamos a considerar los datos numéricos."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "durationfiles          -1.889141e-16\n",
       "bitratefiles            0.000000e+00\n",
       "rmsfiles                3.559763e-16\n",
       "rmsmedianfiles         -2.396672e-16\n",
       "lowenergyfiles          1.543739e-16\n",
       "ASRfiles                1.226532e-16\n",
       "beatspectrumfiles       2.269789e-16\n",
       "eventdensityfiles      -6.132661e-17\n",
       "tempofiles              4.103860e-16\n",
       "pulseclarityfiles      -6.696583e-17\n",
       "zerocrossfiles         -1.092600e-16\n",
       "rolloffsfiles           2.661011e-16\n",
       "brightnessfiles         1.092600e-16\n",
       "spreadfiles             1.519067e-16\n",
       "centroidfiles           1.501444e-16\n",
       "kurtosisfiles           1.875043e-16\n",
       "flatnessfiles          -4.017950e-17\n",
       "entropyfiles            6.012827e-16\n",
       "mfccfiles_1            -4.398598e-16\n",
       "mfccfiles_2            -2.326182e-17\n",
       "mfccfiles_3             6.308886e-17\n",
       "mfccfiles_4             1.718202e-17\n",
       "mfccfiles_5             1.832749e-17\n",
       "mfccfiles_6            -3.172066e-17\n",
       "mfccfiles_7            -1.400996e-16\n",
       "mfccfiles_8             5.110550e-17\n",
       "mfccfiles_9             3.101575e-17\n",
       "mfccfiles_10           -6.485112e-17\n",
       "mfccfiles_11           -4.229421e-18\n",
       "mfccfiles_12           -1.304071e-17\n",
       "mfccfiles_13           -1.233581e-17\n",
       "pitchfiles              0.000000e+00\n",
       "inharmonicityfiles     -1.009422e-15\n",
       "bestkeyfiles            2.424868e-16\n",
       "keyclarityfiles        -3.972131e-16\n",
       "modalityfiles          -3.771234e-17\n",
       "tonalcentroidfiles_1   -1.517745e-17\n",
       "tonalcentroidfiles_2   -5.921189e-17\n",
       "tonalcentroidfiles_3    2.326182e-17\n",
       "tonalcentroidfiles_4    2.502407e-17\n",
       "tonalcentroidfiles_5    3.260179e-17\n",
       "tonalcentroidfiles_6   -2.361427e-17\n",
       "chromagramfiles_1       6.414622e-17\n",
       "chromagramfiles_2      -2.061843e-17\n",
       "chromagramfiles_3      -3.489272e-17\n",
       "chromagramfiles_4      -1.755210e-16\n",
       "chromagramfiles_5       1.797504e-17\n",
       "chromagramfiles_6      -3.101575e-17\n",
       "chromagramfiles_7      -8.776049e-17\n",
       "chromagramfiles_8       7.471977e-17\n",
       "chromagramfiles_9      -4.194176e-17\n",
       "chromagramfiles_10      3.630253e-17\n",
       "chromagramfiles_11      5.057683e-17\n",
       "chromagramfiles_12     -5.894756e-17\n",
       "attackslopefiles       -6.626093e-17\n",
       "attackleapfiles        -1.423905e-16\n",
       "chosen                  7.218212e-16\n",
       "dtype: float64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_n_ps_std = [0]*len(companies)\n",
    "for i in range(len(companies)):\n",
    "    df_n_ps[i] = df_n_ps[i].fillna(0)\n",
    "    df_n_ps_std[i] = pd.DataFrame(preprocessing.scale(df_n_ps[i].iloc[:,8:]))\n",
    "    df_n_ps_std[i].columns=df_n_ps[i].columns[8:]\n",
    "df_n_ps_std[0].mean(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "durationfiles           1.001591\n",
       "bitratefiles            0.000000\n",
       "rmsfiles                1.001591\n",
       "rmsmedianfiles          1.001591\n",
       "lowenergyfiles          1.001591\n",
       "ASRfiles                1.001591\n",
       "beatspectrumfiles       1.001591\n",
       "eventdensityfiles       1.001591\n",
       "tempofiles              1.001591\n",
       "pulseclarityfiles       1.001591\n",
       "zerocrossfiles          1.001591\n",
       "rolloffsfiles           1.001591\n",
       "brightnessfiles         1.001591\n",
       "spreadfiles             1.001591\n",
       "centroidfiles           1.001591\n",
       "kurtosisfiles           1.001591\n",
       "flatnessfiles           1.001591\n",
       "entropyfiles            1.001591\n",
       "mfccfiles_1             1.001591\n",
       "mfccfiles_2             1.001591\n",
       "mfccfiles_3             1.001591\n",
       "mfccfiles_4             1.001591\n",
       "mfccfiles_5             1.001591\n",
       "mfccfiles_6             1.001591\n",
       "mfccfiles_7             1.001591\n",
       "mfccfiles_8             1.001591\n",
       "mfccfiles_9             1.001591\n",
       "mfccfiles_10            1.001591\n",
       "mfccfiles_11            1.001591\n",
       "mfccfiles_12            1.001591\n",
       "mfccfiles_13            1.001591\n",
       "pitchfiles              0.000000\n",
       "inharmonicityfiles      1.001591\n",
       "bestkeyfiles            1.001591\n",
       "keyclarityfiles         1.001591\n",
       "modalityfiles           1.001591\n",
       "tonalcentroidfiles_1    1.001591\n",
       "tonalcentroidfiles_2    1.001591\n",
       "tonalcentroidfiles_3    1.001591\n",
       "tonalcentroidfiles_4    1.001591\n",
       "tonalcentroidfiles_5    1.001591\n",
       "tonalcentroidfiles_6    1.001591\n",
       "chromagramfiles_1       1.001591\n",
       "chromagramfiles_2       1.001591\n",
       "chromagramfiles_3       1.001591\n",
       "chromagramfiles_4       1.001591\n",
       "chromagramfiles_5       1.001591\n",
       "chromagramfiles_6       1.001591\n",
       "chromagramfiles_7       1.001591\n",
       "chromagramfiles_8       1.001591\n",
       "chromagramfiles_9       1.001591\n",
       "chromagramfiles_10      1.001591\n",
       "chromagramfiles_11      1.001591\n",
       "chromagramfiles_12      1.001591\n",
       "attackslopefiles        1.001591\n",
       "attackleapfiles         1.001591\n",
       "chosen                  1.001591\n",
       "dtype: float64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_n_ps_std[0].std(axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Borramos pitch y bitrate porque todos sus valores son 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(companies)):\n",
    "    df_n_ps_std[i] = df_n_ps_std[i].drop(columns=\"pitchfiles\")\n",
    "    df_n_ps_std[i] = df_n_ps_std[i].drop(columns=\"bitratefiles\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'2.3.0'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import keras\n",
    "keras.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Input, Flatten, Dense#, Lambda\n",
    "from keras.models import Model\n",
    "from keras import layers\n",
    "from keras import models, optimizers\n",
    "\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.model_selection import GridSearchCV #permite buscar la mejor configuración de parámetros con C-V\n",
    "from sklearn.metrics import make_scorer # permite crear una clase scorer a partir de una función de score (necesario para el kappa)\n",
    "from sklearn.metrics import accuracy_score, cohen_kappa_score, classification_report, roc_auc_score\n",
    "from sklearn.model_selection import train_test_split #metodo de particionamiento de datasets para evaluación\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MFCC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['mfccfiles_1', 'mfccfiles_2', 'mfccfiles_3', 'mfccfiles_4',\n",
       "       'mfccfiles_5', 'mfccfiles_6', 'mfccfiles_7', 'mfccfiles_8',\n",
       "       'mfccfiles_9', 'mfccfiles_10', 'mfccfiles_11', 'mfccfiles_12',\n",
       "       'mfccfiles_13'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_n_ps_std[0].columns[17:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 315 entries, 0 to 314\n",
      "Data columns (total 13 columns):\n",
      "mfccfiles_1     315 non-null float64\n",
      "mfccfiles_2     315 non-null float64\n",
      "mfccfiles_3     315 non-null float64\n",
      "mfccfiles_4     315 non-null float64\n",
      "mfccfiles_5     315 non-null float64\n",
      "mfccfiles_6     315 non-null float64\n",
      "mfccfiles_7     315 non-null float64\n",
      "mfccfiles_8     315 non-null float64\n",
      "mfccfiles_9     315 non-null float64\n",
      "mfccfiles_10    315 non-null float64\n",
      "mfccfiles_11    315 non-null float64\n",
      "mfccfiles_12    315 non-null float64\n",
      "mfccfiles_13    315 non-null float64\n",
      "dtypes: float64(13)\n",
      "memory usage: 32.1 KB\n"
     ]
    }
   ],
   "source": [
    "df_n_ps_std_mfcc = [None]*len(companies)\n",
    "for i in range(len(companies)):\n",
    "    df_n_ps_std_mfcc[i] = pd.DataFrame(df_n_ps_std[i].iloc[:,17:30])\n",
    "    df_n_ps_std_mfcc[i].columns=df_n_ps_std[i].columns[17:30]\n",
    "df_n_ps_std_mfcc[0].info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2-comp groups combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "two_combs = list(itertools.combinations(range(len(companies)), 2))\n",
    "for comb in two_combs:\n",
    "    print('## '+companies[comb[0]].upper()+' y '+companies[comb[1]].upper())\n",
    "    X = df_n_ps_std_mfcc[comb[0]].append(df_n_ps_std_mfcc[comb[1]])\n",
    "    y = df_n_ps[comb[0]]['chosen'].append(df_n_ps[comb[1]]['chosen'])\n",
    "\n",
    "    print(X.head())\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "    mlp = MLPClassifier(hidden_layer_sizes=(30,30,30))\n",
    "\n",
    "    activation_vec = ['logistic', 'relu', 'tanh']\n",
    "    max_iter_vec = [10, 20, 50, 75, 100, 200, 300, 400, 500, 1000, 2000]\n",
    "    hidden_layer_sizes_vec = [(10,), (20,), (30,), (10, 10), (20, 20), (30, 30), (20, 10), \n",
    "                              (10, 10, 10), (20, 20, 20), (30, 30, 30), (30, 20, 10)]\n",
    "    learning_rate_init_vec = [0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.02]\n",
    "    batch_size_vec = [10, 20, 40, 60, 80, 100, 150]\n",
    "\n",
    "\n",
    "    np.random.seed(1234)\n",
    "    parametros = {'activation': activation_vec,\n",
    "                  'max_iter':max_iter_vec,\n",
    "                  'hidden_layer_sizes': hidden_layer_sizes_vec,\n",
    "                  'learning_rate_init': learning_rate_init_vec#,\n",
    "                  #'batch_size': batch_size_vec\n",
    "                  }\n",
    "    scoring = {'kappa':make_scorer(cohen_kappa_score), 'accuracy':'accuracy'}\n",
    "    grid = GridSearchCV(mlp, param_grid=parametros, cv=5, scoring=scoring, refit='accuracy', n_jobs=-1, iid=True)\n",
    "\n",
    "    grid.fit(X_train, y_train)\n",
    "\n",
    "    print(\"Los parámetros del mejor modelo fueron {0}, que permiten obtener un Accuracy de {1:.2f}% y un Kappa del {2:.2f}\".format(\n",
    "        grid.best_params_, grid.best_score_*100, grid.cv_results_['mean_test_kappa'][grid.best_index_]*100))\n",
    "\n",
    "    n0=X_train.shape[1]\n",
    "    ### hidden_layer_sizes\n",
    "    ns = []\n",
    "    for i in range (len(grid.best_params_['hidden_layer_sizes'])):\n",
    "        ns.append(grid.best_params_['hidden_layer_sizes'][i])\n",
    "\n",
    "    ns.append(1)\n",
    "    lr = grid.best_params_['learning_rate_init']\n",
    "    epochs = grid.best_params_['max_iter']\n",
    "    if grid.best_params_['activation'] == 'logistic':\n",
    "        grid.best_params_['activation'] = 'sigmoid'\n",
    "    \n",
    "    input_tensor = Input(shape = (n0,))\n",
    "    hidden_outputs = [input_tensor]\n",
    "    for i in range (len(ns)-1):\n",
    "        hidden_outputs.append(Dense(ns[i], activation = grid.best_params_['activation'])(hidden_outputs[i]))\n",
    "    classification_output = Dense(ns[-1], activation = 'sigmoid')(hidden_outputs[-1])\n",
    "\n",
    "    model = Model([input_tensor], [classification_output])\n",
    "    weights = model.get_weights()\n",
    "\n",
    "    print(model.summary())\n",
    "\n",
    "    model.set_weights(weights)\n",
    "    adam = keras.optimizers.Adam(lr=lr)\n",
    "    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    history = model.fit(X_train, y_train, epochs=epochs, validation_data=(X_test, y_test), verbose=0, \n",
    "                callbacks=[\n",
    "                    keras.callbacks.ReduceLROnPlateau(\n",
    "                        monitor='val_accuracy', factor=0.5, patience=10, min_delta=0.01, verbose=0\n",
    "                    )\n",
    "                ]\n",
    "             )\n",
    "\n",
    "    acc = history.history['accuracy']\n",
    "    val_acc = history.history['val_accuracy']\n",
    "    loss = history.history['loss']\n",
    "    val_loss = history.history['val_loss']\n",
    "\n",
    "    epochs = range(len(acc))\n",
    "\n",
    "    print(\"epochs: \"+str(len(acc)))\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(X_test, y_test)\n",
    "    print(\"test loss: {}, test accuracy: {}\".format(test_loss, test_acc))\n",
    "\n",
    "    y_pred = model.predict(X_test)\n",
    "    print(\"AUC ROC: \",roc_auc_score(y_test, y_pred))\n",
    "\n",
    "    y_pred = list(map(lambda i: int(i>=0.5), y_pred))\n",
    "    print(\"Kappa: \",cohen_kappa_score(y_test, y_pred))\n",
    "\n",
    "    print(confusion_matrix(y_test, y_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import display, Markdown, Latex\n",
    "two_combs = list(itertools.combinations(range(len(companies)), 2))\n",
    "for comb in two_combs[2:]:\n",
    "    display(Markdown('## '+companies[comb[0]]+' y '+companies[comb[1]]))\n",
    "    X = df_n_ps_std_mfcc[comb[0]].append(df_n_ps_std_mfcc[comb[1]])\n",
    "    y = df_n_ps[comb[0]]['chosen'].append(df_n_ps[comb[1]]['chosen'])\n",
    "\n",
    "    print(X.head())\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "    mlp = MLPClassifier(hidden_layer_sizes=(30,30,30))\n",
    "\n",
    "    activation_vec = ['logistic', 'relu', 'tanh']\n",
    "    max_iter_vec = [10, 20, 50, 75, 100, 200, 300, 400, 500, 1000, 2000]\n",
    "    hidden_layer_sizes_vec = [(10,), (20,), (30,), (10, 10), (20, 20), (30, 30), (20, 10), \n",
    "                              (10, 10, 10), (20, 20, 20), (30, 30, 30), (30, 20, 10)]\n",
    "    learning_rate_init_vec = [0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.02]\n",
    "    batch_size_vec = [10, 20, 40, 60, 80, 100, 150]\n",
    "\n",
    "\n",
    "    np.random.seed(1234)\n",
    "    parametros = {'activation': activation_vec,\n",
    "                  'max_iter':max_iter_vec,\n",
    "                  'hidden_layer_sizes': hidden_layer_sizes_vec,\n",
    "                  'learning_rate_init': learning_rate_init_vec#,\n",
    "                  #'batch_size': batch_size_vec\n",
    "                  }\n",
    "    scoring = {'kappa':make_scorer(cohen_kappa_score), 'accuracy':'accuracy'}\n",
    "    grid = GridSearchCV(mlp, param_grid=parametros, cv=5, scoring=scoring, refit='accuracy', n_jobs=-1, iid=True)\n",
    "\n",
    "    grid.fit(X_train, y_train)\n",
    "\n",
    "    print(\"Los parámetros del mejor modelo fueron {0}, que permiten obtener un Accuracy de {1:.2f}% y un Kappa del {2:.2f}\".format(\n",
    "        grid.best_params_, grid.best_score_*100, grid.cv_results_['mean_test_kappa'][grid.best_index_]*100))\n",
    "\n",
    "    n0=X_train.shape[1]\n",
    "    ### hidden_layer_sizes\n",
    "    ns = []\n",
    "    for i in range (len(grid.best_params_['hidden_layer_sizes'])):\n",
    "        ns.append(grid.best_params_['hidden_layer_sizes'][i])\n",
    "\n",
    "    ns.append(1)\n",
    "    lr = grid.best_params_['learning_rate_init']\n",
    "    epochs = grid.best_params_['max_iter']\n",
    "\n",
    "    input_tensor = Input(shape = (n0,))\n",
    "    hidden_outputs = [input_tensor]\n",
    "    if grid.best_params_['activation'] == 'logistic':\n",
    "        grid.best_params_['activation'] = 'sigmoid'\n",
    "    for i in range (len(ns)-1):\n",
    "        hidden_outputs.append(Dense(ns[i], activation = grid.best_params_['activation'])(hidden_outputs[i]))\n",
    "    classification_output = Dense(ns[-1], activation = 'sigmoid')(hidden_outputs[-1])\n",
    "\n",
    "    model = Model([input_tensor], [classification_output])\n",
    "    weights = model.get_weights()\n",
    "\n",
    "    print(model.summary())\n",
    "\n",
    "    model.set_weights(weights)\n",
    "    adam = keras.optimizers.Adam(lr=lr)\n",
    "    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    history = model.fit(X_train, y_train, epochs=epochs, validation_data=(X_test, y_test), verbose=0, \n",
    "                callbacks=[\n",
    "                    keras.callbacks.ReduceLROnPlateau(\n",
    "                        monitor='val_accuracy', factor=0.5, patience=10, min_delta=0.01, verbose=0\n",
    "                    )\n",
    "                ]\n",
    "             )\n",
    "\n",
    "    acc = history.history['accuracy']\n",
    "    val_acc = history.history['val_accuracy']\n",
    "    loss = history.history['loss']\n",
    "    val_loss = history.history['val_loss']\n",
    "\n",
    "    epochs = range(len(acc))\n",
    "\n",
    "    print(\"epochs: \"+str(len(acc)))\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(X_test, y_test)\n",
    "    print(\"test loss: {}, test accuracy: {}\".format(test_loss, test_acc))\n",
    "\n",
    "    y_pred = model.predict(X_test)\n",
    "    print(\"AUC ROC: \",roc_auc_score(y_test, y_pred))\n",
    "\n",
    "    y_pred = list(map(lambda i: int(i>=0.5), y_pred))\n",
    "    print(\"Kappa: \",cohen_kappa_score(y_test, y_pred))\n",
    "\n",
    "    print(confusion_matrix(y_test, y_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'companies' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-27513bc455c7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mIPython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdisplay\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mdisplay\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mMarkdown\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mLatex\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mtwo_combs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitertools\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcombinations\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcompanies\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mcomb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtwo_combs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0mdisplay\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mMarkdown\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'## '\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mcompanies\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcomb\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m' y '\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mcompanies\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcomb\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf_n_ps_std_mfcc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcomb\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf_n_ps_std_mfcc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcomb\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'companies' is not defined"
     ]
    }
   ],
   "source": [
    "from IPython.display import display, Markdown, Latex\n",
    "two_combs = list(itertools.combinations(range(len(companies)), 2))\n",
    "for comb in two_combs[4:]:\n",
    "    display(Markdown('## '+companies[comb[0]]+' y '+companies[comb[1]]))\n",
    "    X = df_n_ps_std_mfcc[comb[0]].append(df_n_ps_std_mfcc[comb[1]])\n",
    "    y = df_n_ps[comb[0]]['chosen'].append(df_n_ps[comb[1]]['chosen'])\n",
    "\n",
    "    print(X.head())\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "    mlp = MLPClassifier(hidden_layer_sizes=(30,30,30))\n",
    "\n",
    "    activation_vec = ['logistic', 'relu', 'tanh']\n",
    "    max_iter_vec = [10, 20, 50, 75, 100, 200, 300, 400, 500, 1000, 2000]\n",
    "    hidden_layer_sizes_vec = [(10,), (20,), (30,), (10, 10), (20, 20), (30, 30), (20, 10), \n",
    "                              (10, 10, 10), (20, 20, 20), (30, 30, 30), (30, 20, 10)]\n",
    "    learning_rate_init_vec = [0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.02]\n",
    "    batch_size_vec = [10, 20, 40, 60, 80, 100, 150]\n",
    "\n",
    "\n",
    "    np.random.seed(1234)\n",
    "    parametros = {'activation': activation_vec,\n",
    "                  'max_iter':max_iter_vec,\n",
    "                  'hidden_layer_sizes': hidden_layer_sizes_vec,\n",
    "                  'learning_rate_init': learning_rate_init_vec#,\n",
    "                  #'batch_size': batch_size_vec\n",
    "                  }\n",
    "    scoring = {'kappa':make_scorer(cohen_kappa_score), 'accuracy':'accuracy'}\n",
    "    grid = GridSearchCV(mlp, param_grid=parametros, cv=5, scoring=scoring, refit='accuracy', n_jobs=-1, iid=True)\n",
    "\n",
    "    grid.fit(X_train, y_train)\n",
    "\n",
    "    print(\"Los parámetros del mejor modelo fueron {0}, que permiten obtener un Accuracy de {1:.2f}% y un Kappa del {2:.2f}\".format(\n",
    "        grid.best_params_, grid.best_score_*100, grid.cv_results_['mean_test_kappa'][grid.best_index_]*100))\n",
    "\n",
    "    n0=X_train.shape[1]\n",
    "    ### hidden_layer_sizes\n",
    "    ns = []\n",
    "    for i in range (len(grid.best_params_['hidden_layer_sizes'])):\n",
    "        ns.append(grid.best_params_['hidden_layer_sizes'][i])\n",
    "\n",
    "    ns.append(1)\n",
    "    lr = grid.best_params_['learning_rate_init']\n",
    "    epochs = grid.best_params_['max_iter']\n",
    "\n",
    "    input_tensor = Input(shape = (n0,))\n",
    "    hidden_outputs = [input_tensor]\n",
    "    if grid.best_params_['activation'] == 'logistic':\n",
    "        grid.best_params_['activation'] = 'sigmoid'\n",
    "    for i in range (len(ns)-1):\n",
    "        hidden_outputs.append(Dense(ns[i], activation = grid.best_params_['activation'])(hidden_outputs[i]))\n",
    "    classification_output = Dense(ns[-1], activation = 'sigmoid')(hidden_outputs[-1])\n",
    "\n",
    "    model = Model([input_tensor], [classification_output])\n",
    "    weights = model.get_weights()\n",
    "\n",
    "    print(model.summary())\n",
    "\n",
    "    model.set_weights(weights)\n",
    "    adam = keras.optimizers.Adam(lr=lr)\n",
    "    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    history = model.fit(X_train, y_train, epochs=epochs, validation_data=(X_test, y_test), verbose=0, \n",
    "                callbacks=[\n",
    "                    keras.callbacks.ReduceLROnPlateau(\n",
    "                        monitor='val_accuracy', factor=0.5, patience=10, min_delta=0.01, verbose=0\n",
    "                    )\n",
    "                ]\n",
    "             )\n",
    "\n",
    "    acc = history.history['accuracy']\n",
    "    val_acc = history.history['val_accuracy']\n",
    "    loss = history.history['loss']\n",
    "    val_loss = history.history['val_loss']\n",
    "\n",
    "    epochs = range(len(acc))\n",
    "\n",
    "    print(\"epochs: \"+str(len(acc)))\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(X_test, y_test)\n",
    "    print(\"test loss: {}, test accuracy: {}\".format(test_loss, test_acc))\n",
    "\n",
    "    y_pred = model.predict(X_test)\n",
    "    print(\"AUC ROC: \",roc_auc_score(y_test, y_pred))\n",
    "\n",
    "    y_pred = list(map(lambda i: int(i>=0.5), y_pred))\n",
    "    print(\"Kappa: \",cohen_kappa_score(y_test, y_pred))\n",
    "\n",
    "    print(confusion_matrix(y_test, y_pred))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3-comp groups combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Gramma"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0     0.297583     1.225637    -0.367641     0.606499     0.072373   \n",
      "1     0.637676    -1.507256    -1.572737    -0.954161    -0.857425   \n",
      "2     2.236730    -0.319414     0.669910    -1.918119    -0.820882   \n",
      "3     0.662077    -0.381499     0.111981    -1.743808    -1.317593   \n",
      "4     0.736502     0.112932    -0.065024    -1.049458    -0.408043   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0    -2.029620     0.791469     0.752018     2.268802     -1.383289   \n",
      "1     0.327005     0.816764     0.214245     0.241703      0.637066   \n",
      "2    -2.379333    -1.570021    -2.755344    -2.150610     -2.528577   \n",
      "3    -1.348534    -0.627198    -1.629882    -2.075974     -1.248765   \n",
      "4    -0.437499     0.090831    -0.852983    -1.922491     -0.284365   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0      0.548279      1.903211     -1.011470  \n",
      "1      1.601538      0.300317     -0.466779  \n",
      "2     -0.877081     -0.522248     -1.429911  \n",
      "3     -1.126014     -1.316359     -1.126174  \n",
      "4      0.210624     -0.032122     -0.700183  \n",
      "(786, 13)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30, 30, 30), 'learning_rate_init': 0.001, 'max_iter': 100}, que permiten obtener un Accuracy de 74.70% y un Kappa del 26.53\n",
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 2,311\n",
      "Trainable params: 2,311\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 100\n",
      "197/197 [==============================] - 0s 30us/step\n",
      "test loss: 0.6007471054338561, test accuracy: 0.6903553009033203\n",
      "AUC ROC:  0.6249078850405305\n",
      "Kappa:  0.11110289222575631\n",
      "[[125  13]\n",
      " [ 48  11]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Hotel Marrakech"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0     0.297583     1.225637    -0.367641     0.606499     0.072373   \n",
      "1     0.637676    -1.507256    -1.572737    -0.954161    -0.857425   \n",
      "2     2.236730    -0.319414     0.669910    -1.918119    -0.820882   \n",
      "3     0.662077    -0.381499     0.111981    -1.743808    -1.317593   \n",
      "4     0.736502     0.112932    -0.065024    -1.049458    -0.408043   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0    -2.029620     0.791469     0.752018     2.268802     -1.383289   \n",
      "1     0.327005     0.816764     0.214245     0.241703      0.637066   \n",
      "2    -2.379333    -1.570021    -2.755344    -2.150610     -2.528577   \n",
      "3    -1.348534    -0.627198    -1.629882    -2.075974     -1.248765   \n",
      "4    -0.437499     0.090831    -0.852983    -1.922491     -0.284365   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0      0.548279      1.903211     -1.011470  \n",
      "1      1.601538      0.300317     -0.466779  \n",
      "2     -0.877081     -0.522248     -1.429911  \n",
      "3     -1.126014     -1.316359     -1.126174  \n",
      "4      0.210624     -0.032122     -0.700183  \n",
      "(714, 13)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.005, 'max_iter': 500}, que permiten obtener un Accuracy de 74.39% y un Kappa del 37.61\n",
      "Model: \"model_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,381\n",
      "Trainable params: 1,381\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 500\n",
      "179/179 [==============================] - 0s 45us/step\n",
      "test loss: 0.8997574944735905, test accuracy: 0.6759776473045349\n",
      "AUC ROC:  0.6919279907084785\n",
      "Kappa:  0.2158610271903324\n",
      "[[98 25]\n",
      " [33 23]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0     0.297583     1.225637    -0.367641     0.606499     0.072373   \n",
      "1     0.637676    -1.507256    -1.572737    -0.954161    -0.857425   \n",
      "2     2.236730    -0.319414     0.669910    -1.918119    -0.820882   \n",
      "3     0.662077    -0.381499     0.111981    -1.743808    -1.317593   \n",
      "4     0.736502     0.112932    -0.065024    -1.049458    -0.408043   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0    -2.029620     0.791469     0.752018     2.268802     -1.383289   \n",
      "1     0.327005     0.816764     0.214245     0.241703      0.637066   \n",
      "2    -2.379333    -1.570021    -2.755344    -2.150610     -2.528577   \n",
      "3    -1.348534    -0.627198    -1.629882    -2.075974     -1.248765   \n",
      "4    -0.437499     0.090831    -0.852983    -1.922491     -0.284365   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0      0.548279      1.903211     -1.011470  \n",
      "1      1.601538      0.300317     -0.466779  \n",
      "2     -0.877081     -0.522248     -1.429911  \n",
      "3     -1.126014     -1.316359     -1.126174  \n",
      "4      0.210624     -0.032122     -0.700183  \n",
      "(789, 13)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.009, 'max_iter': 300}, que permiten obtener un Accuracy de 69.54% y un Kappa del 32.02\n",
      "Model: \"model_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,381\n",
      "Trainable params: 1,381\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 300\n",
      "198/198 [==============================] - 0s 30us/step\n",
      "test loss: 0.9417000592808531, test accuracy: 0.6717171669006348\n",
      "AUC ROC:  0.7123521521765639\n",
      "Kappa:  0.24833547482770701\n",
      "[[102  37]\n",
      " [ 28  31]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0     0.297583     1.225637    -0.367641     0.606499     0.072373   \n",
      "1     0.637676    -1.507256    -1.572737    -0.954161    -0.857425   \n",
      "2     2.236730    -0.319414     0.669910    -1.918119    -0.820882   \n",
      "3     0.662077    -0.381499     0.111981    -1.743808    -1.317593   \n",
      "4     0.736502     0.112932    -0.065024    -1.049458    -0.408043   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0    -2.029620     0.791469     0.752018     2.268802     -1.383289   \n",
      "1     0.327005     0.816764     0.214245     0.241703      0.637066   \n",
      "2    -2.379333    -1.570021    -2.755344    -2.150610     -2.528577   \n",
      "3    -1.348534    -0.627198    -1.629882    -2.075974     -1.248765   \n",
      "4    -0.437499     0.090831    -0.852983    -1.922491     -0.284365   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0      0.548279      1.903211     -1.011470  \n",
      "1      1.601538      0.300317     -0.466779  \n",
      "2     -0.877081     -0.522248     -1.429911  \n",
      "3     -1.126014     -1.316359     -1.126174  \n",
      "4      0.210624     -0.032122     -0.700183  \n",
      "(786, 13)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30,), 'learning_rate_init': 0.002, 'max_iter': 20}, que permiten obtener un Accuracy de 70.80% y un Kappa del 7.30\n",
      "Model: \"model_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_4 (InputLayer)         (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 451\n",
      "Trainable params: 451\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 20\n",
      "197/197 [==============================] - 0s 41us/step\n",
      "test loss: 0.5736924785042777, test accuracy: 0.720812201499939\n",
      "AUC ROC:  0.579973474801061\n",
      "Kappa:  0.03834206088577252\n",
      "[[138   7]\n",
      " [ 48   4]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma y Hotel Marrakech"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0     0.297583     1.225637    -0.367641     0.606499     0.072373   \n",
      "1     0.637676    -1.507256    -1.572737    -0.954161    -0.857425   \n",
      "2     2.236730    -0.319414     0.669910    -1.918119    -0.820882   \n",
      "3     0.662077    -0.381499     0.111981    -1.743808    -1.317593   \n",
      "4     0.736502     0.112932    -0.065024    -1.049458    -0.408043   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0    -2.029620     0.791469     0.752018     2.268802     -1.383289   \n",
      "1     0.327005     0.816764     0.214245     0.241703      0.637066   \n",
      "2    -2.379333    -1.570021    -2.755344    -2.150610     -2.528577   \n",
      "3    -1.348534    -0.627198    -1.629882    -2.075974     -1.248765   \n",
      "4    -0.437499     0.090831    -0.852983    -1.922491     -0.284365   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0      0.548279      1.903211     -1.011470  \n",
      "1      1.601538      0.300317     -0.466779  \n",
      "2     -0.877081     -0.522248     -1.429911  \n",
      "3     -1.126014     -1.316359     -1.126174  \n",
      "4      0.210624     -0.032122     -0.700183  \n",
      "(675, 13)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.01, 'max_iter': 100}, que permiten obtener un Accuracy de 69.96% y un Kappa del 15.14\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_5 (InputLayer)         (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,381\n",
      "Trainable params: 1,381\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 100\n",
      "169/169 [==============================] - 0s 35us/step\n",
      "test loss: 1.5889130231191422, test accuracy: 0.668639063835144\n",
      "AUC ROC:  0.6099159663865547\n",
      "Kappa:  0.21382289416846645\n",
      "[[90 29]\n",
      " [27 23]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0     0.297583     1.225637    -0.367641     0.606499     0.072373   \n",
      "1     0.637676    -1.507256    -1.572737    -0.954161    -0.857425   \n",
      "2     2.236730    -0.319414     0.669910    -1.918119    -0.820882   \n",
      "3     0.662077    -0.381499     0.111981    -1.743808    -1.317593   \n",
      "4     0.736502     0.112932    -0.065024    -1.049458    -0.408043   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0    -2.029620     0.791469     0.752018     2.268802     -1.383289   \n",
      "1     0.327005     0.816764     0.214245     0.241703      0.637066   \n",
      "2    -2.379333    -1.570021    -2.755344    -2.150610     -2.528577   \n",
      "3    -1.348534    -0.627198    -1.629882    -2.075974     -1.248765   \n",
      "4    -0.437499     0.090831    -0.852983    -1.922491     -0.284365   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0      0.548279      1.903211     -1.011470  \n",
      "1      1.601538      0.300317     -0.466779  \n",
      "2     -0.877081     -0.522248     -1.429911  \n",
      "3     -1.126014     -1.316359     -1.126174  \n",
      "4      0.210624     -0.032122     -0.700183  \n",
      "(750, 13)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30, 20, 10), 'learning_rate_init': 0.01, 'max_iter': 50}, que permiten obtener un Accuracy de 69.22% y un Kappa del 28.64\n",
      "Model: \"model_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_6 (InputLayer)         (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_17 (Dense)             (None, 20)                620       \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 1,261\n",
      "Trainable params: 1,261\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 50\n",
      "188/188 [==============================] - 0s 37us/step\n",
      "test loss: 0.9825812349928186, test accuracy: 0.5797872543334961\n",
      "AUC ROC:  0.5978638847491307\n",
      "Kappa:  0.054735234215886\n",
      "[[86 36]\n",
      " [43 23]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0     0.297583     1.225637    -0.367641     0.606499     0.072373   \n",
      "1     0.637676    -1.507256    -1.572737    -0.954161    -0.857425   \n",
      "2     2.236730    -0.319414     0.669910    -1.918119    -0.820882   \n",
      "3     0.662077    -0.381499     0.111981    -1.743808    -1.317593   \n",
      "4     0.736502     0.112932    -0.065024    -1.049458    -0.408043   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0    -2.029620     0.791469     0.752018     2.268802     -1.383289   \n",
      "1     0.327005     0.816764     0.214245     0.241703      0.637066   \n",
      "2    -2.379333    -1.570021    -2.755344    -2.150610     -2.528577   \n",
      "3    -1.348534    -0.627198    -1.629882    -2.075974     -1.248765   \n",
      "4    -0.437499     0.090831    -0.852983    -1.922491     -0.284365   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0      0.548279      1.903211     -1.011470  \n",
      "1      1.601538      0.300317     -0.466779  \n",
      "2     -0.877081     -0.522248     -1.429911  \n",
      "3     -1.126014     -1.316359     -1.126174  \n",
      "4      0.210624     -0.032122     -0.700183  \n",
      "(747, 13)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20, 20), 'learning_rate_init': 0.004, 'max_iter': 20}, que permiten obtener un Accuracy de 68.57% y un Kappa del 9.44\n",
      "Model: \"model_7\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_7 (InputLayer)         (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 20)                280       \n",
      "_________________________________________________________________\n",
      "dense_21 (Dense)             (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_22 (Dense)             (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 721\n",
      "Trainable params: 721\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 20\n",
      "187/187 [==============================] - 0s 48us/step\n",
      "test loss: 0.5900288157284579, test accuracy: 0.7272727489471436\n",
      "AUC ROC:  0.6224523889074508\n",
      "Kappa:  0.15128593040847205\n",
      "[[124  22]\n",
      " [ 29  12]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0     0.297583     1.225637    -0.367641     0.606499     0.072373   \n",
      "1     0.637676    -1.507256    -1.572737    -0.954161    -0.857425   \n",
      "2     2.236730    -0.319414     0.669910    -1.918119    -0.820882   \n",
      "3     0.662077    -0.381499     0.111981    -1.743808    -1.317593   \n",
      "4     0.736502     0.112932    -0.065024    -1.049458    -0.408043   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0    -2.029620     0.791469     0.752018     2.268802     -1.383289   \n",
      "1     0.327005     0.816764     0.214245     0.241703      0.637066   \n",
      "2    -2.379333    -1.570021    -2.755344    -2.150610     -2.528577   \n",
      "3    -1.348534    -0.627198    -1.629882    -2.075974     -1.248765   \n",
      "4    -0.437499     0.090831    -0.852983    -1.922491     -0.284365   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0      0.548279      1.903211     -1.011470  \n",
      "1      1.601538      0.300317     -0.466779  \n",
      "2     -0.877081     -0.522248     -1.429911  \n",
      "3     -1.126014     -1.316359     -1.126174  \n",
      "4      0.210624     -0.032122     -0.700183  \n",
      "(678, 13)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20,), 'learning_rate_init': 0.009, 'max_iter': 20}, que permiten obtener un Accuracy de 64.57% y un Kappa del 12.16\n",
      "Model: \"model_8\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_8 (InputLayer)         (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_23 (Dense)             (None, 20)                280       \n",
      "_________________________________________________________________\n",
      "dense_24 (Dense)             (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 301\n",
      "Trainable params: 301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 20\n",
      "170/170 [==============================] - 0s 71us/step\n",
      "test loss: 0.6898417111705332, test accuracy: 0.6529411673545837\n",
      "AUC ROC:  0.6288325471698113\n",
      "Kappa:  0.2145653876272514\n",
      "[[86 20]\n",
      " [39 25]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0     0.297583     1.225637    -0.367641     0.606499     0.072373   \n",
      "1     0.637676    -1.507256    -1.572737    -0.954161    -0.857425   \n",
      "2     2.236730    -0.319414     0.669910    -1.918119    -0.820882   \n",
      "3     0.662077    -0.381499     0.111981    -1.743808    -1.317593   \n",
      "4     0.736502     0.112932    -0.065024    -1.049458    -0.408043   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0    -2.029620     0.791469     0.752018     2.268802     -1.383289   \n",
      "1     0.327005     0.816764     0.214245     0.241703      0.637066   \n",
      "2    -2.379333    -1.570021    -2.755344    -2.150610     -2.528577   \n",
      "3    -1.348534    -0.627198    -1.629882    -2.075974     -1.248765   \n",
      "4    -0.437499     0.090831    -0.852983    -1.922491     -0.284365   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0      0.548279      1.903211     -1.011470  \n",
      "1      1.601538      0.300317     -0.466779  \n",
      "2     -0.877081     -0.522248     -1.429911  \n",
      "3     -1.126014     -1.316359     -1.126174  \n",
      "4      0.210624     -0.032122     -0.700183  \n",
      "(675, 13)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.003, 'max_iter': 300}, que permiten obtener un Accuracy de 70.95% y un Kappa del 16.81\n",
      "Model: \"model_9\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_9 (InputLayer)         (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_25 (Dense)             (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_26 (Dense)             (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_27 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,381\n",
      "Trainable params: 1,381\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 300\n",
      "169/169 [==============================] - 0s 53us/step\n",
      "test loss: 0.7179625766164452, test accuracy: 0.6568047404289246\n",
      "AUC ROC:  0.6463187325256291\n",
      "Kappa:  0.1395716292134832\n",
      "[[96 15]\n",
      " [43 15]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0     0.297583     1.225637    -0.367641     0.606499     0.072373   \n",
      "1     0.637676    -1.507256    -1.572737    -0.954161    -0.857425   \n",
      "2     2.236730    -0.319414     0.669910    -1.918119    -0.820882   \n",
      "3     0.662077    -0.381499     0.111981    -1.743808    -1.317593   \n",
      "4     0.736502     0.112932    -0.065024    -1.049458    -0.408043   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0    -2.029620     0.791469     0.752018     2.268802     -1.383289   \n",
      "1     0.327005     0.816764     0.214245     0.241703      0.637066   \n",
      "2    -2.379333    -1.570021    -2.755344    -2.150610     -2.528577   \n",
      "3    -1.348534    -0.627198    -1.629882    -2.075974     -1.248765   \n",
      "4    -0.437499     0.090831    -0.852983    -1.922491     -0.284365   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0      0.548279      1.903211     -1.011470  \n",
      "1      1.601538      0.300317     -0.466779  \n",
      "2     -0.877081     -0.522248     -1.429911  \n",
      "3     -1.126014     -1.316359     -1.126174  \n",
      "4      0.210624     -0.032122     -0.700183  \n",
      "(750, 13)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30,), 'learning_rate_init': 0.002, 'max_iter': 1000}, que permiten obtener un Accuracy de 71.89% y un Kappa del 36.07\n",
      "Model: \"model_10\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_10 (InputLayer)        (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_28 (Dense)             (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_29 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 451\n",
      "Trainable params: 451\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 1000\n",
      "188/188 [==============================] - 0s 37us/step\n",
      "test loss: 0.6679085125314429, test accuracy: 0.6329787373542786\n",
      "AUC ROC:  0.543974343160232\n",
      "Kappa:  0.05890887986070803\n",
      "[[109  12]\n",
      " [ 57  10]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma y Hotel Marrakech"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0    -0.339415     0.847773     0.497198    -0.389310     1.225458   \n",
      "1     0.587658    -1.195426     0.636375     0.199876     0.765321   \n",
      "2     1.465595    -2.307943     0.354567    -0.058273    -1.298853   \n",
      "3     0.749403    -1.690498    -0.125200    -1.016135     0.825845   \n",
      "4    -0.280577     0.393332     0.744917     2.411400    -0.777421   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0     1.947033    -0.736267     0.492219     0.576682      1.504697   \n",
      "1     0.061181     0.379367    -0.440867     0.232893      1.339920   \n",
      "2    -0.811453    -1.551580    -3.934320    -1.079432      2.546130   \n",
      "3     0.271444    -0.104786    -0.992141     0.049182      1.425948   \n",
      "4    -0.420018     1.258355    -1.544565    -0.498071      0.421527   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0     -1.796460      0.724954      0.958600  \n",
      "1      0.110001      0.807525      0.815678  \n",
      "2      1.421407      0.639359      0.199094  \n",
      "3     -0.343269     -0.789558     -0.411898  \n",
      "4     -0.632908     -0.056846     -0.072348  \n",
      "(615, 13)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (75) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20, 10), 'learning_rate_init': 0.005, 'max_iter': 75}, que permiten obtener un Accuracy de 74.62% y un Kappa del 38.11\n",
      "Model: \"model_11\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_11 (InputLayer)        (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_30 (Dense)             (None, 20)                280       \n",
      "_________________________________________________________________\n",
      "dense_31 (Dense)             (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_32 (Dense)             (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 501\n",
      "Trainable params: 501\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 75\n",
      "154/154 [==============================] - 0s 58us/step\n",
      "test loss: 0.7654741104547079, test accuracy: 0.6948052048683167\n",
      "AUC ROC:  0.7051208833047782\n",
      "Kappa:  0.3076334417447867\n",
      "[[80 23]\n",
      " [24 27]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0    -0.339415     0.847773     0.497198    -0.389310     1.225458   \n",
      "1     0.587658    -1.195426     0.636375     0.199876     0.765321   \n",
      "2     1.465595    -2.307943     0.354567    -0.058273    -1.298853   \n",
      "3     0.749403    -1.690498    -0.125200    -1.016135     0.825845   \n",
      "4    -0.280577     0.393332     0.744917     2.411400    -0.777421   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0     1.947033    -0.736267     0.492219     0.576682      1.504697   \n",
      "1     0.061181     0.379367    -0.440867     0.232893      1.339920   \n",
      "2    -0.811453    -1.551580    -3.934320    -1.079432      2.546130   \n",
      "3     0.271444    -0.104786    -0.992141     0.049182      1.425948   \n",
      "4    -0.420018     1.258355    -1.544565    -0.498071      0.421527   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0     -1.796460      0.724954      0.958600  \n",
      "1      0.110001      0.807525      0.815678  \n",
      "2      1.421407      0.639359      0.199094  \n",
      "3     -0.343269     -0.789558     -0.411898  \n",
      "4     -0.632908     -0.056846     -0.072348  \n",
      "(690, 13)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (75) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.007, 'max_iter': 75}, que permiten obtener un Accuracy de 74.08% y un Kappa del 38.81\n",
      "Model: \"model_12\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_12 (InputLayer)        (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_33 (Dense)             (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_34 (Dense)             (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_35 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,381\n",
      "Trainable params: 1,381\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 75\n",
      "173/173 [==============================] - 0s 35us/step\n",
      "test loss: 1.4033675400507932, test accuracy: 0.676300585269928\n",
      "AUC ROC:  0.6981255448997384\n",
      "Kappa:  0.3108550291648883\n",
      "[[80 31]\n",
      " [25 37]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0    -0.339415     0.847773     0.497198    -0.389310     1.225458   \n",
      "1     0.587658    -1.195426     0.636375     0.199876     0.765321   \n",
      "2     1.465595    -2.307943     0.354567    -0.058273    -1.298853   \n",
      "3     0.749403    -1.690498    -0.125200    -1.016135     0.825845   \n",
      "4    -0.280577     0.393332     0.744917     2.411400    -0.777421   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0     1.947033    -0.736267     0.492219     0.576682      1.504697   \n",
      "1     0.061181     0.379367    -0.440867     0.232893      1.339920   \n",
      "2    -0.811453    -1.551580    -3.934320    -1.079432      2.546130   \n",
      "3     0.271444    -0.104786    -0.992141     0.049182      1.425948   \n",
      "4    -0.420018     1.258355    -1.544565    -0.498071      0.421527   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0     -1.796460      0.724954      0.958600  \n",
      "1      0.110001      0.807525      0.815678  \n",
      "2      1.421407      0.639359      0.199094  \n",
      "3     -0.343269     -0.789558     -0.411898  \n",
      "4     -0.632908     -0.056846     -0.072348  \n",
      "(687, 13)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (400) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (20,), 'learning_rate_init': 0.004, 'max_iter': 400}, que permiten obtener un Accuracy de 73.01% y un Kappa del 21.67\n",
      "Model: \"model_13\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_13 (InputLayer)        (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_36 (Dense)             (None, 20)                280       \n",
      "_________________________________________________________________\n",
      "dense_37 (Dense)             (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 301\n",
      "Trainable params: 301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 400\n",
      "172/172 [==============================] - 0s 41us/step\n",
      "test loss: 0.6511809784312581, test accuracy: 0.6569767594337463\n",
      "AUC ROC:  0.5520833333333333\n",
      "Kappa:  0.034259611724400396\n",
      "[[105  15]\n",
      " [ 44   8]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0    -0.339415     0.847773     0.497198    -0.389310     1.225458   \n",
      "1     0.587658    -1.195426     0.636375     0.199876     0.765321   \n",
      "2     1.465595    -2.307943     0.354567    -0.058273    -1.298853   \n",
      "3     0.749403    -1.690498    -0.125200    -1.016135     0.825845   \n",
      "4    -0.280577     0.393332     0.744917     2.411400    -0.777421   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0     1.947033    -0.736267     0.492219     0.576682      1.504697   \n",
      "1     0.061181     0.379367    -0.440867     0.232893      1.339920   \n",
      "2    -0.811453    -1.551580    -3.934320    -1.079432      2.546130   \n",
      "3     0.271444    -0.104786    -0.992141     0.049182      1.425948   \n",
      "4    -0.420018     1.258355    -1.544565    -0.498071      0.421527   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0     -1.796460      0.724954      0.958600  \n",
      "1      0.110001      0.807525      0.815678  \n",
      "2      1.421407      0.639359      0.199094  \n",
      "3     -0.343269     -0.789558     -0.411898  \n",
      "4     -0.632908     -0.056846     -0.072348  \n",
      "(618, 13)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.008, 'max_iter': 400}, que permiten obtener un Accuracy de 72.79% y un Kappa del 42.30\n",
      "Model: \"model_14\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_14 (InputLayer)        (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_38 (Dense)             (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_39 (Dense)             (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_40 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,381\n",
      "Trainable params: 1,381\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 400\n",
      "155/155 [==============================] - 0s 32us/step\n",
      "test loss: 1.1106072802697458, test accuracy: 0.6580645442008972\n",
      "AUC ROC:  0.6626377532883042\n",
      "Kappa:  0.28234471914038617\n",
      "[[68 29]\n",
      " [24 34]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0    -0.339415     0.847773     0.497198    -0.389310     1.225458   \n",
      "1     0.587658    -1.195426     0.636375     0.199876     0.765321   \n",
      "2     1.465595    -2.307943     0.354567    -0.058273    -1.298853   \n",
      "3     0.749403    -1.690498    -0.125200    -1.016135     0.825845   \n",
      "4    -0.280577     0.393332     0.744917     2.411400    -0.777421   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0     1.947033    -0.736267     0.492219     0.576682      1.504697   \n",
      "1     0.061181     0.379367    -0.440867     0.232893      1.339920   \n",
      "2    -0.811453    -1.551580    -3.934320    -1.079432      2.546130   \n",
      "3     0.271444    -0.104786    -0.992141     0.049182      1.425948   \n",
      "4    -0.420018     1.258355    -1.544565    -0.498071      0.421527   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0     -1.796460      0.724954      0.958600  \n",
      "1      0.110001      0.807525      0.815678  \n",
      "2      1.421407      0.639359      0.199094  \n",
      "3     -0.343269     -0.789558     -0.411898  \n",
      "4     -0.632908     -0.056846     -0.072348  \n",
      "(615, 13)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (10,), 'learning_rate_init': 0.004, 'max_iter': 100}, que permiten obtener un Accuracy de 68.11% y un Kappa del 20.47\n",
      "Model: \"model_15\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_15 (InputLayer)        (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_41 (Dense)             (None, 10)                140       \n",
      "_________________________________________________________________\n",
      "dense_42 (Dense)             (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 151\n",
      "Trainable params: 151\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 100\n",
      "154/154 [==============================] - 0s 39us/step\n",
      "test loss: 0.5610680889773678, test accuracy: 0.7272727489471436\n",
      "AUC ROC:  0.6909276248725789\n",
      "Kappa:  0.25294525294525294\n",
      "[[97 12]\n",
      " [30 15]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0    -0.339415     0.847773     0.497198    -0.389310     1.225458   \n",
      "1     0.587658    -1.195426     0.636375     0.199876     0.765321   \n",
      "2     1.465595    -2.307943     0.354567    -0.058273    -1.298853   \n",
      "3     0.749403    -1.690498    -0.125200    -1.016135     0.825845   \n",
      "4    -0.280577     0.393332     0.744917     2.411400    -0.777421   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0     1.947033    -0.736267     0.492219     0.576682      1.504697   \n",
      "1     0.061181     0.379367    -0.440867     0.232893      1.339920   \n",
      "2    -0.811453    -1.551580    -3.934320    -1.079432      2.546130   \n",
      "3     0.271444    -0.104786    -0.992141     0.049182      1.425948   \n",
      "4    -0.420018     1.258355    -1.544565    -0.498071      0.421527   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0     -1.796460      0.724954      0.958600  \n",
      "1      0.110001      0.807525      0.815678  \n",
      "2      1.421407      0.639359      0.199094  \n",
      "3     -0.343269     -0.789558     -0.411898  \n",
      "4     -0.632908     -0.056846     -0.072348  \n",
      "(690, 13)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.007, 'max_iter': 50}, que permiten obtener un Accuracy de 69.44% y un Kappa del 26.20\n",
      "Model: \"model_16\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_16 (InputLayer)        (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_43 (Dense)             (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_44 (Dense)             (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_45 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,381\n",
      "Trainable params: 1,381\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 50\n",
      "173/173 [==============================] - 0s 58us/step\n",
      "test loss: 1.0460927052304925, test accuracy: 0.6878612637519836\n",
      "AUC ROC:  0.7035953177257525\n",
      "Kappa:  0.3157046586580722\n",
      "[[87 17]\n",
      " [37 32]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Gramma y Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0    -0.674917     0.169246     0.673543     1.157142    -0.633186   \n",
      "1     0.277269     0.514176     0.200398     0.988939    -1.756594   \n",
      "2     1.483921     0.724793     0.473099     0.439577    -0.358096   \n",
      "3    -0.734008    -0.683844    -0.764866    -0.225060    -0.261235   \n",
      "4    -0.834815    -0.735908    -1.177596    -0.093532     0.508050   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0     0.688145     0.215883    -0.452048     1.101066      0.064017   \n",
      "1    -0.022788    -0.235704     0.523508    -0.604231      1.188209   \n",
      "2    -0.452581    -0.213173    -0.596057    -0.767473      0.696227   \n",
      "3    -0.243429     0.588768     0.874148     1.302526      0.091256   \n",
      "4     0.503458     1.380798     1.847226     1.227896      0.017729   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0     -0.153703      1.751289      0.812723  \n",
      "1      0.863617     -0.801768      0.229305  \n",
      "2     -0.111259     -0.370649     -1.325817  \n",
      "3     -0.600323     -0.827452      0.390838  \n",
      "4     -0.329325     -0.953249     -0.125917  \n",
      "(579, 13)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (75) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 20, 10), 'learning_rate_init': 0.005, 'max_iter': 75}, que permiten obtener un Accuracy de 65.67% y un Kappa del 28.05\n",
      "Model: \"model_17\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_17 (InputLayer)        (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_46 (Dense)             (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_47 (Dense)             (None, 20)                620       \n",
      "_________________________________________________________________\n",
      "dense_48 (Dense)             (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_49 (Dense)             (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 1,261\n",
      "Trainable params: 1,261\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 75\n",
      "145/145 [==============================] - 0s 62us/step\n",
      "test loss: 1.071592510979751, test accuracy: 0.6413792967796326\n",
      "AUC ROC:  0.6563275434243176\n",
      "Kappa:  0.23976608187134507\n",
      "[[64 29]\n",
      " [23 29]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Gramma y Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0    -0.674917     0.169246     0.673543     1.157142    -0.633186   \n",
      "1     0.277269     0.514176     0.200398     0.988939    -1.756594   \n",
      "2     1.483921     0.724793     0.473099     0.439577    -0.358096   \n",
      "3    -0.734008    -0.683844    -0.764866    -0.225060    -0.261235   \n",
      "4    -0.834815    -0.735908    -1.177596    -0.093532     0.508050   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0     0.688145     0.215883    -0.452048     1.101066      0.064017   \n",
      "1    -0.022788    -0.235704     0.523508    -0.604231      1.188209   \n",
      "2    -0.452581    -0.213173    -0.596057    -0.767473      0.696227   \n",
      "3    -0.243429     0.588768     0.874148     1.302526      0.091256   \n",
      "4     0.503458     1.380798     1.847226     1.227896      0.017729   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0     -0.153703      1.751289      0.812723  \n",
      "1      0.863617     -0.801768      0.229305  \n",
      "2     -0.111259     -0.370649     -1.325817  \n",
      "3     -0.600323     -0.827452      0.390838  \n",
      "4     -0.329325     -0.953249     -0.125917  \n",
      "(576, 13)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'logistic', 'hidden_layer_sizes': (30,), 'learning_rate_init': 0.007, 'max_iter': 500}, que permiten obtener un Accuracy de 67.82% y un Kappa del 7.15\n",
      "Model: \"model_18\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_18 (InputLayer)        (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_50 (Dense)             (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_51 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 451\n",
      "Trainable params: 451\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 500\n",
      "144/144 [==============================] - 0s 42us/step\n",
      "test loss: 0.6109174754884508, test accuracy: 0.6666666865348816\n",
      "AUC ROC:  0.6595744680851063\n",
      "Kappa:  0.10559006211180133\n",
      "[[89  5]\n",
      " [43  7]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Gramma y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0    -0.674917     0.169246     0.673543     1.157142    -0.633186   \n",
      "1     0.277269     0.514176     0.200398     0.988939    -1.756594   \n",
      "2     1.483921     0.724793     0.473099     0.439577    -0.358096   \n",
      "3    -0.734008    -0.683844    -0.764866    -0.225060    -0.261235   \n",
      "4    -0.834815    -0.735908    -1.177596    -0.093532     0.508050   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0     0.688145     0.215883    -0.452048     1.101066      0.064017   \n",
      "1    -0.022788    -0.235704     0.523508    -0.604231      1.188209   \n",
      "2    -0.452581    -0.213173    -0.596057    -0.767473      0.696227   \n",
      "3    -0.243429     0.588768     0.874148     1.302526      0.091256   \n",
      "4     0.503458     1.380798     1.847226     1.227896      0.017729   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0     -0.153703      1.751289      0.812723  \n",
      "1      0.863617     -0.801768      0.229305  \n",
      "2     -0.111259     -0.370649     -1.325817  \n",
      "3     -0.600323     -0.827452      0.390838  \n",
      "4     -0.329325     -0.953249     -0.125917  \n",
      "(651, 13)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (300) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30,), 'learning_rate_init': 0.002, 'max_iter': 300}, que permiten obtener un Accuracy de 68.44% y un Kappa del 27.07\n",
      "Model: \"model_19\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_19 (InputLayer)        (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_52 (Dense)             (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_53 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 451\n",
      "Trainable params: 451\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 300\n",
      "163/163 [==============================] - 0s 43us/step\n",
      "test loss: 0.6811560271707781, test accuracy: 0.5828220844268799\n",
      "AUC ROC:  0.5981424148606811\n",
      "Kappa:  0.03751302535602652\n",
      "[[87  8]\n",
      " [60  8]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Hotel Marrakech y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0     0.221235     1.617887     0.929874    -0.231486    -0.525862   \n",
      "1     0.836735    -0.529605    -1.268139    -0.791053     0.815880   \n",
      "2    -0.190995     1.202756     0.050028    -2.631154     3.701544   \n",
      "3     0.521202     1.354284     1.423683    -0.634173     0.934734   \n",
      "4     0.250234     1.586078    -1.791096     0.127156     1.573000   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0     1.384826     0.709441     0.512679    -2.231286     -2.278872   \n",
      "1    -1.992230    -0.371430    -0.356669     1.323871      0.946394   \n",
      "2    -1.158173     0.439586     2.317548    -2.282526     -1.571775   \n",
      "3     0.214772    -0.349135     1.009101    -2.193012     -0.301254   \n",
      "4     0.288525     1.962471     1.500627     1.352853     -1.921935   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0     -0.728806     -2.187766     -1.206544  \n",
      "1     -1.085097      0.673490     -1.496313  \n",
      "2     -2.541951     -2.587380     -2.132445  \n",
      "3     -0.356046     -0.668937     -0.421263  \n",
      "4      0.705405     -0.230103     -0.803009  \n",
      "(579, 13)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30, 30, 30), 'learning_rate_init': 0.009, 'max_iter': 400}, que permiten obtener un Accuracy de 66.13% y un Kappa del 28.79\n",
      "Model: \"model_20\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_20 (InputLayer)        (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_54 (Dense)             (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_55 (Dense)             (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_56 (Dense)             (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_57 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 2,311\n",
      "Trainable params: 2,311\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 400\n",
      "145/145 [==============================] - 0s 34us/step\n",
      "test loss: 1.6027446672834198, test accuracy: 0.6758620738983154\n",
      "AUC ROC:  0.7168021680216802\n",
      "Kappa:  0.33918355473674\n",
      "[[59 23]\n",
      " [24 39]]\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import display, Markdown, Latex\n",
    "import itertools\n",
    "three_combs = list(itertools.combinations(range(len(companies)), 3))\n",
    "for comb in three_combs:\n",
    "    title = '## '\n",
    "    for no in comb[:-1]:\n",
    "        title+=companies[no]\n",
    "        title+=' y '\n",
    "    title+=companies[comb[-1]]\n",
    "    display(Markdown(title))\n",
    "    X = df_n_ps_std_mfcc[comb[0]]\n",
    "    y = df_n_ps[comb[0]]['chosen']\n",
    "    for no in comb[1:]:\n",
    "        X= X.append(df_n_ps_std_mfcc[no])\n",
    "        y= y.append(df_n_ps[no]['chosen'])\n",
    "    \n",
    "\n",
    "    print(X.head())\n",
    "    print(X.shape)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "    mlp = MLPClassifier(hidden_layer_sizes=(30,30,30))\n",
    "\n",
    "    activation_vec = ['logistic', 'relu', 'tanh']\n",
    "    max_iter_vec = [10, 20, 50, 75, 100, 200, 300, 400, 500, 1000, 2000]\n",
    "    hidden_layer_sizes_vec = [(10,), (20,), (30,), (10, 10), (20, 20), (30, 30), (20, 10), \n",
    "                              (10, 10, 10), (20, 20, 20), (30, 30, 30), (30, 20, 10)]\n",
    "    learning_rate_init_vec = [0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.02]\n",
    "    batch_size_vec = [10, 20, 40, 60, 80, 100, 150]\n",
    "\n",
    "\n",
    "    np.random.seed(1234)\n",
    "    parametros = {'activation': activation_vec,\n",
    "                  'max_iter':max_iter_vec,\n",
    "                  'hidden_layer_sizes': hidden_layer_sizes_vec,\n",
    "                  'learning_rate_init': learning_rate_init_vec#,\n",
    "                  #'batch_size': batch_size_vec\n",
    "                  }\n",
    "    scoring = {'kappa':make_scorer(cohen_kappa_score), 'accuracy':'accuracy'}\n",
    "    grid = GridSearchCV(mlp, param_grid=parametros, cv=5, scoring=scoring, refit='accuracy', n_jobs=-1, iid=True)\n",
    "\n",
    "    grid.fit(X_train, y_train)\n",
    "\n",
    "    print(\"Los parámetros del mejor modelo fueron {0}, que permiten obtener un Accuracy de {1:.2f}% y un Kappa del {2:.2f}\".format(\n",
    "        grid.best_params_, grid.best_score_*100, grid.cv_results_['mean_test_kappa'][grid.best_index_]*100))\n",
    "\n",
    "    n0=X_train.shape[1]\n",
    "    ### hidden_layer_sizes\n",
    "    ns = []\n",
    "    for i in range (len(grid.best_params_['hidden_layer_sizes'])):\n",
    "        ns.append(grid.best_params_['hidden_layer_sizes'][i])\n",
    "\n",
    "    ns.append(1)\n",
    "    lr = grid.best_params_['learning_rate_init']\n",
    "    epochs = grid.best_params_['max_iter']\n",
    "\n",
    "    input_tensor = Input(shape = (n0,))\n",
    "    hidden_outputs = [input_tensor]\n",
    "    if grid.best_params_['activation'] == 'logistic':\n",
    "        grid.best_params_['activation'] = 'sigmoid'\n",
    "    for i in range (len(ns)-1):\n",
    "        hidden_outputs.append(Dense(ns[i], activation = grid.best_params_['activation'])(hidden_outputs[i]))\n",
    "    classification_output = Dense(ns[-1], activation = 'sigmoid')(hidden_outputs[-1])\n",
    "\n",
    "    model = Model([input_tensor], [classification_output])\n",
    "    weights = model.get_weights()\n",
    "\n",
    "    print(model.summary())\n",
    "\n",
    "    model.set_weights(weights)\n",
    "    adam = keras.optimizers.Adam(lr=lr)\n",
    "    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    history = model.fit(X_train, y_train, epochs=epochs, validation_data=(X_test, y_test), verbose=0, \n",
    "                callbacks=[\n",
    "                    keras.callbacks.ReduceLROnPlateau(\n",
    "                        monitor='val_accuracy', factor=0.5, patience=10, min_delta=0.01, verbose=0\n",
    "                    )\n",
    "                ]\n",
    "             )\n",
    "\n",
    "    acc = history.history['accuracy']\n",
    "    val_acc = history.history['val_accuracy']\n",
    "    loss = history.history['loss']\n",
    "    val_loss = history.history['val_loss']\n",
    "\n",
    "    epochs = range(len(acc))\n",
    "\n",
    "    print(\"epochs: \"+str(len(acc)))\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(X_test, y_test)\n",
    "    print(\"test loss: {}, test accuracy: {}\".format(test_loss, test_acc))\n",
    "\n",
    "    y_pred = model.predict(X_test)\n",
    "    print(\"AUC ROC: \",roc_auc_score(y_test, y_pred))\n",
    "\n",
    "    y_pred = list(map(lambda i: int(i>=0.5), y_pred))\n",
    "    print(\"Kappa: \",cohen_kappa_score(y_test, y_pred))\n",
    "\n",
    "    print(confusion_matrix(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5-comp groups combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Gramma y Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0     0.297583     1.225637    -0.367641     0.606499     0.072373   \n",
      "1     0.637676    -1.507256    -1.572737    -0.954161    -0.857425   \n",
      "2     2.236730    -0.319414     0.669910    -1.918119    -0.820882   \n",
      "3     0.662077    -0.381499     0.111981    -1.743808    -1.317593   \n",
      "4     0.736502     0.112932    -0.065024    -1.049458    -0.408043   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0    -2.029620     0.791469     0.752018     2.268802     -1.383289   \n",
      "1     0.327005     0.816764     0.214245     0.241703      0.637066   \n",
      "2    -2.379333    -1.570021    -2.755344    -2.150610     -2.528577   \n",
      "3    -1.348534    -0.627198    -1.629882    -2.075974     -1.248765   \n",
      "4    -0.437499     0.090831    -0.852983    -1.922491     -0.284365   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0      0.548279      1.903211     -1.011470  \n",
      "1      1.601538      0.300317     -0.466779  \n",
      "2     -0.877081     -0.522248     -1.429911  \n",
      "3     -1.126014     -1.316359     -1.126174  \n",
      "4      0.210624     -0.032122     -0.700183  \n",
      "(1149, 13)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30, 20, 10), 'learning_rate_init': 0.003, 'max_iter': 300}, que permiten obtener un Accuracy de 68.76% y un Kappa del 29.93\n",
      "Model: \"model_118\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_118 (InputLayer)       (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_354 (Dense)            (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_355 (Dense)            (None, 20)                620       \n",
      "_________________________________________________________________\n",
      "dense_356 (Dense)            (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_357 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 1,261\n",
      "Trainable params: 1,261\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 300\n",
      "288/288 [==============================] - 0s 193us/step\n",
      "test loss: 0.7107465730773078, test accuracy: 0.6319444179534912\n",
      "AUC ROC:  0.5935739387516037\n",
      "Kappa:  0.11178353215013093\n",
      "[[151  46]\n",
      " [ 60  31]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Gramma y Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0     0.297583     1.225637    -0.367641     0.606499     0.072373   \n",
      "1     0.637676    -1.507256    -1.572737    -0.954161    -0.857425   \n",
      "2     2.236730    -0.319414     0.669910    -1.918119    -0.820882   \n",
      "3     0.662077    -0.381499     0.111981    -1.743808    -1.317593   \n",
      "4     0.736502     0.112932    -0.065024    -1.049458    -0.408043   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0    -2.029620     0.791469     0.752018     2.268802     -1.383289   \n",
      "1     0.327005     0.816764     0.214245     0.241703      0.637066   \n",
      "2    -2.379333    -1.570021    -2.755344    -2.150610     -2.528577   \n",
      "3    -1.348534    -0.627198    -1.629882    -2.075974     -1.248765   \n",
      "4    -0.437499     0.090831    -0.852983    -1.922491     -0.284365   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0      0.548279      1.903211     -1.011470  \n",
      "1      1.601538      0.300317     -0.466779  \n",
      "2     -0.877081     -0.522248     -1.429911  \n",
      "3     -1.126014     -1.316359     -1.126174  \n",
      "4      0.210624     -0.032122     -0.700183  \n",
      "(1146, 13)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30,), 'learning_rate_init': 0.003, 'max_iter': 100}, que permiten obtener un Accuracy de 69.85% y un Kappa del 14.32\n",
      "Model: \"model_119\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_119 (InputLayer)       (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_358 (Dense)            (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_359 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 451\n",
      "Trainable params: 451\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 100\n",
      "287/287 [==============================] - 0s 43us/step\n",
      "test loss: 0.6095467818323328, test accuracy: 0.693379819393158\n",
      "AUC ROC:  0.5735984048791931\n",
      "Kappa:  0.07153885743695321\n",
      "[[188  15]\n",
      " [ 73  11]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Gramma y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0     0.297583     1.225637    -0.367641     0.606499     0.072373   \n",
      "1     0.637676    -1.507256    -1.572737    -0.954161    -0.857425   \n",
      "2     2.236730    -0.319414     0.669910    -1.918119    -0.820882   \n",
      "3     0.662077    -0.381499     0.111981    -1.743808    -1.317593   \n",
      "4     0.736502     0.112932    -0.065024    -1.049458    -0.408043   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0    -2.029620     0.791469     0.752018     2.268802     -1.383289   \n",
      "1     0.327005     0.816764     0.214245     0.241703      0.637066   \n",
      "2    -2.379333    -1.570021    -2.755344    -2.150610     -2.528577   \n",
      "3    -1.348534    -0.627198    -1.629882    -2.075974     -1.248765   \n",
      "4    -0.437499     0.090831    -0.852983    -1.922491     -0.284365   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0      0.548279      1.903211     -1.011470  \n",
      "1      1.601538      0.300317     -0.466779  \n",
      "2     -0.877081     -0.522248     -1.429911  \n",
      "3     -1.126014     -1.316359     -1.126174  \n",
      "4      0.210624     -0.032122     -0.700183  \n",
      "(1221, 13)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (10,), 'learning_rate_init': 0.005, 'max_iter': 50}, que permiten obtener un Accuracy de 69.95% y un Kappa del 14.91\n",
      "Model: \"model_120\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_120 (InputLayer)       (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_360 (Dense)            (None, 10)                140       \n",
      "_________________________________________________________________\n",
      "dense_361 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 151\n",
      "Trainable params: 151\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 50\n",
      "306/306 [==============================] - 0s 20us/step\n",
      "test loss: 0.6717599630355835, test accuracy: 0.6339869499206543\n",
      "AUC ROC:  0.545661994132166\n",
      "Kappa:  0.04080604534005028\n",
      "[[182  15]\n",
      " [ 97  12]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Hotel Marrakech y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0     0.297583     1.225637    -0.367641     0.606499     0.072373   \n",
      "1     0.637676    -1.507256    -1.572737    -0.954161    -0.857425   \n",
      "2     2.236730    -0.319414     0.669910    -1.918119    -0.820882   \n",
      "3     0.662077    -0.381499     0.111981    -1.743808    -1.317593   \n",
      "4     0.736502     0.112932    -0.065024    -1.049458    -0.408043   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0    -2.029620     0.791469     0.752018     2.268802     -1.383289   \n",
      "1     0.327005     0.816764     0.214245     0.241703      0.637066   \n",
      "2    -2.379333    -1.570021    -2.755344    -2.150610     -2.528577   \n",
      "3    -1.348534    -0.627198    -1.629882    -2.075974     -1.248765   \n",
      "4    -0.437499     0.090831    -0.852983    -1.922491     -0.284365   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0      0.548279      1.903211     -1.011470  \n",
      "1      1.601538      0.300317     -0.466779  \n",
      "2     -0.877081     -0.522248     -1.429911  \n",
      "3     -1.126014     -1.316359     -1.126174  \n",
      "4      0.210624     -0.032122     -0.700183  \n",
      "(1149, 13)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20, 20, 20), 'learning_rate_init': 0.009, 'max_iter': 100}, que permiten obtener un Accuracy de 65.74% y un Kappa del 24.58\n",
      "Model: \"model_121\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_121 (InputLayer)       (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_362 (Dense)            (None, 20)                280       \n",
      "_________________________________________________________________\n",
      "dense_363 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_364 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_365 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 1,141\n",
      "Trainable params: 1,141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 100\n",
      "288/288 [==============================] - 0s 67us/step\n",
      "test loss: 1.5485331747266982, test accuracy: 0.6458333134651184\n",
      "AUC ROC:  0.6237143696738172\n",
      "Kappa:  0.17801779618333424\n",
      "[[147  58]\n",
      " [ 44  39]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma y Hotel Marrakech y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0     0.297583     1.225637    -0.367641     0.606499     0.072373   \n",
      "1     0.637676    -1.507256    -1.572737    -0.954161    -0.857425   \n",
      "2     2.236730    -0.319414     0.669910    -1.918119    -0.820882   \n",
      "3     0.662077    -0.381499     0.111981    -1.743808    -1.317593   \n",
      "4     0.736502     0.112932    -0.065024    -1.049458    -0.408043   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0    -2.029620     0.791469     0.752018     2.268802     -1.383289   \n",
      "1     0.327005     0.816764     0.214245     0.241703      0.637066   \n",
      "2    -2.379333    -1.570021    -2.755344    -2.150610     -2.528577   \n",
      "3    -1.348534    -0.627198    -1.629882    -2.075974     -1.248765   \n",
      "4    -0.437499     0.090831    -0.852983    -1.922491     -0.284365   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0      0.548279      1.903211     -1.011470  \n",
      "1      1.601538      0.300317     -0.466779  \n",
      "2     -0.877081     -0.522248     -1.429911  \n",
      "3     -1.126014     -1.316359     -1.126174  \n",
      "4      0.210624     -0.032122     -0.700183  \n",
      "(1110, 13)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'logistic', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.004, 'max_iter': 200}, que permiten obtener un Accuracy de 65.87% y un Kappa del 4.22\n",
      "Model: \"model_122\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_122 (InputLayer)       (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_366 (Dense)            (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_367 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_368 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,381\n",
      "Trainable params: 1,381\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 200\n",
      "278/278 [==============================] - 0s 79us/step\n",
      "test loss: 0.6395446550931861, test accuracy: 0.6582733988761902\n",
      "AUC ROC:  0.5623238423928674\n",
      "Kappa:  0.025964446411448128\n",
      "[[179   4]\n",
      " [ 91   4]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma y Hotel Marrakech y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   mfccfiles_1  mfccfiles_2  mfccfiles_3  mfccfiles_4  mfccfiles_5  \\\n",
      "0    -0.339415     0.847773     0.497198    -0.389310     1.225458   \n",
      "1     0.587658    -1.195426     0.636375     0.199876     0.765321   \n",
      "2     1.465595    -2.307943     0.354567    -0.058273    -1.298853   \n",
      "3     0.749403    -1.690498    -0.125200    -1.016135     0.825845   \n",
      "4    -0.280577     0.393332     0.744917     2.411400    -0.777421   \n",
      "\n",
      "   mfccfiles_6  mfccfiles_7  mfccfiles_8  mfccfiles_9  mfccfiles_10  \\\n",
      "0     1.947033    -0.736267     0.492219     0.576682      1.504697   \n",
      "1     0.061181     0.379367    -0.440867     0.232893      1.339920   \n",
      "2    -0.811453    -1.551580    -3.934320    -1.079432      2.546130   \n",
      "3     0.271444    -0.104786    -0.992141     0.049182      1.425948   \n",
      "4    -0.420018     1.258355    -1.544565    -0.498071      0.421527   \n",
      "\n",
      "   mfccfiles_11  mfccfiles_12  mfccfiles_13  \n",
      "0     -1.796460      0.724954      0.958600  \n",
      "1      0.110001      0.807525      0.815678  \n",
      "2      1.421407      0.639359      0.199094  \n",
      "3     -0.343269     -0.789558     -0.411898  \n",
      "4     -0.632908     -0.056846     -0.072348  \n",
      "(1050, 13)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30, 30), 'learning_rate_init': 0.008, 'max_iter': 2000}, que permiten obtener un Accuracy de 68.61% y un Kappa del 24.39\n",
      "Model: \"model_123\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_123 (InputLayer)       (None, 13)                0         \n",
      "_________________________________________________________________\n",
      "dense_369 (Dense)            (None, 30)                420       \n",
      "_________________________________________________________________\n",
      "dense_370 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_371 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_372 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 2,311\n",
      "Trainable params: 2,311\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 2000\n",
      "263/263 [==============================] - 0s 72us/step\n",
      "test loss: 3.171573793026884, test accuracy: 0.5969581604003906\n",
      "AUC ROC:  0.593746118494597\n",
      "Kappa:  0.1075613035405597\n",
      "[[120  46]\n",
      " [ 60  37]]\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import display, Markdown, Latex\n",
    "import itertools\n",
    "three_combs = list(itertools.combinations(range(len(companies)), 5))\n",
    "for comb in three_combs:\n",
    "    title = '## '\n",
    "    for no in comb[:-1]:\n",
    "        title+=companies[no]\n",
    "        title+=' y '\n",
    "    title+=companies[comb[-1]]\n",
    "    display(Markdown(title))\n",
    "    X = df_n_ps_std_mfcc[comb[0]]\n",
    "    y = df_n_ps[comb[0]]['chosen']\n",
    "    for no in comb[1:]:\n",
    "        X= X.append(df_n_ps_std_mfcc[no])\n",
    "        y= y.append(df_n_ps[no]['chosen'])\n",
    "    \n",
    "\n",
    "    print(X.head())\n",
    "    print(X.shape)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "    mlp = MLPClassifier(hidden_layer_sizes=(30,30,30))\n",
    "\n",
    "    activation_vec = ['logistic', 'relu', 'tanh']\n",
    "    max_iter_vec = [10, 20, 50, 75, 100, 200, 300, 400, 500, 1000, 2000]\n",
    "    hidden_layer_sizes_vec = [(10,), (20,), (30,), (10, 10), (20, 20), (30, 30), (20, 10), \n",
    "                              (10, 10, 10), (20, 20, 20), (30, 30, 30), (30, 20, 10)]\n",
    "    learning_rate_init_vec = [0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.02]\n",
    "    batch_size_vec = [10, 20, 40, 60, 80, 100, 150]\n",
    "\n",
    "\n",
    "    np.random.seed(1234)\n",
    "    parametros = {'activation': activation_vec,\n",
    "                  'max_iter':max_iter_vec,\n",
    "                  'hidden_layer_sizes': hidden_layer_sizes_vec,\n",
    "                  'learning_rate_init': learning_rate_init_vec#,\n",
    "                  #'batch_size': batch_size_vec\n",
    "                  }\n",
    "    scoring = {'kappa':make_scorer(cohen_kappa_score), 'accuracy':'accuracy'}\n",
    "    grid = GridSearchCV(mlp, param_grid=parametros, cv=5, scoring=scoring, refit='accuracy', n_jobs=-1, iid=True)\n",
    "\n",
    "    grid.fit(X_train, y_train)\n",
    "\n",
    "    print(\"Los parámetros del mejor modelo fueron {0}, que permiten obtener un Accuracy de {1:.2f}% y un Kappa del {2:.2f}\".format(\n",
    "        grid.best_params_, grid.best_score_*100, grid.cv_results_['mean_test_kappa'][grid.best_index_]*100))\n",
    "\n",
    "    n0=X_train.shape[1]\n",
    "    ### hidden_layer_sizes\n",
    "    ns = []\n",
    "    for i in range (len(grid.best_params_['hidden_layer_sizes'])):\n",
    "        ns.append(grid.best_params_['hidden_layer_sizes'][i])\n",
    "\n",
    "    ns.append(1)\n",
    "    lr = grid.best_params_['learning_rate_init']\n",
    "    epochs = grid.best_params_['max_iter']\n",
    "\n",
    "    input_tensor = Input(shape = (n0,))\n",
    "    hidden_outputs = [input_tensor]\n",
    "    if grid.best_params_['activation'] == 'logistic':\n",
    "        grid.best_params_['activation'] = 'sigmoid'\n",
    "    for i in range (len(ns)-1):\n",
    "        hidden_outputs.append(Dense(ns[i], activation = grid.best_params_['activation'])(hidden_outputs[i]))\n",
    "    classification_output = Dense(ns[-1], activation = 'sigmoid')(hidden_outputs[-1])\n",
    "\n",
    "    model = Model([input_tensor], [classification_output])\n",
    "    weights = model.get_weights()\n",
    "\n",
    "    print(model.summary())\n",
    "\n",
    "    model.set_weights(weights)\n",
    "    adam = keras.optimizers.Adam(lr=lr)\n",
    "    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    history = model.fit(X_train, y_train, epochs=epochs, validation_data=(X_test, y_test), verbose=0, \n",
    "                callbacks=[\n",
    "                    keras.callbacks.ReduceLROnPlateau(\n",
    "                        monitor='val_accuracy', factor=0.5, patience=10, min_delta=0.01, verbose=0\n",
    "                    )\n",
    "                ]\n",
    "             )\n",
    "\n",
    "    acc = history.history['accuracy']\n",
    "    val_acc = history.history['val_accuracy']\n",
    "    loss = history.history['loss']\n",
    "    val_loss = history.history['val_loss']\n",
    "\n",
    "    epochs = range(len(acc))\n",
    "\n",
    "    print(\"epochs: \"+str(len(acc)))\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(X_test, y_test)\n",
    "    print(\"test loss: {}, test accuracy: {}\".format(test_loss, test_acc))\n",
    "\n",
    "    y_pred = model.predict(X_test)\n",
    "    print(\"AUC ROC: \",roc_auc_score(y_test, y_pred))\n",
    "\n",
    "    y_pred = list(map(lambda i: int(i>=0.5), y_pred))\n",
    "    print(\"Kappa: \",cohen_kappa_score(y_test, y_pred))\n",
    "\n",
    "    print(confusion_matrix(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tonal Centroid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['durationfiles', 'rmsfiles', 'rmsmedianfiles', 'lowenergyfiles',\n",
       "       'ASRfiles', 'beatspectrumfiles', 'eventdensityfiles', 'tempofiles',\n",
       "       'pulseclarityfiles', 'zerocrossfiles', 'rolloffsfiles',\n",
       "       'brightnessfiles', 'spreadfiles', 'centroidfiles', 'kurtosisfiles',\n",
       "       'flatnessfiles', 'entropyfiles', 'mfccfiles_1', 'mfccfiles_2',\n",
       "       'mfccfiles_3', 'mfccfiles_4', 'mfccfiles_5', 'mfccfiles_6',\n",
       "       'mfccfiles_7', 'mfccfiles_8', 'mfccfiles_9', 'mfccfiles_10',\n",
       "       'mfccfiles_11', 'mfccfiles_12', 'mfccfiles_13', 'inharmonicityfiles',\n",
       "       'bestkeyfiles', 'keyclarityfiles', 'modalityfiles',\n",
       "       'tonalcentroidfiles_1', 'tonalcentroidfiles_2', 'tonalcentroidfiles_3',\n",
       "       'tonalcentroidfiles_4', 'tonalcentroidfiles_5', 'tonalcentroidfiles_6',\n",
       "       'chromagramfiles_1', 'chromagramfiles_2', 'chromagramfiles_3',\n",
       "       'chromagramfiles_4', 'chromagramfiles_5', 'chromagramfiles_6',\n",
       "       'chromagramfiles_7', 'chromagramfiles_8', 'chromagramfiles_9',\n",
       "       'chromagramfiles_10', 'chromagramfiles_11', 'chromagramfiles_12',\n",
       "       'attackslopefiles', 'attackleapfiles', 'chosen'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_n_ps_std[0].columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['tonalcentroidfiles_1', 'tonalcentroidfiles_2', 'tonalcentroidfiles_3',\n",
       "       'tonalcentroidfiles_4', 'tonalcentroidfiles_5', 'tonalcentroidfiles_6'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_n_ps_std[0].columns[34:40]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 315 entries, 0 to 314\n",
      "Data columns (total 6 columns):\n",
      "tonalcentroidfiles_1    315 non-null float64\n",
      "tonalcentroidfiles_2    315 non-null float64\n",
      "tonalcentroidfiles_3    315 non-null float64\n",
      "tonalcentroidfiles_4    315 non-null float64\n",
      "tonalcentroidfiles_5    315 non-null float64\n",
      "tonalcentroidfiles_6    315 non-null float64\n",
      "dtypes: float64(6)\n",
      "memory usage: 14.8 KB\n"
     ]
    }
   ],
   "source": [
    "df_n_ps_std_tc = [None]*len(companies)\n",
    "for i in range(len(companies)):\n",
    "    df_n_ps_std_tc[i] = pd.DataFrame(df_n_ps_std[i].iloc[:,34:40])\n",
    "    df_n_ps_std_tc[i].columns=df_n_ps_std[i].columns[34:40]\n",
    "df_n_ps_std_tc[0].info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2-comp groups combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30, 20, 10), 'learning_rate_init': 0.009, 'max_iter': 20}, que permiten obtener un Accuracy de 74.24% y un Kappa del 11.33\n",
      "Model: \"model_76\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_76 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_226 (Dense)            (None, 30)                210       \n",
      "_________________________________________________________________\n",
      "dense_227 (Dense)            (None, 20)                620       \n",
      "_________________________________________________________________\n",
      "dense_228 (Dense)            (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_229 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 1,051\n",
      "Trainable params: 1,051\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 20\n",
      "143/143 [==============================] - 0s 3ms/step\n",
      "test loss: 0.6188117474109143, test accuracy: 0.6783216595649719\n",
      "AUC ROC:  0.569583931133429\n",
      "Kappa:  -0.006734006734006703\n",
      "[[94  8]\n",
      " [38  3]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (400) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (10,), 'learning_rate_init': 0.008, 'max_iter': 400}, que permiten obtener un Accuracy de 71.61% y un Kappa del 21.68\n",
      "Model: \"model_77\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_77 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_230 (Dense)            (None, 10)                70        \n",
      "_________________________________________________________________\n",
      "dense_231 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 81\n",
      "Trainable params: 81\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 400\n",
      "133/133 [==============================] - 0s 53us/step\n",
      "test loss: 0.5738246676168943, test accuracy: 0.7293233275413513\n",
      "AUC ROC:  0.536509900990099\n",
      "Kappa:  0.004159733777038266\n",
      "[[95  6]\n",
      " [30  2]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Hotel Marrakech"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (75) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'logistic', 'hidden_layer_sizes': (10, 10), 'learning_rate_init': 0.001, 'max_iter': 75}, que permiten obtener un Accuracy de 69.19% y un Kappa del 4.57\n",
      "Model: \"model_78\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_78 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_232 (Dense)            (None, 10)                70        \n",
      "_________________________________________________________________\n",
      "dense_233 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_234 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 191\n",
      "Trainable params: 191\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 75\n",
      "115/115 [==============================] - 0s 52us/step\n",
      "test loss: 0.628456962108612, test accuracy: 0.678260862827301\n",
      "AUC ROC:  0.5100485100485102\n",
      "Kappa:  0.0\n",
      "[[78  0]\n",
      " [37  0]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.007, 'max_iter': 400}, que permiten obtener un Accuracy de 67.00% y un Kappa del 26.90\n",
      "Model: \"model_79\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_79 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_235 (Dense)            (None, 30)                210       \n",
      "_________________________________________________________________\n",
      "dense_236 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_237 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,171\n",
      "Trainable params: 1,171\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 400\n",
      "134/134 [==============================] - 0s 37us/step\n",
      "test loss: 0.6912476874109524, test accuracy: 0.6567164063453674\n",
      "AUC ROC:  0.6013637555730396\n",
      "Kappa:  0.13256403039684772\n",
      "[[75 18]\n",
      " [28 13]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30,), 'learning_rate_init': 0.006, 'max_iter': 20}, que permiten obtener un Accuracy de 71.61% y un Kappa del 14.01\n",
      "Model: \"model_80\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_80 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_238 (Dense)            (None, 30)                210       \n",
      "_________________________________________________________________\n",
      "dense_239 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 241\n",
      "Trainable params: 241\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 20\n",
      "133/133 [==============================] - 0s 342us/step\n",
      "test loss: 0.5433918479689979, test accuracy: 0.7067669034004211\n",
      "AUC ROC:  0.7081901489117984\n",
      "Kappa:  0.07059666726393132\n",
      "[[89  8]\n",
      " [31  5]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.898091              0.151819             -1.172713   \n",
      "1              0.618513             -0.762588              0.061946   \n",
      "2              0.685649              0.002933              0.719805   \n",
      "3              1.175209             -0.552349              0.336427   \n",
      "4              1.350337             -1.407757              0.258917   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              0.474387             -0.020230              1.228657  \n",
      "1              0.944076              0.697880              0.021150  \n",
      "2             -1.251700             -0.952424              1.444556  \n",
      "3              0.482978             -0.212146             -0.144225  \n",
      "4             -0.523670              0.099306              1.706064  \n",
      "Los parámetros del mejor modelo fueron {'activation': 'logistic', 'hidden_layer_sizes': (30, 30, 30), 'learning_rate_init': 0.02, 'max_iter': 500}, que permiten obtener un Accuracy de 73.09% y un Kappa del 4.70\n",
      "Model: \"model_81\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_81 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_240 (Dense)            (None, 30)                210       \n",
      "_________________________________________________________________\n",
      "dense_241 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_242 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_243 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 2,101\n",
      "Trainable params: 2,101\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 500\n",
      "118/118 [==============================] - 0s 478us/step\n",
      "test loss: 0.6341226717173043, test accuracy: 0.6864407062530518\n",
      "AUC ROC:  0.4621287954621288\n",
      "Kappa:  0.0\n",
      "[[81  0]\n",
      " [37  0]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Hotel Marrakech"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.898091              0.151819             -1.172713   \n",
      "1              0.618513             -0.762588              0.061946   \n",
      "2              0.685649              0.002933              0.719805   \n",
      "3              1.175209             -0.552349              0.336427   \n",
      "4              1.350337             -1.407757              0.258917   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              0.474387             -0.020230              1.228657  \n",
      "1              0.944076              0.697880              0.021150  \n",
      "2             -1.251700             -0.952424              1.444556  \n",
      "3              0.482978             -0.212146             -0.144225  \n",
      "4             -0.523670              0.099306              1.706064  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (300) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (20, 20), 'learning_rate_init': 0.001, 'max_iter': 300}, que permiten obtener un Accuracy de 71.24% y un Kappa del 19.29\n",
      "Model: \"model_82\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_82 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_244 (Dense)            (None, 20)                140       \n",
      "_________________________________________________________________\n",
      "dense_245 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_246 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 581\n",
      "Trainable params: 581\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 300\n",
      "100/100 [==============================] - 0s 156us/step\n",
      "test loss: 0.6810387420654297, test accuracy: 0.6499999761581421\n",
      "AUC ROC:  0.5298155298155298\n",
      "Kappa:  0.12761714855433703\n",
      "[[58  5]\n",
      " [30  7]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.898091              0.151819             -1.172713   \n",
      "1              0.618513             -0.762588              0.061946   \n",
      "2              0.685649              0.002933              0.719805   \n",
      "3              1.175209             -0.552349              0.336427   \n",
      "4              1.350337             -1.407757              0.258917   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              0.474387             -0.020230              1.228657  \n",
      "1              0.944076              0.697880              0.021150  \n",
      "2             -1.251700             -0.952424              1.444556  \n",
      "3              0.482978             -0.212146             -0.144225  \n",
      "4             -0.523670              0.099306              1.706064  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (10, 10, 10), 'learning_rate_init': 0.005, 'max_iter': 100}, que permiten obtener un Accuracy de 65.63% y un Kappa del 26.11\n",
      "Model: \"model_83\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_83 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_247 (Dense)            (None, 10)                70        \n",
      "_________________________________________________________________\n",
      "dense_248 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_249 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_250 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 301\n",
      "Trainable params: 301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 100\n",
      "119/119 [==============================] - 0s 0us/step\n",
      "test loss: 0.6054192986808905, test accuracy: 0.6134454011917114\n",
      "AUC ROC:  0.6615384615384615\n",
      "Kappa:  0.06106346483704972\n",
      "[[62 18]\n",
      " [28 11]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.898091              0.151819             -1.172713   \n",
      "1              0.618513             -0.762588              0.061946   \n",
      "2              0.685649              0.002933              0.719805   \n",
      "3              1.175209             -0.552349              0.336427   \n",
      "4              1.350337             -1.407757              0.258917   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              0.474387             -0.020230              1.228657  \n",
      "1              0.944076              0.697880              0.021150  \n",
      "2             -1.251700             -0.952424              1.444556  \n",
      "3              0.482978             -0.212146             -0.144225  \n",
      "4             -0.523670              0.099306              1.706064  \n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (20, 20, 20), 'learning_rate_init': 0.02, 'max_iter': 500}, que permiten obtener un Accuracy de 71.95% y un Kappa del 30.44\n",
      "Model: \"model_84\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_84 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_251 (Dense)            (None, 20)                140       \n",
      "_________________________________________________________________\n",
      "dense_252 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_253 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_254 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 1,001\n",
      "Trainable params: 1,001\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 500\n",
      "118/118 [==============================] - 0s 51us/step\n",
      "test loss: 0.7824493232419936, test accuracy: 0.6101694703102112\n",
      "AUC ROC:  0.5108552631578948\n",
      "Kappa:  0.011653313911143437\n",
      "[[64 16]\n",
      " [30  8]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Gramma y Hotel Marrakech"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.618349              0.564005              0.091611   \n",
      "1             -0.149103             -1.303101             -0.498811   \n",
      "2             -1.141294             -1.317570              0.575363   \n",
      "3             -0.988346             -0.540855              1.006800   \n",
      "4             -0.640925             -0.228256              0.461986   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.885208              0.563756              1.803892  \n",
      "1              0.416525             -0.660751             -1.179350  \n",
      "2             -1.560836              0.336340              1.197709  \n",
      "3             -0.214650             -0.364940              0.068869  \n",
      "4             -1.274446             -1.494581             -0.125833  \n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20, 20), 'learning_rate_init': 0.005, 'max_iter': 400}, que permiten obtener un Accuracy de 67.78% y un Kappa del 21.34\n",
      "Model: \"model_85\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_85 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_255 (Dense)            (None, 20)                140       \n",
      "_________________________________________________________________\n",
      "dense_256 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_257 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 581\n",
      "Trainable params: 581\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 400\n",
      "90/90 [==============================] - 0s 22us/step\n",
      "test loss: 0.6540486441718207, test accuracy: 0.6222222447395325\n",
      "AUC ROC:  0.5673133450911229\n",
      "Kappa:  -0.05590062111801242\n",
      "[[53 10]\n",
      " [24  3]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Gramma y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.618349              0.564005              0.091611   \n",
      "1             -0.149103             -1.303101             -0.498811   \n",
      "2             -1.141294             -1.317570              0.575363   \n",
      "3             -0.988346             -0.540855              1.006800   \n",
      "4             -0.640925             -0.228256              0.461986   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.885208              0.563756              1.803892  \n",
      "1              0.416525             -0.660751             -1.179350  \n",
      "2             -1.560836              0.336340              1.197709  \n",
      "3             -0.214650             -0.364940              0.068869  \n",
      "4             -1.274446             -1.494581             -0.125833  \n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (10, 10, 10), 'learning_rate_init': 0.02, 'max_iter': 300}, que permiten obtener un Accuracy de 64.42% y un Kappa del 22.36\n",
      "Model: \"model_86\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_86 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_258 (Dense)            (None, 10)                70        \n",
      "_________________________________________________________________\n",
      "dense_259 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_260 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_261 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 301\n",
      "Trainable params: 301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 300\n",
      "109/109 [==============================] - 0s 0us/step\n",
      "test loss: 0.9619177902510406, test accuracy: 0.4954128563404083\n",
      "AUC ROC:  0.4577114427860697\n",
      "Kappa:  -0.05120112221637729\n",
      "[[38 29]\n",
      " [26 16]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Gramma y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.618349              0.564005              0.091611   \n",
      "1             -0.149103             -1.303101             -0.498811   \n",
      "2             -1.141294             -1.317570              0.575363   \n",
      "3             -0.988346             -0.540855              1.006800   \n",
      "4             -0.640925             -0.228256              0.461986   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.885208              0.563756              1.803892  \n",
      "1              0.416525             -0.660751             -1.179350  \n",
      "2             -1.560836              0.336340              1.197709  \n",
      "3             -0.214650             -0.364940              0.068869  \n",
      "4             -1.274446             -1.494581             -0.125833  \n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (20,), 'learning_rate_init': 0.008, 'max_iter': 300}, que permiten obtener un Accuracy de 72.22% y un Kappa del 27.60\n",
      "Model: \"model_87\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_87 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_262 (Dense)            (None, 20)                140       \n",
      "_________________________________________________________________\n",
      "dense_263 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 161\n",
      "Trainable params: 161\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 300\n",
      "108/108 [==============================] - 0s 28us/step\n",
      "test loss: 0.6455202676631786, test accuracy: 0.6759259104728699\n",
      "AUC ROC:  0.5218600953895072\n",
      "Kappa:  0.04352226720647778\n",
      "[[70  4]\n",
      " [31  3]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.297447              0.231356              0.546295   \n",
      "1              0.766614             -1.227377             -0.605576   \n",
      "2             -0.431734             -1.183099              1.287522   \n",
      "3              0.078607             -1.572049              1.188424   \n",
      "4              0.007655             -1.466562              1.294323   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.007345              0.269632              0.236051  \n",
      "1             -1.216250              0.473081             -0.446139  \n",
      "2              0.927452             -0.309857             -1.666006  \n",
      "3              0.092330             -0.244414             -0.490911  \n",
      "4             -0.410616             -0.763440             -1.007284  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 20, 10), 'learning_rate_init': 0.003, 'max_iter': 100}, que permiten obtener un Accuracy de 61.03% y un Kappa del 20.53\n",
      "Model: \"model_88\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_88 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_264 (Dense)            (None, 30)                210       \n",
      "_________________________________________________________________\n",
      "dense_265 (Dense)            (None, 20)                620       \n",
      "_________________________________________________________________\n",
      "dense_266 (Dense)            (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_267 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 1,051\n",
      "Trainable params: 1,051\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 100\n",
      "91/91 [==============================] - 0s 172us/step\n",
      "test loss: 0.6024310569186787, test accuracy: 0.6373626589775085\n",
      "AUC ROC:  0.7359496124031008\n",
      "Kappa:  0.27164685908319186\n",
      "[[32 16]\n",
      " [17 26]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.297447              0.231356              0.546295   \n",
      "1              0.766614             -1.227377             -0.605576   \n",
      "2             -0.431734             -1.183099              1.287522   \n",
      "3              0.078607             -1.572049              1.188424   \n",
      "4              0.007655             -1.466562              1.294323   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.007345              0.269632              0.236051  \n",
      "1             -1.216250              0.473081             -0.446139  \n",
      "2              0.927452             -0.309857             -1.666006  \n",
      "3              0.092330             -0.244414             -0.490911  \n",
      "4             -0.410616             -0.763440             -1.007284  \n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30, 30), 'learning_rate_init': 0.004, 'max_iter': 1000}, que permiten obtener un Accuracy de 68.15% y un Kappa del 27.24\n",
      "Model: \"model_89\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_89 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_268 (Dense)            (None, 30)                210       \n",
      "_________________________________________________________________\n",
      "dense_269 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_270 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_271 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 2,101\n",
      "Trainable params: 2,101\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 1000\n",
      "90/90 [==============================] - 0s 33us/step\n",
      "test loss: 0.8504453420639038, test accuracy: 0.644444465637207\n",
      "AUC ROC:  0.7267410951621478\n",
      "Kappa:  0.2534992223950233\n",
      "[[39 18]\n",
      " [14 19]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0             -1.035481              1.779354              1.874576   \n",
      "1              0.965487             -0.399971             -1.606069   \n",
      "2             -0.141249             -1.969933             -0.960470   \n",
      "3             -1.590590             -0.729741             -0.575342   \n",
      "4             -0.391524             -0.894181             -0.426309   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              0.924814             -0.129662              2.608421  \n",
      "1              0.008311              0.834341              0.513694  \n",
      "2              1.005123             -1.117123             -2.399517  \n",
      "3              0.587988             -0.885561             -0.752828  \n",
      "4              1.017585             -0.391173             -0.920259  \n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20, 20, 20), 'learning_rate_init': 0.007, 'max_iter': 500}, que permiten obtener un Accuracy de 69.33% y un Kappa del 36.04\n",
      "Model: \"model_90\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_90 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_272 (Dense)            (None, 20)                140       \n",
      "_________________________________________________________________\n",
      "dense_273 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_274 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_275 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 1,001\n",
      "Trainable params: 1,001\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 500\n",
      "109/109 [==============================] - 0s 37us/step\n",
      "test loss: 1.320278326852606, test accuracy: 0.5963302850723267\n",
      "AUC ROC:  0.6201098146877144\n",
      "Kappa:  0.17281821317695767\n",
      "[[41 21]\n",
      " [23 24]]\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import display, Markdown, Latex\n",
    "import itertools\n",
    "two_combs = list(itertools.combinations(range(len(companies)), 2))\n",
    "for comb in two_combs:\n",
    "    display(Markdown('## '+companies[comb[0]]+' y '+companies[comb[1]]))\n",
    "    X = df_n_ps_std_tc[comb[0]].append(df_n_ps_std_tc[comb[1]])\n",
    "    y = df_n_ps[comb[0]]['chosen'].append(df_n_ps[comb[1]]['chosen'])\n",
    "\n",
    "    print(X.head())\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "    mlp = MLPClassifier(hidden_layer_sizes=(30,30,30))\n",
    "\n",
    "    activation_vec = ['logistic', 'relu', 'tanh']\n",
    "    max_iter_vec = [10, 20, 50, 75, 100, 200, 300, 400, 500, 1000, 2000]\n",
    "    hidden_layer_sizes_vec = [(10,), (20,), (30,), (10, 10), (20, 20), (30, 30), (20, 10), \n",
    "                              (10, 10, 10), (20, 20, 20), (30, 30, 30), (30, 20, 10)]\n",
    "    learning_rate_init_vec = [0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.02]\n",
    "    batch_size_vec = [10, 20, 40, 60, 80, 100, 150]\n",
    "\n",
    "\n",
    "    np.random.seed(1234)\n",
    "    parametros = {'activation': activation_vec,\n",
    "                  'max_iter':max_iter_vec,\n",
    "                  'hidden_layer_sizes': hidden_layer_sizes_vec,\n",
    "                  'learning_rate_init': learning_rate_init_vec#,\n",
    "                  #'batch_size': batch_size_vec\n",
    "                  }\n",
    "    scoring = {'kappa':make_scorer(cohen_kappa_score), 'accuracy':'accuracy'}\n",
    "    grid = GridSearchCV(mlp, param_grid=parametros, cv=5, scoring=scoring, refit='accuracy', n_jobs=-1, iid=True)\n",
    "\n",
    "    grid.fit(X_train, y_train)\n",
    "\n",
    "    print(\"Los parámetros del mejor modelo fueron {0}, que permiten obtener un Accuracy de {1:.2f}% y un Kappa del {2:.2f}\".format(\n",
    "        grid.best_params_, grid.best_score_*100, grid.cv_results_['mean_test_kappa'][grid.best_index_]*100))\n",
    "\n",
    "    n0=X_train.shape[1]\n",
    "    ### hidden_layer_sizes\n",
    "    ns = []\n",
    "    for i in range (len(grid.best_params_['hidden_layer_sizes'])):\n",
    "        ns.append(grid.best_params_['hidden_layer_sizes'][i])\n",
    "\n",
    "    ns.append(1)\n",
    "    lr = grid.best_params_['learning_rate_init']\n",
    "    epochs = grid.best_params_['max_iter']\n",
    "\n",
    "    input_tensor = Input(shape = (n0,))\n",
    "    hidden_outputs = [input_tensor]\n",
    "    if grid.best_params_['activation'] == 'logistic':\n",
    "        grid.best_params_['activation'] = 'sigmoid'\n",
    "    for i in range (len(ns)-1):\n",
    "        hidden_outputs.append(Dense(ns[i], activation = grid.best_params_['activation'])(hidden_outputs[i]))\n",
    "    classification_output = Dense(ns[-1], activation = 'sigmoid')(hidden_outputs[-1])\n",
    "\n",
    "    model = Model([input_tensor], [classification_output])\n",
    "    weights = model.get_weights()\n",
    "\n",
    "    print(model.summary())\n",
    "\n",
    "    model.set_weights(weights)\n",
    "    adam = keras.optimizers.Adam(lr=lr)\n",
    "    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    history = model.fit(X_train, y_train, epochs=epochs, validation_data=(X_test, y_test), verbose=0, \n",
    "                callbacks=[\n",
    "                    keras.callbacks.ReduceLROnPlateau(\n",
    "                        monitor='val_accuracy', factor=0.5, patience=10, min_delta=0.01, verbose=0\n",
    "                    )\n",
    "                ]\n",
    "             )\n",
    "\n",
    "    acc = history.history['accuracy']\n",
    "    val_acc = history.history['val_accuracy']\n",
    "    loss = history.history['loss']\n",
    "    val_loss = history.history['val_loss']\n",
    "\n",
    "    epochs = range(len(acc))\n",
    "\n",
    "    print(\"epochs: \"+str(len(acc)))\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(X_test, y_test)\n",
    "    print(\"test loss: {}, test accuracy: {}\".format(test_loss, test_acc))\n",
    "\n",
    "    y_pred = model.predict(X_test)\n",
    "    print(\"AUC ROC: \",roc_auc_score(y_test, y_pred))\n",
    "\n",
    "    y_pred = list(map(lambda i: int(i>=0.5), y_pred))\n",
    "    print(\"Kappa: \",cohen_kappa_score(y_test, y_pred))\n",
    "\n",
    "    print(confusion_matrix(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3-comp groups combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Gramma"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n",
      "(786, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.004, 'max_iter': 20}, que permiten obtener un Accuracy de 73.34% y un Kappa del 7.33\n",
      "Model: \"model_21\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_21 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_58 (Dense)             (None, 30)                210       \n",
      "_________________________________________________________________\n",
      "dense_59 (Dense)             (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_60 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,171\n",
      "Trainable params: 1,171\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 20\n",
      "197/197 [==============================] - 0s 41us/step\n",
      "test loss: 0.6227250885842416, test accuracy: 0.6700507402420044\n",
      "AUC ROC:  0.5804136253041362\n",
      "Kappa:  -0.02333573083992646\n",
      "[[130   7]\n",
      " [ 58   2]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Hotel Marrakech"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n",
      "(714, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30, 30), 'learning_rate_init': 0.001, 'max_iter': 50}, que permiten obtener un Accuracy de 72.15% y un Kappa del 12.71\n",
      "Model: \"model_22\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_22 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_61 (Dense)             (None, 30)                210       \n",
      "_________________________________________________________________\n",
      "dense_62 (Dense)             (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_63 (Dense)             (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_64 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 2,101\n",
      "Trainable params: 2,101\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 50\n",
      "179/179 [==============================] - 0s 28us/step\n",
      "test loss: 0.6578460099310849, test accuracy: 0.659217894077301\n",
      "AUC ROC:  0.5349883855981417\n",
      "Kappa:  0.033460210675400504\n",
      "[[111  12]\n",
      " [ 49   7]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n",
      "(789, 6)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.008, 'max_iter': 300}, que permiten obtener un Accuracy de 68.19% y un Kappa del 26.70\n",
      "Model: \"model_23\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_23 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_65 (Dense)             (None, 30)                210       \n",
      "_________________________________________________________________\n",
      "dense_66 (Dense)             (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_67 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,171\n",
      "Trainable params: 1,171\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 300\n",
      "198/198 [==============================] - 0s 40us/step\n",
      "test loss: 0.7369338743614428, test accuracy: 0.6464646458625793\n",
      "AUC ROC:  0.5769809138230192\n",
      "Kappa:  0.13730860201668127\n",
      "[[107  26]\n",
      " [ 44  21]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n",
      "(786, 6)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20,), 'learning_rate_init': 0.02, 'max_iter': 400}, que permiten obtener un Accuracy de 73.85% y un Kappa del 28.45\n",
      "Model: \"model_24\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_24 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_68 (Dense)             (None, 20)                140       \n",
      "_________________________________________________________________\n",
      "dense_69 (Dense)             (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 161\n",
      "Trainable params: 161\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 400\n",
      "197/197 [==============================] - 0s 25us/step\n",
      "test loss: 0.6419816567813079, test accuracy: 0.6446700692176819\n",
      "AUC ROC:  0.6370370370370371\n",
      "Kappa:  0.05223367697594494\n",
      "[[115  20]\n",
      " [ 50  12]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma y Hotel Marrakech"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n",
      "(675, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30, 30, 30), 'learning_rate_init': 0.001, 'max_iter': 100}, que permiten obtener un Accuracy de 69.96% y un Kappa del 5.23\n",
      "Model: \"model_25\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_25 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_70 (Dense)             (None, 30)                210       \n",
      "_________________________________________________________________\n",
      "dense_71 (Dense)             (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_72 (Dense)             (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_73 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 2,101\n",
      "Trainable params: 2,101\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 100\n",
      "169/169 [==============================] - 0s 30us/step\n",
      "test loss: 0.6389501479250439, test accuracy: 0.6804733872413635\n",
      "AUC ROC:  0.4647039687703318\n",
      "Kappa:  -0.011751662971175136\n",
      "[[115   1]\n",
      " [ 53   0]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n",
      "(750, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30,), 'learning_rate_init': 0.002, 'max_iter': 10}, que permiten obtener un Accuracy de 67.26% y un Kappa del 3.98\n",
      "Model: \"model_26\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_26 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_74 (Dense)             (None, 30)                210       \n",
      "_________________________________________________________________\n",
      "dense_75 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 241\n",
      "Trainable params: 241\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 10\n",
      "188/188 [==============================] - 0s 32us/step\n",
      "test loss: 0.6565095820325486, test accuracy: 0.6436170339584351\n",
      "AUC ROC:  0.4995032290114257\n",
      "Kappa:  -0.010590500641848521\n",
      "[[121   1]\n",
      " [ 66   0]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n",
      "(747, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (10, 10, 10), 'learning_rate_init': 0.02, 'max_iter': 20}, que permiten obtener un Accuracy de 70.71% y un Kappa del 9.67\n",
      "Model: \"model_27\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_27 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_76 (Dense)             (None, 10)                70        \n",
      "_________________________________________________________________\n",
      "dense_77 (Dense)             (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_78 (Dense)             (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_79 (Dense)             (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 301\n",
      "Trainable params: 301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 20\n",
      "187/187 [==============================] - 0s 32us/step\n",
      "test loss: 0.6191649401889128, test accuracy: 0.6844919919967651\n",
      "AUC ROC:  0.5633903133903134\n",
      "Kappa:  0.009160305343511421\n",
      "[[123  12]\n",
      " [ 47   5]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n",
      "(678, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20, 20, 20), 'learning_rate_init': 0.005, 'max_iter': 50}, que permiten obtener un Accuracy de 67.32% y un Kappa del 10.31\n",
      "Model: \"model_28\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_28 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_80 (Dense)             (None, 20)                140       \n",
      "_________________________________________________________________\n",
      "dense_81 (Dense)             (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_82 (Dense)             (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_83 (Dense)             (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 1,001\n",
      "Trainable params: 1,001\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 50\n",
      "170/170 [==============================] - 0s 29us/step\n",
      "test loss: 1.1758464588838464, test accuracy: 0.5529412031173706\n",
      "AUC ROC:  0.5280327637339359\n",
      "Kappa:  0.07197241775606944\n",
      "[[64 33]\n",
      " [43 30]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n",
      "(675, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 20, 10), 'learning_rate_init': 0.02, 'max_iter': 20}, que permiten obtener un Accuracy de 69.57% y un Kappa del 10.97\n",
      "Model: \"model_29\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_29 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_84 (Dense)             (None, 30)                210       \n",
      "_________________________________________________________________\n",
      "dense_85 (Dense)             (None, 20)                620       \n",
      "_________________________________________________________________\n",
      "dense_86 (Dense)             (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_87 (Dense)             (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 1,051\n",
      "Trainable params: 1,051\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 20\n",
      "169/169 [==============================] - 0s 35us/step\n",
      "test loss: 0.8209835008756649, test accuracy: 0.6863905191421509\n",
      "AUC ROC:  0.6336125158027813\n",
      "Kappa:  0.1859492865582114\n",
      "[[101  12]\n",
      " [ 41  15]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n",
      "(750, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (10, 10), 'learning_rate_init': 0.004, 'max_iter': 200}, que permiten obtener un Accuracy de 67.97% y un Kappa del 23.62\n",
      "Model: \"model_30\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_30 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_88 (Dense)             (None, 10)                70        \n",
      "_________________________________________________________________\n",
      "dense_89 (Dense)             (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_90 (Dense)             (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 191\n",
      "Trainable params: 191\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 200\n",
      "188/188 [==============================] - 0s 32us/step\n",
      "test loss: 0.6406431362984029, test accuracy: 0.6489361524581909\n",
      "AUC ROC:  0.6210131332082551\n",
      "Kappa:  0.1633175994605529\n",
      "[[100  23]\n",
      " [ 43  22]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma y Hotel Marrakech"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.898091              0.151819             -1.172713   \n",
      "1              0.618513             -0.762588              0.061946   \n",
      "2              0.685649              0.002933              0.719805   \n",
      "3              1.175209             -0.552349              0.336427   \n",
      "4              1.350337             -1.407757              0.258917   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              0.474387             -0.020230              1.228657  \n",
      "1              0.944076              0.697880              0.021150  \n",
      "2             -1.251700             -0.952424              1.444556  \n",
      "3              0.482978             -0.212146             -0.144225  \n",
      "4             -0.523670              0.099306              1.706064  \n",
      "(615, 6)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30,), 'learning_rate_init': 0.006, 'max_iter': 500}, que permiten obtener un Accuracy de 70.50% y un Kappa del 7.47\n",
      "Model: \"model_31\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_31 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_91 (Dense)             (None, 30)                210       \n",
      "_________________________________________________________________\n",
      "dense_92 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 241\n",
      "Trainable params: 241\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 500\n",
      "154/154 [==============================] - 0s 39us/step\n",
      "test loss: 0.645158732866312, test accuracy: 0.6623376607894897\n",
      "AUC ROC:  0.5677190360545488\n",
      "Kappa:  0.02460414129110844\n",
      "[[101   0]\n",
      " [ 52   1]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.898091              0.151819             -1.172713   \n",
      "1              0.618513             -0.762588              0.061946   \n",
      "2              0.685649              0.002933              0.719805   \n",
      "3              1.175209             -0.552349              0.336427   \n",
      "4              1.350337             -1.407757              0.258917   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              0.474387             -0.020230              1.228657  \n",
      "1              0.944076              0.697880              0.021150  \n",
      "2             -1.251700             -0.952424              1.444556  \n",
      "3              0.482978             -0.212146             -0.144225  \n",
      "4             -0.523670              0.099306              1.706064  \n",
      "(690, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.004, 'max_iter': 10}, que permiten obtener un Accuracy de 66.15% y un Kappa del 3.87\n",
      "Model: \"model_32\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_32 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_93 (Dense)             (None, 30)                210       \n",
      "_________________________________________________________________\n",
      "dense_94 (Dense)             (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_95 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,171\n",
      "Trainable params: 1,171\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 10\n",
      "173/173 [==============================] - 0s 23us/step\n",
      "test loss: 0.6482885759000834, test accuracy: 0.6416184902191162\n",
      "AUC ROC:  0.5390855457227139\n",
      "Kappa:  -0.012651057401812826\n",
      "[[110   3]\n",
      " [ 59   1]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.898091              0.151819             -1.172713   \n",
      "1              0.618513             -0.762588              0.061946   \n",
      "2              0.685649              0.002933              0.719805   \n",
      "3              1.175209             -0.552349              0.336427   \n",
      "4              1.350337             -1.407757              0.258917   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              0.474387             -0.020230              1.228657  \n",
      "1              0.944076              0.697880              0.021150  \n",
      "2             -1.251700             -0.952424              1.444556  \n",
      "3              0.482978             -0.212146             -0.144225  \n",
      "4             -0.523670              0.099306              1.706064  \n",
      "(687, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.008, 'max_iter': 20}, que permiten obtener un Accuracy de 71.26% y un Kappa del 6.69\n",
      "Model: \"model_33\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_33 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_96 (Dense)             (None, 30)                210       \n",
      "_________________________________________________________________\n",
      "dense_97 (Dense)             (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_98 (Dense)             (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,171\n",
      "Trainable params: 1,171\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 20\n",
      "172/172 [==============================] - 0s 41us/step\n",
      "test loss: 0.6352972180344337, test accuracy: 0.680232584476471\n",
      "AUC ROC:  0.5673061677501189\n",
      "Kappa:  0.09352242238405528\n",
      "[[108  11]\n",
      " [ 44   9]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.898091              0.151819             -1.172713   \n",
      "1              0.618513             -0.762588              0.061946   \n",
      "2              0.685649              0.002933              0.719805   \n",
      "3              1.175209             -0.552349              0.336427   \n",
      "4              1.350337             -1.407757              0.258917   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              0.474387             -0.020230              1.228657  \n",
      "1              0.944076              0.697880              0.021150  \n",
      "2             -1.251700             -0.952424              1.444556  \n",
      "3              0.482978             -0.212146             -0.144225  \n",
      "4             -0.523670              0.099306              1.706064  \n",
      "(618, 6)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (10, 10, 10), 'learning_rate_init': 0.009, 'max_iter': 300}, que permiten obtener un Accuracy de 64.15% y un Kappa del 23.85\n",
      "Model: \"model_34\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_34 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_99 (Dense)             (None, 10)                70        \n",
      "_________________________________________________________________\n",
      "dense_100 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_101 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_102 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 301\n",
      "Trainable params: 301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 300\n",
      "155/155 [==============================] - 0s 32us/step\n",
      "test loss: 0.6952421103754352, test accuracy: 0.6451612710952759\n",
      "AUC ROC:  0.6549422799422799\n",
      "Kappa:  0.19033146547630342\n",
      "[[78 21]\n",
      " [34 22]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.898091              0.151819             -1.172713   \n",
      "1              0.618513             -0.762588              0.061946   \n",
      "2              0.685649              0.002933              0.719805   \n",
      "3              1.175209             -0.552349              0.336427   \n",
      "4              1.350337             -1.407757              0.258917   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              0.474387             -0.020230              1.228657  \n",
      "1              0.944076              0.697880              0.021150  \n",
      "2             -1.251700             -0.952424              1.444556  \n",
      "3              0.482978             -0.212146             -0.144225  \n",
      "4             -0.523670              0.099306              1.706064  \n",
      "(615, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (10, 10), 'learning_rate_init': 0.008, 'max_iter': 50}, que permiten obtener un Accuracy de 69.20% y un Kappa del 16.25\n",
      "Model: \"model_35\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_35 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_103 (Dense)            (None, 10)                70        \n",
      "_________________________________________________________________\n",
      "dense_104 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_105 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 191\n",
      "Trainable params: 191\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 50\n",
      "154/154 [==============================] - 0s 52us/step\n",
      "test loss: 0.6325509857821774, test accuracy: 0.649350643157959\n",
      "AUC ROC:  0.6517722473604827\n",
      "Kappa:  0.14338689740420263\n",
      "[[84 18]\n",
      " [36 16]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.898091              0.151819             -1.172713   \n",
      "1              0.618513             -0.762588              0.061946   \n",
      "2              0.685649              0.002933              0.719805   \n",
      "3              1.175209             -0.552349              0.336427   \n",
      "4              1.350337             -1.407757              0.258917   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              0.474387             -0.020230              1.228657  \n",
      "1              0.944076              0.697880              0.021150  \n",
      "2             -1.251700             -0.952424              1.444556  \n",
      "3              0.482978             -0.212146             -0.144225  \n",
      "4             -0.523670              0.099306              1.706064  \n",
      "(690, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (10, 10, 10), 'learning_rate_init': 0.008, 'max_iter': 10}, que permiten obtener un Accuracy de 65.96% y un Kappa del 7.09\n",
      "Model: \"model_36\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_36 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_106 (Dense)            (None, 10)                70        \n",
      "_________________________________________________________________\n",
      "dense_107 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_108 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_109 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 301\n",
      "Trainable params: 301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 10\n",
      "173/173 [==============================] - 0s 35us/step\n",
      "test loss: 0.6230642020357826, test accuracy: 0.6242774724960327\n",
      "AUC ROC:  0.6555332568807339\n",
      "Kappa:  0.07119848021805564\n",
      "[[97 12]\n",
      " [53 11]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Gramma y Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.618349              0.564005              0.091611   \n",
      "1             -0.149103             -1.303101             -0.498811   \n",
      "2             -1.141294             -1.317570              0.575363   \n",
      "3             -0.988346             -0.540855              1.006800   \n",
      "4             -0.640925             -0.228256              0.461986   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.885208              0.563756              1.803892  \n",
      "1              0.416525             -0.660751             -1.179350  \n",
      "2             -1.560836              0.336340              1.197709  \n",
      "3             -0.214650             -0.364940              0.068869  \n",
      "4             -1.274446             -1.494581             -0.125833  \n",
      "(579, 6)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (10,), 'learning_rate_init': 0.02, 'max_iter': 50}, que permiten obtener un Accuracy de 63.13% y un Kappa del 16.42\n",
      "Model: \"model_37\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_37 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_110 (Dense)            (None, 10)                70        \n",
      "_________________________________________________________________\n",
      "dense_111 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 81\n",
      "Trainable params: 81\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 50\n",
      "145/145 [==============================] - 0s 48us/step\n",
      "test loss: 0.7010458753026765, test accuracy: 0.5793103575706482\n",
      "AUC ROC:  0.5068686868686869\n",
      "Kappa:  0.012283640424343933\n",
      "[[73 17]\n",
      " [44 11]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Gramma y Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.618349              0.564005              0.091611   \n",
      "1             -0.149103             -1.303101             -0.498811   \n",
      "2             -1.141294             -1.317570              0.575363   \n",
      "3             -0.988346             -0.540855              1.006800   \n",
      "4             -0.640925             -0.228256              0.461986   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.885208              0.563756              1.803892  \n",
      "1              0.416525             -0.660751             -1.179350  \n",
      "2             -1.560836              0.336340              1.197709  \n",
      "3             -0.214650             -0.364940              0.068869  \n",
      "4             -1.274446             -1.494581             -0.125833  \n",
      "(576, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20, 20), 'learning_rate_init': 0.005, 'max_iter': 50}, que permiten obtener un Accuracy de 68.75% y un Kappa del 21.36\n",
      "Model: \"model_38\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_38 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_112 (Dense)            (None, 20)                140       \n",
      "_________________________________________________________________\n",
      "dense_113 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_114 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 581\n",
      "Trainable params: 581\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 50\n",
      "144/144 [==============================] - 0s 49us/step\n",
      "test loss: 0.6971434023645189, test accuracy: 0.6041666865348816\n",
      "AUC ROC:  0.5875402792696026\n",
      "Kappa:  0.03752345215759856\n",
      "[[75 20]\n",
      " [37 12]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Gramma y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.618349              0.564005              0.091611   \n",
      "1             -0.149103             -1.303101             -0.498811   \n",
      "2             -1.141294             -1.317570              0.575363   \n",
      "3             -0.988346             -0.540855              1.006800   \n",
      "4             -0.640925             -0.228256              0.461986   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.885208              0.563756              1.803892  \n",
      "1              0.416525             -0.660751             -1.179350  \n",
      "2             -1.560836              0.336340              1.197709  \n",
      "3             -0.214650             -0.364940              0.068869  \n",
      "4             -1.274446             -1.494581             -0.125833  \n",
      "(651, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (75) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (10, 10), 'learning_rate_init': 0.002, 'max_iter': 75}, que permiten obtener un Accuracy de 65.37% y un Kappa del 7.82\n",
      "Model: \"model_39\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_39 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_115 (Dense)            (None, 10)                70        \n",
      "_________________________________________________________________\n",
      "dense_116 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_117 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 191\n",
      "Trainable params: 191\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 75\n",
      "163/163 [==============================] - 0s 43us/step\n",
      "test loss: 0.6838659391812751, test accuracy: 0.6073619723320007\n",
      "AUC ROC:  0.5291982323232323\n",
      "Kappa:  0.044863578099249346\n",
      "[[92  7]\n",
      " [57  7]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Hotel Marrakech y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.297447              0.231356              0.546295   \n",
      "1              0.766614             -1.227377             -0.605576   \n",
      "2             -0.431734             -1.183099              1.287522   \n",
      "3              0.078607             -1.572049              1.188424   \n",
      "4              0.007655             -1.466562              1.294323   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.007345              0.269632              0.236051  \n",
      "1             -1.216250              0.473081             -0.446139  \n",
      "2              0.927452             -0.309857             -1.666006  \n",
      "3              0.092330             -0.244414             -0.490911  \n",
      "4             -0.410616             -0.763440             -1.007284  \n",
      "(579, 6)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20, 20, 20), 'learning_rate_init': 0.003, 'max_iter': 400}, que permiten obtener un Accuracy de 67.05% y un Kappa del 31.01\n",
      "Model: \"model_40\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_40 (InputLayer)        (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_118 (Dense)            (None, 20)                140       \n",
      "_________________________________________________________________\n",
      "dense_119 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_120 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_121 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 1,001\n",
      "Trainable params: 1,001\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 400\n",
      "145/145 [==============================] - 0s 28us/step\n",
      "test loss: 0.8860082655117429, test accuracy: 0.6137930750846863\n",
      "AUC ROC:  0.6517647058823529\n",
      "Kappa:  0.1960396039603961\n",
      "[[59 26]\n",
      " [30 30]]\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import display, Markdown, Latex\n",
    "import itertools\n",
    "three_combs = list(itertools.combinations(range(len(companies)), 3))\n",
    "for comb in three_combs:\n",
    "    title = '## '\n",
    "    for no in comb[:-1]:\n",
    "        title+=companies[no]\n",
    "        title+=' y '\n",
    "    title+=companies[comb[-1]]\n",
    "    display(Markdown(title))\n",
    "    X = df_n_ps_std_tc[comb[0]]\n",
    "    y = df_n_ps[comb[0]]['chosen']\n",
    "    for no in comb[1:]:\n",
    "        X= X.append(df_n_ps_std_tc[no])\n",
    "        y= y.append(df_n_ps[no]['chosen'])\n",
    "    \n",
    "\n",
    "    print(X.head())\n",
    "    print(X.shape)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "    mlp = MLPClassifier(hidden_layer_sizes=(30,30,30))\n",
    "\n",
    "    activation_vec = ['logistic', 'relu', 'tanh']\n",
    "    max_iter_vec = [10, 20, 50, 75, 100, 200, 300, 400, 500, 1000, 2000]\n",
    "    hidden_layer_sizes_vec = [(10,), (20,), (30,), (10, 10), (20, 20), (30, 30), (20, 10), \n",
    "                              (10, 10, 10), (20, 20, 20), (30, 30, 30), (30, 20, 10)]\n",
    "    learning_rate_init_vec = [0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.02]\n",
    "    batch_size_vec = [10, 20, 40, 60, 80, 100, 150]\n",
    "\n",
    "\n",
    "    np.random.seed(1234)\n",
    "    parametros = {'activation': activation_vec,\n",
    "                  'max_iter':max_iter_vec,\n",
    "                  'hidden_layer_sizes': hidden_layer_sizes_vec,\n",
    "                  'learning_rate_init': learning_rate_init_vec#,\n",
    "                  #'batch_size': batch_size_vec\n",
    "                  }\n",
    "    scoring = {'kappa':make_scorer(cohen_kappa_score), 'accuracy':'accuracy'}\n",
    "    grid = GridSearchCV(mlp, param_grid=parametros, cv=5, scoring=scoring, refit='accuracy', n_jobs=-1, iid=True)\n",
    "\n",
    "    grid.fit(X_train, y_train)\n",
    "\n",
    "    print(\"Los parámetros del mejor modelo fueron {0}, que permiten obtener un Accuracy de {1:.2f}% y un Kappa del {2:.2f}\".format(\n",
    "        grid.best_params_, grid.best_score_*100, grid.cv_results_['mean_test_kappa'][grid.best_index_]*100))\n",
    "\n",
    "    n0=X_train.shape[1]\n",
    "    ### hidden_layer_sizes\n",
    "    ns = []\n",
    "    for i in range (len(grid.best_params_['hidden_layer_sizes'])):\n",
    "        ns.append(grid.best_params_['hidden_layer_sizes'][i])\n",
    "\n",
    "    ns.append(1)\n",
    "    lr = grid.best_params_['learning_rate_init']\n",
    "    epochs = grid.best_params_['max_iter']\n",
    "\n",
    "    input_tensor = Input(shape = (n0,))\n",
    "    hidden_outputs = [input_tensor]\n",
    "    if grid.best_params_['activation'] == 'logistic':\n",
    "        grid.best_params_['activation'] = 'sigmoid'\n",
    "    for i in range (len(ns)-1):\n",
    "        hidden_outputs.append(Dense(ns[i], activation = grid.best_params_['activation'])(hidden_outputs[i]))\n",
    "    classification_output = Dense(ns[-1], activation = 'sigmoid')(hidden_outputs[-1])\n",
    "\n",
    "    model = Model([input_tensor], [classification_output])\n",
    "    weights = model.get_weights()\n",
    "\n",
    "    print(model.summary())\n",
    "\n",
    "    model.set_weights(weights)\n",
    "    adam = keras.optimizers.Adam(lr=lr)\n",
    "    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    history = model.fit(X_train, y_train, epochs=epochs, validation_data=(X_test, y_test), verbose=0, \n",
    "                callbacks=[\n",
    "                    keras.callbacks.ReduceLROnPlateau(\n",
    "                        monitor='val_accuracy', factor=0.5, patience=10, min_delta=0.01, verbose=0\n",
    "                    )\n",
    "                ]\n",
    "             )\n",
    "\n",
    "    acc = history.history['accuracy']\n",
    "    val_acc = history.history['val_accuracy']\n",
    "    loss = history.history['loss']\n",
    "    val_loss = history.history['val_loss']\n",
    "\n",
    "    epochs = range(len(acc))\n",
    "\n",
    "    print(\"epochs: \"+str(len(acc)))\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(X_test, y_test)\n",
    "    print(\"test loss: {}, test accuracy: {}\".format(test_loss, test_acc))\n",
    "\n",
    "    y_pred = model.predict(X_test)\n",
    "    print(\"AUC ROC: \",roc_auc_score(y_test, y_pred))\n",
    "\n",
    "    y_pred = list(map(lambda i: int(i>=0.5), y_pred))\n",
    "    print(\"Kappa: \",cohen_kappa_score(y_test, y_pred))\n",
    "\n",
    "    print(confusion_matrix(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5-comp groups combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Gramma y Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n",
      "(1149, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (10, 10), 'learning_rate_init': 0.02, 'max_iter': 10}, que permiten obtener un Accuracy de 67.25% y un Kappa del 3.61\n",
      "Model: \"model_112\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_112 (InputLayer)       (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_340 (Dense)            (None, 10)                70        \n",
      "_________________________________________________________________\n",
      "dense_341 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_342 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 191\n",
      "Trainable params: 191\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 10\n",
      "288/288 [==============================] - 0s 59us/step\n",
      "test loss: 0.6484600371784635, test accuracy: 0.6666666865348816\n",
      "AUC ROC:  0.5241138560687433\n",
      "Kappa:  0.026753027316248956\n",
      "[[190   0]\n",
      " [ 96   2]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Gramma y Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n",
      "(1146, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20, 20), 'learning_rate_init': 0.008, 'max_iter': 10}, que permiten obtener un Accuracy de 68.80% y un Kappa del 4.54\n",
      "Model: \"model_113\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_113 (InputLayer)       (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_343 (Dense)            (None, 20)                140       \n",
      "_________________________________________________________________\n",
      "dense_344 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_345 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 581\n",
      "Trainable params: 581\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 10\n",
      "287/287 [==============================] - 0s 30us/step\n",
      "test loss: 0.6213179724972423, test accuracy: 0.6585366129875183\n",
      "AUC ROC:  0.5731198625935467\n",
      "Kappa:  0.03671484348243037\n",
      "[[173  36]\n",
      " [ 62  16]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Gramma y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n",
      "(1221, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (10,), 'learning_rate_init': 0.001, 'max_iter': 100}, que permiten obtener un Accuracy de 67.98% y un Kappa del 3.10\n",
      "Model: \"model_114\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_114 (InputLayer)       (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_346 (Dense)            (None, 10)                70        \n",
      "_________________________________________________________________\n",
      "dense_347 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 81\n",
      "Trainable params: 81\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 100\n",
      "306/306 [==============================] - 0s 46us/step\n",
      "test loss: 0.6286440378700207, test accuracy: 0.6764705777168274\n",
      "AUC ROC:  0.5523837407895379\n",
      "Kappa:  0.0\n",
      "[[207   0]\n",
      " [ 99   0]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Hotel Marrakech y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n",
      "(1149, 6)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20,), 'learning_rate_init': 0.005, 'max_iter': 400}, que permiten obtener un Accuracy de 66.67% y un Kappa del 13.94\n",
      "Model: \"model_115\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_115 (InputLayer)       (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_348 (Dense)            (None, 20)                140       \n",
      "_________________________________________________________________\n",
      "dense_349 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 161\n",
      "Trainable params: 161\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 400\n",
      "288/288 [==============================] - 0s 41us/step\n",
      "test loss: 0.6312494774659475, test accuracy: 0.6631944179534912\n",
      "AUC ROC:  0.6036702127659574\n",
      "Kappa:  0.07250996015936262\n",
      "[[182   6]\n",
      " [ 91   9]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma y Hotel Marrakech y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              1.585484              0.923309             -0.748807   \n",
      "1              1.129768              0.963814              0.209096   \n",
      "2             -0.066076              1.857866              1.921193   \n",
      "3              0.119831              1.429286              1.472808   \n",
      "4             -0.123292              0.197415              0.503797   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              1.209820             -1.073924              0.283035  \n",
      "1             -0.143471              0.115184              0.020905  \n",
      "2              1.093619              2.089353              1.984310  \n",
      "3              1.282029              1.096897              2.122871  \n",
      "4              1.431215              1.715761              0.683611  \n",
      "(1110, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (20,), 'learning_rate_init': 0.008, 'max_iter': 50}, que permiten obtener un Accuracy de 66.23% y un Kappa del 7.40\n",
      "Model: \"model_116\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_116 (InputLayer)       (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_350 (Dense)            (None, 20)                140       \n",
      "_________________________________________________________________\n",
      "dense_351 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 161\n",
      "Trainable params: 161\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 50\n",
      "278/278 [==============================] - 0s 43us/step\n",
      "test loss: 0.6296036659384803, test accuracy: 0.633093535900116\n",
      "AUC ROC:  0.5993293246993524\n",
      "Kappa:  -0.008679567444507708\n",
      "[[169  15]\n",
      " [ 87   7]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma y Hotel Marrakech y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   tonalcentroidfiles_1  tonalcentroidfiles_2  tonalcentroidfiles_3  \\\n",
      "0              0.898091              0.151819             -1.172713   \n",
      "1              0.618513             -0.762588              0.061946   \n",
      "2              0.685649              0.002933              0.719805   \n",
      "3              1.175209             -0.552349              0.336427   \n",
      "4              1.350337             -1.407757              0.258917   \n",
      "\n",
      "   tonalcentroidfiles_4  tonalcentroidfiles_5  tonalcentroidfiles_6  \n",
      "0              0.474387             -0.020230              1.228657  \n",
      "1              0.944076              0.697880              0.021150  \n",
      "2             -1.251700             -0.952424              1.444556  \n",
      "3              0.482978             -0.212146             -0.144225  \n",
      "4             -0.523670              0.099306              1.706064  \n",
      "(1050, 6)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (10,), 'learning_rate_init': 0.006, 'max_iter': 400}, que permiten obtener un Accuracy de 65.31% y un Kappa del 4.74\n",
      "Model: \"model_117\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_117 (InputLayer)       (None, 6)                 0         \n",
      "_________________________________________________________________\n",
      "dense_352 (Dense)            (None, 10)                70        \n",
      "_________________________________________________________________\n",
      "dense_353 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 81\n",
      "Trainable params: 81\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 400\n",
      "263/263 [==============================] - 0s 34us/step\n",
      "test loss: 0.6532349291863097, test accuracy: 0.6197718381881714\n",
      "AUC ROC:  0.5350877192982456\n",
      "Kappa:  -0.026621906472011903\n",
      "[[158  13]\n",
      " [ 87   5]]\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import display, Markdown, Latex\n",
    "import itertools\n",
    "three_combs = list(itertools.combinations(range(len(companies)), 5))\n",
    "for comb in three_combs:\n",
    "    title = '## '\n",
    "    for no in comb[:-1]:\n",
    "        title+=companies[no]\n",
    "        title+=' y '\n",
    "    title+=companies[comb[-1]]\n",
    "    display(Markdown(title))\n",
    "    X = df_n_ps_std_tc[comb[0]]\n",
    "    y = df_n_ps[comb[0]]['chosen']\n",
    "    for no in comb[1:]:\n",
    "        X= X.append(df_n_ps_std_tc[no])\n",
    "        y= y.append(df_n_ps[no]['chosen'])\n",
    "    \n",
    "\n",
    "    print(X.head())\n",
    "    print(X.shape)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "    mlp = MLPClassifier(hidden_layer_sizes=(30,30,30))\n",
    "\n",
    "    activation_vec = ['logistic', 'relu', 'tanh']\n",
    "    max_iter_vec = [10, 20, 50, 75, 100, 200, 300, 400, 500, 1000, 2000]\n",
    "    hidden_layer_sizes_vec = [(10,), (20,), (30,), (10, 10), (20, 20), (30, 30), (20, 10), \n",
    "                              (10, 10, 10), (20, 20, 20), (30, 30, 30), (30, 20, 10)]\n",
    "    learning_rate_init_vec = [0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.02]\n",
    "    batch_size_vec = [10, 20, 40, 60, 80, 100, 150]\n",
    "\n",
    "\n",
    "    np.random.seed(1234)\n",
    "    parametros = {'activation': activation_vec,\n",
    "                  'max_iter':max_iter_vec,\n",
    "                  'hidden_layer_sizes': hidden_layer_sizes_vec,\n",
    "                  'learning_rate_init': learning_rate_init_vec#,\n",
    "                  #'batch_size': batch_size_vec\n",
    "                  }\n",
    "    scoring = {'kappa':make_scorer(cohen_kappa_score), 'accuracy':'accuracy'}\n",
    "    grid = GridSearchCV(mlp, param_grid=parametros, cv=5, scoring=scoring, refit='accuracy', n_jobs=-1, iid=True)\n",
    "\n",
    "    grid.fit(X_train, y_train)\n",
    "\n",
    "    print(\"Los parámetros del mejor modelo fueron {0}, que permiten obtener un Accuracy de {1:.2f}% y un Kappa del {2:.2f}\".format(\n",
    "        grid.best_params_, grid.best_score_*100, grid.cv_results_['mean_test_kappa'][grid.best_index_]*100))\n",
    "\n",
    "    n0=X_train.shape[1]\n",
    "    ### hidden_layer_sizes\n",
    "    ns = []\n",
    "    for i in range (len(grid.best_params_['hidden_layer_sizes'])):\n",
    "        ns.append(grid.best_params_['hidden_layer_sizes'][i])\n",
    "\n",
    "    ns.append(1)\n",
    "    lr = grid.best_params_['learning_rate_init']\n",
    "    epochs = grid.best_params_['max_iter']\n",
    "\n",
    "    input_tensor = Input(shape = (n0,))\n",
    "    hidden_outputs = [input_tensor]\n",
    "    if grid.best_params_['activation'] == 'logistic':\n",
    "        grid.best_params_['activation'] = 'sigmoid'\n",
    "    for i in range (len(ns)-1):\n",
    "        hidden_outputs.append(Dense(ns[i], activation = grid.best_params_['activation'])(hidden_outputs[i]))\n",
    "    classification_output = Dense(ns[-1], activation = 'sigmoid')(hidden_outputs[-1])\n",
    "\n",
    "    model = Model([input_tensor], [classification_output])\n",
    "    weights = model.get_weights()\n",
    "\n",
    "    print(model.summary())\n",
    "\n",
    "    model.set_weights(weights)\n",
    "    adam = keras.optimizers.Adam(lr=lr)\n",
    "    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    history = model.fit(X_train, y_train, epochs=epochs, validation_data=(X_test, y_test), verbose=0, \n",
    "                callbacks=[\n",
    "                    keras.callbacks.ReduceLROnPlateau(\n",
    "                        monitor='val_accuracy', factor=0.5, patience=10, min_delta=0.01, verbose=0\n",
    "                    )\n",
    "                ]\n",
    "             )\n",
    "\n",
    "    acc = history.history['accuracy']\n",
    "    val_acc = history.history['val_accuracy']\n",
    "    loss = history.history['loss']\n",
    "    val_loss = history.history['val_loss']\n",
    "\n",
    "    epochs = range(len(acc))\n",
    "\n",
    "    print(\"epochs: \"+str(len(acc)))\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(X_test, y_test)\n",
    "    print(\"test loss: {}, test accuracy: {}\".format(test_loss, test_acc))\n",
    "\n",
    "    y_pred = model.predict(X_test)\n",
    "    print(\"AUC ROC: \",roc_auc_score(y_test, y_pred))\n",
    "\n",
    "    y_pred = list(map(lambda i: int(i>=0.5), y_pred))\n",
    "    print(\"Kappa: \",cohen_kappa_score(y_test, y_pred))\n",
    "\n",
    "    print(confusion_matrix(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chromagram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['durationfiles', 'rmsfiles', 'rmsmedianfiles', 'lowenergyfiles',\n",
       "       'ASRfiles', 'beatspectrumfiles', 'eventdensityfiles', 'tempofiles',\n",
       "       'pulseclarityfiles', 'zerocrossfiles', 'rolloffsfiles',\n",
       "       'brightnessfiles', 'spreadfiles', 'centroidfiles', 'kurtosisfiles',\n",
       "       'flatnessfiles', 'entropyfiles', 'mfccfiles_1', 'mfccfiles_2',\n",
       "       'mfccfiles_3', 'mfccfiles_4', 'mfccfiles_5', 'mfccfiles_6',\n",
       "       'mfccfiles_7', 'mfccfiles_8', 'mfccfiles_9', 'mfccfiles_10',\n",
       "       'mfccfiles_11', 'mfccfiles_12', 'mfccfiles_13', 'inharmonicityfiles',\n",
       "       'bestkeyfiles', 'keyclarityfiles', 'modalityfiles',\n",
       "       'tonalcentroidfiles_1', 'tonalcentroidfiles_2', 'tonalcentroidfiles_3',\n",
       "       'tonalcentroidfiles_4', 'tonalcentroidfiles_5', 'tonalcentroidfiles_6',\n",
       "       'chromagramfiles_1', 'chromagramfiles_2', 'chromagramfiles_3',\n",
       "       'chromagramfiles_4', 'chromagramfiles_5', 'chromagramfiles_6',\n",
       "       'chromagramfiles_7', 'chromagramfiles_8', 'chromagramfiles_9',\n",
       "       'chromagramfiles_10', 'chromagramfiles_11', 'chromagramfiles_12',\n",
       "       'attackslopefiles', 'attackleapfiles', 'chosen'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_n_ps_std[0].columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['chromagramfiles_1', 'chromagramfiles_2', 'chromagramfiles_3',\n",
       "       'chromagramfiles_4', 'chromagramfiles_5', 'chromagramfiles_6',\n",
       "       'chromagramfiles_7', 'chromagramfiles_8', 'chromagramfiles_9',\n",
       "       'chromagramfiles_10', 'chromagramfiles_11', 'chromagramfiles_12'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_n_ps_std[0].columns[40:52]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 315 entries, 0 to 314\n",
      "Data columns (total 12 columns):\n",
      "chromagramfiles_1     315 non-null float64\n",
      "chromagramfiles_2     315 non-null float64\n",
      "chromagramfiles_3     315 non-null float64\n",
      "chromagramfiles_4     315 non-null float64\n",
      "chromagramfiles_5     315 non-null float64\n",
      "chromagramfiles_6     315 non-null float64\n",
      "chromagramfiles_7     315 non-null float64\n",
      "chromagramfiles_8     315 non-null float64\n",
      "chromagramfiles_9     315 non-null float64\n",
      "chromagramfiles_10    315 non-null float64\n",
      "chromagramfiles_11    315 non-null float64\n",
      "chromagramfiles_12    315 non-null float64\n",
      "dtypes: float64(12)\n",
      "memory usage: 29.6 KB\n"
     ]
    }
   ],
   "source": [
    "df_n_ps_std_ch = [None]*len(companies)\n",
    "for i in range(len(companies)):\n",
    "    df_n_ps_std_ch[i] = pd.DataFrame(df_n_ps_std[i].iloc[:,40:52])\n",
    "    df_n_ps_std_ch[i].columns=df_n_ps_std[i].columns[40:52]\n",
    "df_n_ps_std_ch[0].info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2-comp groups combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(570, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'logistic', 'hidden_layer_sizes': (20,), 'learning_rate_init': 0.02, 'max_iter': 500}, que permiten obtener un Accuracy de 73.54% y un Kappa del 7.92\n",
      "Model: \"model_41\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_41 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_122 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_123 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 281\n",
      "Trainable params: 281\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 500\n",
      "143/143 [==============================] - 0s 49us/step\n",
      "test loss: 0.6013401425801791, test accuracy: 0.7132866978645325\n",
      "AUC ROC:  0.5564323290291727\n",
      "Kappa:  0.058757424947824655\n",
      "[[99  3]\n",
      " [38  3]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(531, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (10, 10), 'learning_rate_init': 0.02, 'max_iter': 10}, que permiten obtener un Accuracy de 71.61% y un Kappa del 8.27\n",
      "Model: \"model_42\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_42 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_124 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_125 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_126 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 251\n",
      "Trainable params: 251\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 10\n",
      "133/133 [==============================] - 0s 53us/step\n",
      "test loss: 0.6136606084673029, test accuracy: 0.7293233275413513\n",
      "AUC ROC:  0.5542424242424242\n",
      "Kappa:  0.06593835349200161\n",
      "[[93  7]\n",
      " [29  4]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Hotel Marrakech"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(459, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (20,), 'learning_rate_init': 0.01, 'max_iter': 100}, que permiten obtener un Accuracy de 72.97% y un Kappa del 30.60\n",
      "Model: \"model_43\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_43 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_127 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_128 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 281\n",
      "Trainable params: 281\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 100\n",
      "115/115 [==============================] - 0s 52us/step\n",
      "test loss: 0.6349654368732287, test accuracy: 0.6608695387840271\n",
      "AUC ROC:  0.5981012658227849\n",
      "Kappa:  0.11064842355740634\n",
      "[[67 12]\n",
      " [27  9]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(534, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (20, 20), 'learning_rate_init': 0.01, 'max_iter': 2000}, que permiten obtener un Accuracy de 68.25% y un Kappa del 31.03\n",
      "Model: \"model_44\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_44 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_129 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_130 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_131 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 701\n",
      "Trainable params: 701\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 2000\n",
      "134/134 [==============================] - 0s 52us/step\n",
      "test loss: 0.8510398740199074, test accuracy: 0.611940324306488\n",
      "AUC ROC:  0.6521739130434783\n",
      "Kappa:  0.0821917808219178\n",
      "[[68 20]\n",
      " [32 14]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(531, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'logistic', 'hidden_layer_sizes': (10,), 'learning_rate_init': 0.009, 'max_iter': 200}, que permiten obtener un Accuracy de 71.61% y un Kappa del 23.98\n",
      "Model: \"model_45\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_45 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_132 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_133 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 141\n",
      "Trainable params: 141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 200\n",
      "133/133 [==============================] - 0s 53us/step\n",
      "test loss: 0.5838545421908673, test accuracy: 0.7218044996261597\n",
      "AUC ROC:  0.588180693069307\n",
      "Kappa:  0.09357156013998891\n",
      "[[90 11]\n",
      " [26  6]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.369691          -0.881824          -0.095656          -0.923999   \n",
      "1          -0.175875          -0.403800          -0.657709          -0.201259   \n",
      "2           0.894452          -0.189794           1.959063           0.169276   \n",
      "3           0.060782          -0.392075           0.826233          -0.048480   \n",
      "4          -1.116536          -0.923742          -1.238971          -0.919598   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0          -0.030645          -0.834931          -1.031650          -0.840942   \n",
      "1           1.691433          -0.672783          -0.119944          -0.440080   \n",
      "2          -0.403611          -1.036954           1.447615          -0.340767   \n",
      "3           1.789786          -0.552163           0.121028          -0.111355   \n",
      "4           0.313068          -1.160111           1.316032          -0.700013   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.677716            1.084098           -1.064999   \n",
      "1           0.339906            1.084098            0.504608   \n",
      "2          -0.846170           -0.515065           -0.699878   \n",
      "3           0.220614            1.084098            0.073241   \n",
      "4          -1.600210            1.084098           -1.072155   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -1.156623  \n",
      "1            0.931676  \n",
      "2            0.032355  \n",
      "3            1.176257  \n",
      "4            1.270095  \n",
      "(471, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.002, 'max_iter': 200}, que permiten obtener un Accuracy de 71.95% y un Kappa del 27.04\n",
      "Model: \"model_46\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_46 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_134 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_135 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_136 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,351\n",
      "Trainable params: 1,351\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 200\n",
      "118/118 [==============================] - 0s 51us/step\n",
      "test loss: 0.6019874669737735, test accuracy: 0.7372881174087524\n",
      "AUC ROC:  0.5298479792361883\n",
      "Kappa:  0.13276434329065911\n",
      "[[82  5]\n",
      " [26  5]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Hotel Marrakech"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.369691          -0.881824          -0.095656          -0.923999   \n",
      "1          -0.175875          -0.403800          -0.657709          -0.201259   \n",
      "2           0.894452          -0.189794           1.959063           0.169276   \n",
      "3           0.060782          -0.392075           0.826233          -0.048480   \n",
      "4          -1.116536          -0.923742          -1.238971          -0.919598   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0          -0.030645          -0.834931          -1.031650          -0.840942   \n",
      "1           1.691433          -0.672783          -0.119944          -0.440080   \n",
      "2          -0.403611          -1.036954           1.447615          -0.340767   \n",
      "3           1.789786          -0.552163           0.121028          -0.111355   \n",
      "4           0.313068          -1.160111           1.316032          -0.700013   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.677716            1.084098           -1.064999   \n",
      "1           0.339906            1.084098            0.504608   \n",
      "2          -0.846170           -0.515065           -0.699878   \n",
      "3           0.220614            1.084098            0.073241   \n",
      "4          -1.600210            1.084098           -1.072155   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -1.156623  \n",
      "1            0.931676  \n",
      "2            0.032355  \n",
      "3            1.176257  \n",
      "4            1.270095  \n",
      "(399, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (75) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 20, 10), 'learning_rate_init': 0.001, 'max_iter': 75}, que permiten obtener un Accuracy de 72.24% y un Kappa del 20.51\n",
      "Model: \"model_47\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_47 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_137 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_138 (Dense)            (None, 20)                620       \n",
      "_________________________________________________________________\n",
      "dense_139 (Dense)            (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_140 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 1,231\n",
      "Trainable params: 1,231\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 75\n",
      "100/100 [==============================] - 0s 60us/step\n",
      "test loss: 0.6944141483306885, test accuracy: 0.5699999928474426\n",
      "AUC ROC:  0.5492359932088284\n",
      "Kappa:  -0.08366935483870952\n",
      "[[56  6]\n",
      " [37  1]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.369691          -0.881824          -0.095656          -0.923999   \n",
      "1          -0.175875          -0.403800          -0.657709          -0.201259   \n",
      "2           0.894452          -0.189794           1.959063           0.169276   \n",
      "3           0.060782          -0.392075           0.826233          -0.048480   \n",
      "4          -1.116536          -0.923742          -1.238971          -0.919598   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0          -0.030645          -0.834931          -1.031650          -0.840942   \n",
      "1           1.691433          -0.672783          -0.119944          -0.440080   \n",
      "2          -0.403611          -1.036954           1.447615          -0.340767   \n",
      "3           1.789786          -0.552163           0.121028          -0.111355   \n",
      "4           0.313068          -1.160111           1.316032          -0.700013   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.677716            1.084098           -1.064999   \n",
      "1           0.339906            1.084098            0.504608   \n",
      "2          -0.846170           -0.515065           -0.699878   \n",
      "3           0.220614            1.084098            0.073241   \n",
      "4          -1.600210            1.084098           -1.072155   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -1.156623  \n",
      "1            0.931676  \n",
      "2            0.032355  \n",
      "3            1.176257  \n",
      "4            1.270095  \n",
      "(474, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30,), 'learning_rate_init': 0.006, 'max_iter': 100}, que permiten obtener un Accuracy de 69.01% y un Kappa del 30.47\n",
      "Model: \"model_48\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_48 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_141 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_142 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 421\n",
      "Trainable params: 421\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 100\n",
      "119/119 [==============================] - 0s 59us/step\n",
      "test loss: 0.6398004749241997, test accuracy: 0.6218487620353699\n",
      "AUC ROC:  0.626777983920841\n",
      "Kappa:  0.14905450500556172\n",
      "[[57 20]\n",
      " [25 17]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.369691          -0.881824          -0.095656          -0.923999   \n",
      "1          -0.175875          -0.403800          -0.657709          -0.201259   \n",
      "2           0.894452          -0.189794           1.959063           0.169276   \n",
      "3           0.060782          -0.392075           0.826233          -0.048480   \n",
      "4          -1.116536          -0.923742          -1.238971          -0.919598   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0          -0.030645          -0.834931          -1.031650          -0.840942   \n",
      "1           1.691433          -0.672783          -0.119944          -0.440080   \n",
      "2          -0.403611          -1.036954           1.447615          -0.340767   \n",
      "3           1.789786          -0.552163           0.121028          -0.111355   \n",
      "4           0.313068          -1.160111           1.316032          -0.700013   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.677716            1.084098           -1.064999   \n",
      "1           0.339906            1.084098            0.504608   \n",
      "2          -0.846170           -0.515065           -0.699878   \n",
      "3           0.220614            1.084098            0.073241   \n",
      "4          -1.600210            1.084098           -1.072155   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -1.156623  \n",
      "1            0.931676  \n",
      "2            0.032355  \n",
      "3            1.176257  \n",
      "4            1.270095  \n",
      "(471, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (10, 10, 10), 'learning_rate_init': 0.006, 'max_iter': 10}, que permiten obtener un Accuracy de 69.97% y un Kappa del 6.67\n",
      "Model: \"model_49\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_49 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_143 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_144 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_145 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_146 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 361\n",
      "Trainable params: 361\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 10\n",
      "118/118 [==============================] - 0s 51us/step\n",
      "test loss: 0.5701536152322414, test accuracy: 0.7118644118309021\n",
      "AUC ROC:  0.6657487091222032\n",
      "Kappa:  0.17278350515463914\n",
      "[[76  7]\n",
      " [27  8]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Gramma y Hotel Marrakech"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           2.134448          -0.644769          -0.741425          -0.871269   \n",
      "1          -0.027862           1.690489           0.055411          -0.561004   \n",
      "2          -0.377201           2.125071          -0.304151           2.056775   \n",
      "3          -0.590231           0.143657          -0.601072           1.282246   \n",
      "4          -0.233782           0.378000           1.366350           0.941326   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           0.189298          -0.623423          -0.529336           0.357204   \n",
      "1           0.495091          -0.306711           0.886175          -0.561042   \n",
      "2          -0.998633          -0.428416           1.821160          -1.229826   \n",
      "3          -0.578812          -0.045810           1.094766           0.365693   \n",
      "4          -0.898589          -0.324641           1.117103           0.022453   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.325100            0.746582           -0.476439   \n",
      "1           1.431598            0.158038            0.235945   \n",
      "2          -0.732612           -1.686385           -0.297109   \n",
      "3           1.431598           -0.618841            0.417565   \n",
      "4           1.431598           -0.507270            0.477771   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.217416  \n",
      "1            1.435645  \n",
      "2            0.515440  \n",
      "3            0.135593  \n",
      "4           -0.297988  \n",
      "(360, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (400) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30,), 'learning_rate_init': 0.003, 'max_iter': 400}, que permiten obtener un Accuracy de 69.26% y un Kappa del 18.20\n",
      "Model: \"model_50\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_50 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_147 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_148 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 421\n",
      "Trainable params: 421\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 400\n",
      "90/90 [==============================] - 0s 56us/step\n",
      "test loss: 0.6487891912460327, test accuracy: 0.6555555462837219\n",
      "AUC ROC:  0.6119085592769803\n",
      "Kappa:  0.14364640883977908\n",
      "[[52  5]\n",
      " [26  7]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Gramma y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           2.134448          -0.644769          -0.741425          -0.871269   \n",
      "1          -0.027862           1.690489           0.055411          -0.561004   \n",
      "2          -0.377201           2.125071          -0.304151           2.056775   \n",
      "3          -0.590231           0.143657          -0.601072           1.282246   \n",
      "4          -0.233782           0.378000           1.366350           0.941326   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           0.189298          -0.623423          -0.529336           0.357204   \n",
      "1           0.495091          -0.306711           0.886175          -0.561042   \n",
      "2          -0.998633          -0.428416           1.821160          -1.229826   \n",
      "3          -0.578812          -0.045810           1.094766           0.365693   \n",
      "4          -0.898589          -0.324641           1.117103           0.022453   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.325100            0.746582           -0.476439   \n",
      "1           1.431598            0.158038            0.235945   \n",
      "2          -0.732612           -1.686385           -0.297109   \n",
      "3           1.431598           -0.618841            0.417565   \n",
      "4           1.431598           -0.507270            0.477771   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.217416  \n",
      "1            1.435645  \n",
      "2            0.515440  \n",
      "3            0.135593  \n",
      "4           -0.297988  \n",
      "(435, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30, 20, 10), 'learning_rate_init': 0.01, 'max_iter': 300}, que permiten obtener un Accuracy de 65.95% y un Kappa del 28.44\n",
      "Model: \"model_51\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_51 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_149 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_150 (Dense)            (None, 20)                620       \n",
      "_________________________________________________________________\n",
      "dense_151 (Dense)            (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_152 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 1,231\n",
      "Trainable params: 1,231\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 300\n",
      "109/109 [==============================] - 0s 73us/step\n",
      "test loss: 1.135751496897925, test accuracy: 0.6972476840019226\n",
      "AUC ROC:  0.7272053372868792\n",
      "Kappa:  0.33744704365444833\n",
      "[[54 17]\n",
      " [16 22]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Gramma y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           2.134448          -0.644769          -0.741425          -0.871269   \n",
      "1          -0.027862           1.690489           0.055411          -0.561004   \n",
      "2          -0.377201           2.125071          -0.304151           2.056775   \n",
      "3          -0.590231           0.143657          -0.601072           1.282246   \n",
      "4          -0.233782           0.378000           1.366350           0.941326   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           0.189298          -0.623423          -0.529336           0.357204   \n",
      "1           0.495091          -0.306711           0.886175          -0.561042   \n",
      "2          -0.998633          -0.428416           1.821160          -1.229826   \n",
      "3          -0.578812          -0.045810           1.094766           0.365693   \n",
      "4          -0.898589          -0.324641           1.117103           0.022453   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.325100            0.746582           -0.476439   \n",
      "1           1.431598            0.158038            0.235945   \n",
      "2          -0.732612           -1.686385           -0.297109   \n",
      "3           1.431598           -0.618841            0.417565   \n",
      "4           1.431598           -0.507270            0.477771   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.217416  \n",
      "1            1.435645  \n",
      "2            0.515440  \n",
      "3            0.135593  \n",
      "4           -0.297988  \n",
      "(432, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30,), 'learning_rate_init': 0.02, 'max_iter': 500}, que permiten obtener un Accuracy de 73.15% y un Kappa del 32.98\n",
      "Model: \"model_52\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_52 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_153 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_154 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 421\n",
      "Trainable params: 421\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 500\n",
      "108/108 [==============================] - 0s 318us/step\n",
      "test loss: 0.8460848066541884, test accuracy: 0.6018518805503845\n",
      "AUC ROC:  0.5864197530864197\n",
      "Kappa:  0.0444444444444444\n",
      "[[55 17]\n",
      " [26 10]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.580880          -0.990276           0.508438           0.658218   \n",
      "1          -0.886270           1.680400           1.708314           0.223308   \n",
      "2          -0.133384           0.811454          -1.227033          -0.981611   \n",
      "3          -0.551071           0.259668          -1.073758          -0.208557   \n",
      "4          -1.234016          -0.118636          -1.227033          -0.725200   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           1.668751          -0.289029          -0.581158           1.476860   \n",
      "1           0.928680          -0.795001           0.621735          -0.834425   \n",
      "2          -1.054713          -1.027505          -0.745868           0.198884   \n",
      "3           0.002105          -0.655399          -0.056661          -1.206025   \n",
      "4          -1.112673          -0.992824          -0.809889          -1.428072   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0           0.248923            1.428276           -0.870636   \n",
      "1          -0.922785           -0.428866           -1.169994   \n",
      "2           0.508413           -1.114895           -0.312476   \n",
      "3          -1.133355           -1.217881           -0.812411   \n",
      "4          -1.315307           -1.379943           -0.903603   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -0.174217  \n",
      "1           -0.427540  \n",
      "2            1.844563  \n",
      "3            1.844563  \n",
      "4            1.844563  \n",
      "(363, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30, 30), 'learning_rate_init': 0.008, 'max_iter': 2000}, que permiten obtener un Accuracy de 68.01% y un Kappa del 35.45\n",
      "Model: \"model_53\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_53 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_155 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_156 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_157 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_158 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 2,281\n",
      "Trainable params: 2,281\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 2000\n",
      "91/91 [==============================] - 0s 22us/step\n",
      "test loss: 2.334119492834741, test accuracy: 0.5604395866394043\n",
      "AUC ROC:  0.6426841085271318\n",
      "Kappa:  0.11392405063291144\n",
      "[[30 18]\n",
      " [22 21]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.580880          -0.990276           0.508438           0.658218   \n",
      "1          -0.886270           1.680400           1.708314           0.223308   \n",
      "2          -0.133384           0.811454          -1.227033          -0.981611   \n",
      "3          -0.551071           0.259668          -1.073758          -0.208557   \n",
      "4          -1.234016          -0.118636          -1.227033          -0.725200   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           1.668751          -0.289029          -0.581158           1.476860   \n",
      "1           0.928680          -0.795001           0.621735          -0.834425   \n",
      "2          -1.054713          -1.027505          -0.745868           0.198884   \n",
      "3           0.002105          -0.655399          -0.056661          -1.206025   \n",
      "4          -1.112673          -0.992824          -0.809889          -1.428072   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0           0.248923            1.428276           -0.870636   \n",
      "1          -0.922785           -0.428866           -1.169994   \n",
      "2           0.508413           -1.114895           -0.312476   \n",
      "3          -1.133355           -1.217881           -0.812411   \n",
      "4          -1.315307           -1.379943           -0.903603   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -0.174217  \n",
      "1           -0.427540  \n",
      "2            1.844563  \n",
      "3            1.844563  \n",
      "4            1.844563  \n",
      "(360, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30, 20, 10), 'learning_rate_init': 0.009, 'max_iter': 1000}, que permiten obtener un Accuracy de 72.96% y un Kappa del 39.86\n",
      "Model: \"model_54\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_54 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_159 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_160 (Dense)            (None, 20)                620       \n",
      "_________________________________________________________________\n",
      "dense_161 (Dense)            (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_162 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 1,231\n",
      "Trainable params: 1,231\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 1000\n",
      "90/90 [==============================] - 0s 56us/step\n",
      "test loss: 1.3178687890370686, test accuracy: 0.6333333253860474\n",
      "AUC ROC:  0.6229423868312757\n",
      "Kappa:  0.19512195121951215\n",
      "[[43 11]\n",
      " [22 14]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.752761          -1.114598          -0.348132           2.966304   \n",
      "1          -1.387006          -0.333039           0.041297          -0.917052   \n",
      "2          -0.628834           2.234144           0.613536          -0.978644   \n",
      "3           0.081693           1.765530          -0.365668           0.759057   \n",
      "4           0.056206           1.646501           0.508800           0.525847   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0          -0.925235           0.552064          -0.710296           0.412009   \n",
      "1           0.656635          -1.022407          -0.805166          -0.905135   \n",
      "2           0.836157          -0.735689          -0.059767          -1.571350   \n",
      "3          -1.136519          -0.071939          -0.412587          -1.310708   \n",
      "4           0.506842          -0.390517          -0.241209          -0.409725   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.784684           -1.667162           -0.834151   \n",
      "1          -0.708805            1.299875           -0.948816   \n",
      "2           1.278264           -1.103616           -1.153426   \n",
      "3           1.465231           -1.266573            0.040076   \n",
      "4           1.465231           -0.148455           -0.354779   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -1.566379  \n",
      "1           -1.413196  \n",
      "2            0.685062  \n",
      "3           -0.308065  \n",
      "4            0.188297  \n",
      "(435, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (75) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30, 30), 'learning_rate_init': 0.005, 'max_iter': 75}, que permiten obtener un Accuracy de 69.94% y un Kappa del 37.58\n",
      "Model: \"model_55\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_55 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_163 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_164 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_165 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_166 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 2,281\n",
      "Trainable params: 2,281\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 75\n",
      "109/109 [==============================] - 0s 156us/step\n",
      "test loss: 1.371637195075324, test accuracy: 0.6513761281967163\n",
      "AUC ROC:  0.6590277777777778\n",
      "Kappa:  0.26141226818830243\n",
      "[[49 15]\n",
      " [23 22]]\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import display, Markdown, Latex\n",
    "import itertools\n",
    "two_combs = list(itertools.combinations(range(len(companies)), 2))\n",
    "for comb in two_combs:\n",
    "    display(Markdown('## '+companies[comb[0]]+' y '+companies[comb[1]]))\n",
    "    X = df_n_ps_std_ch[comb[0]].append(df_n_ps_std_ch[comb[1]])\n",
    "    y = df_n_ps[comb[0]]['chosen'].append(df_n_ps[comb[1]]['chosen'])\n",
    "\n",
    "    print(X.head())\n",
    "    print(X.shape)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "    mlp = MLPClassifier(hidden_layer_sizes=(30,30,30))\n",
    "\n",
    "    activation_vec = ['logistic', 'relu', 'tanh']\n",
    "    max_iter_vec = [10, 20, 50, 75, 100, 200, 300, 400, 500, 1000, 2000]\n",
    "    hidden_layer_sizes_vec = [(10,), (20,), (30,), (10, 10), (20, 20), (30, 30), (20, 10), \n",
    "                              (10, 10, 10), (20, 20, 20), (30, 30, 30), (30, 20, 10)]\n",
    "    learning_rate_init_vec = [0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.02]\n",
    "    batch_size_vec = [10, 20, 40, 60, 80, 100, 150]\n",
    "\n",
    "\n",
    "    np.random.seed(1234)\n",
    "    parametros = {'activation': activation_vec,\n",
    "                  'max_iter':max_iter_vec,\n",
    "                  'hidden_layer_sizes': hidden_layer_sizes_vec,\n",
    "                  'learning_rate_init': learning_rate_init_vec#,\n",
    "                  #'batch_size': batch_size_vec\n",
    "                  }\n",
    "    scoring = {'kappa':make_scorer(cohen_kappa_score), 'accuracy':'accuracy'}\n",
    "    grid = GridSearchCV(mlp, param_grid=parametros, cv=5, scoring=scoring, refit='accuracy', n_jobs=-1, iid=True)\n",
    "\n",
    "    grid.fit(X_train, y_train)\n",
    "\n",
    "    print(\"Los parámetros del mejor modelo fueron {0}, que permiten obtener un Accuracy de {1:.2f}% y un Kappa del {2:.2f}\".format(\n",
    "        grid.best_params_, grid.best_score_*100, grid.cv_results_['mean_test_kappa'][grid.best_index_]*100))\n",
    "\n",
    "    n0=X_train.shape[1]\n",
    "    ### hidden_layer_sizes\n",
    "    ns = []\n",
    "    for i in range (len(grid.best_params_['hidden_layer_sizes'])):\n",
    "        ns.append(grid.best_params_['hidden_layer_sizes'][i])\n",
    "\n",
    "    ns.append(1)\n",
    "    lr = grid.best_params_['learning_rate_init']\n",
    "    epochs = grid.best_params_['max_iter']\n",
    "\n",
    "    input_tensor = Input(shape = (n0,))\n",
    "    hidden_outputs = [input_tensor]\n",
    "    if grid.best_params_['activation'] == 'logistic':\n",
    "        grid.best_params_['activation'] = 'sigmoid'\n",
    "    for i in range (len(ns)-1):\n",
    "        hidden_outputs.append(Dense(ns[i], activation = grid.best_params_['activation'])(hidden_outputs[i]))\n",
    "    classification_output = Dense(ns[-1], activation = 'sigmoid')(hidden_outputs[-1])\n",
    "\n",
    "    model = Model([input_tensor], [classification_output])\n",
    "    weights = model.get_weights()\n",
    "\n",
    "    print(model.summary())\n",
    "\n",
    "    model.set_weights(weights)\n",
    "    adam = keras.optimizers.Adam(lr=lr)\n",
    "    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    history = model.fit(X_train, y_train, epochs=epochs, validation_data=(X_test, y_test), verbose=0, \n",
    "                callbacks=[\n",
    "                    keras.callbacks.ReduceLROnPlateau(\n",
    "                        monitor='val_accuracy', factor=0.5, patience=10, min_delta=0.01, verbose=0\n",
    "                    )\n",
    "                ]\n",
    "             )\n",
    "\n",
    "    acc = history.history['accuracy']\n",
    "    val_acc = history.history['val_accuracy']\n",
    "    loss = history.history['loss']\n",
    "    val_loss = history.history['val_loss']\n",
    "\n",
    "    epochs = range(len(acc))\n",
    "\n",
    "    print(\"epochs: \"+str(len(acc)))\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(X_test, y_test)\n",
    "    print(\"test loss: {}, test accuracy: {}\".format(test_loss, test_acc))\n",
    "\n",
    "    y_pred = model.predict(X_test)\n",
    "    print(\"AUC ROC: \",roc_auc_score(y_test, y_pred))\n",
    "\n",
    "    y_pred = list(map(lambda i: int(i>=0.5), y_pred))\n",
    "    print(\"Kappa: \",cohen_kappa_score(y_test, y_pred))\n",
    "\n",
    "    print(confusion_matrix(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3-comp groups combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Gramma"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(786, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.01, 'max_iter': 10}, que permiten obtener un Accuracy de 72.50% y un Kappa del 3.55\n",
      "Model: \"model_56\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_56 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_167 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_168 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_169 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,351\n",
      "Trainable params: 1,351\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 10\n",
      "197/197 [==============================] - 0s 46us/step\n",
      "test loss: 0.7241835497357518, test accuracy: 0.6852791905403137\n",
      "AUC ROC:  0.5324561403508772\n",
      "Kappa:  0.05843354918285548\n",
      "[[127  13]\n",
      " [ 49   8]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Hotel Marrakech"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(714, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30,), 'learning_rate_init': 0.003, 'max_iter': 10}, que permiten obtener un Accuracy de 70.28% y un Kappa del 12.37\n",
      "Model: \"model_57\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_57 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_170 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_171 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 421\n",
      "Trainable params: 421\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 10\n",
      "179/179 [==============================] - 0s 73us/step\n",
      "test loss: 0.6142176179912503, test accuracy: 0.6871508359909058\n",
      "AUC ROC:  0.573225516621743\n",
      "Kappa:  0.025850340136054473\n",
      "[[119   7]\n",
      " [ 49   4]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(789, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.004, 'max_iter': 2000}, que permiten obtener un Accuracy de 68.36% y un Kappa del 28.80\n",
      "Model: \"model_58\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_58 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_172 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_173 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_174 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,351\n",
      "Trainable params: 1,351\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 2000\n",
      "198/198 [==============================] - 0s 45us/step\n",
      "test loss: 0.7680782693805117, test accuracy: 0.6161616444587708\n",
      "AUC ROC:  0.5500293944738389\n",
      "Kappa:  0.04238258877434131\n",
      "[[106  29]\n",
      " [ 47  16]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(786, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (10, 10), 'learning_rate_init': 0.002, 'max_iter': 100}, que permiten obtener un Accuracy de 70.63% y un Kappa del 9.04\n",
      "Model: \"model_59\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_59 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_175 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_176 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_177 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 251\n",
      "Trainable params: 251\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 100\n",
      "197/197 [==============================] - 0s 52us/step\n",
      "test loss: 0.5366850498364056, test accuracy: 0.7411167621612549\n",
      "AUC ROC:  0.6993006993006994\n",
      "Kappa:  0.14819838914794392\n",
      "[[138   5]\n",
      " [ 46   8]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma y Hotel Marrakech"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(675, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (20, 10), 'learning_rate_init': 0.006, 'max_iter': 20}, que permiten obtener un Accuracy de 70.95% y un Kappa del 6.16\n",
      "Model: \"model_60\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_60 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_178 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_179 (Dense)            (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_180 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 481\n",
      "Trainable params: 481\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 20\n",
      "169/169 [==============================] - 0s 41us/step\n",
      "test loss: 0.6914184241605228, test accuracy: 0.6153846383094788\n",
      "AUC ROC:  0.5368126747437093\n",
      "Kappa:  -0.05777563793933549\n",
      "[[102   9]\n",
      " [ 56   2]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(750, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (300) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (10,), 'learning_rate_init': 0.002, 'max_iter': 300}, que permiten obtener un Accuracy de 67.44% y un Kappa del 17.12\n",
      "Model: \"model_61\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_61 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_181 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_182 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 141\n",
      "Trainable params: 141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 300\n",
      "188/188 [==============================] - 0s 70us/step\n",
      "test loss: 0.630799282104411, test accuracy: 0.686170220375061\n",
      "AUC ROC:  0.5169412079817864\n",
      "Kappa:  0.03177374301675984\n",
      "[[125   6]\n",
      " [ 53   4]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(747, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20, 10), 'learning_rate_init': 0.003, 'max_iter': 20}, que permiten obtener un Accuracy de 69.64% y un Kappa del 8.56\n",
      "Model: \"model_62\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_62 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_183 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_184 (Dense)            (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_185 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 481\n",
      "Trainable params: 481\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 20\n",
      "187/187 [==============================] - 0s 84us/step\n",
      "test loss: 0.5751026288710813, test accuracy: 0.7058823704719543\n",
      "AUC ROC:  0.5953237410071942\n",
      "Kappa:  0.055295306328648874\n",
      "[[125  14]\n",
      " [ 41   7]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(678, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (20, 10), 'learning_rate_init': 0.006, 'max_iter': 500}, que permiten obtener un Accuracy de 66.14% y un Kappa del 14.83\n",
      "Model: \"model_63\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_63 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_186 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_187 (Dense)            (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_188 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 481\n",
      "Trainable params: 481\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 500\n",
      "170/170 [==============================] - 0s 59us/step\n",
      "test loss: 0.8022954926771276, test accuracy: 0.6411764621734619\n",
      "AUC ROC:  0.657051282051282\n",
      "Kappa:  0.2382841192889672\n",
      "[[75 29]\n",
      " [32 34]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(675, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20, 10), 'learning_rate_init': 0.004, 'max_iter': 50}, que permiten obtener un Accuracy de 70.36% y un Kappa del 12.18\n",
      "Model: \"model_64\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_64 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_189 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_190 (Dense)            (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_191 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 481\n",
      "Trainable params: 481\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 50\n",
      "169/169 [==============================] - 0s 83us/step\n",
      "test loss: 0.6866499541779242, test accuracy: 0.6863905191421509\n",
      "AUC ROC:  0.6460708782742681\n",
      "Kappa:  0.24138223087998634\n",
      "[[95 15]\n",
      " [38 21]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(750, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30, 30, 30), 'learning_rate_init': 0.008, 'max_iter': 200}, que permiten obtener un Accuracy de 70.28% y un Kappa del 33.86\n",
      "Model: \"model_65\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_65 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_192 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_193 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_194 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_195 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 2,281\n",
      "Trainable params: 2,281\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 200\n",
      "188/188 [==============================] - 0s 70us/step\n",
      "test loss: 1.2561726265765252, test accuracy: 0.6329787373542786\n",
      "AUC ROC:  0.6555916696761765\n",
      "Kappa:  0.20378099680824946\n",
      "[[86 31]\n",
      " [38 33]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma y Hotel Marrakech"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.369691          -0.881824          -0.095656          -0.923999   \n",
      "1          -0.175875          -0.403800          -0.657709          -0.201259   \n",
      "2           0.894452          -0.189794           1.959063           0.169276   \n",
      "3           0.060782          -0.392075           0.826233          -0.048480   \n",
      "4          -1.116536          -0.923742          -1.238971          -0.919598   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0          -0.030645          -0.834931          -1.031650          -0.840942   \n",
      "1           1.691433          -0.672783          -0.119944          -0.440080   \n",
      "2          -0.403611          -1.036954           1.447615          -0.340767   \n",
      "3           1.789786          -0.552163           0.121028          -0.111355   \n",
      "4           0.313068          -1.160111           1.316032          -0.700013   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.677716            1.084098           -1.064999   \n",
      "1           0.339906            1.084098            0.504608   \n",
      "2          -0.846170           -0.515065           -0.699878   \n",
      "3           0.220614            1.084098            0.073241   \n",
      "4          -1.600210            1.084098           -1.072155   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -1.156623  \n",
      "1            0.931676  \n",
      "2            0.032355  \n",
      "3            1.176257  \n",
      "4            1.270095  \n",
      "(615, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'logistic', 'hidden_layer_sizes': (10, 10), 'learning_rate_init': 0.005, 'max_iter': 500}, que permiten obtener un Accuracy de 69.63% y un Kappa del 5.43\n",
      "Model: \"model_66\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_66 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_196 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_197 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_198 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 251\n",
      "Trainable params: 251\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 500\n",
      "154/154 [==============================] - 0s 104us/step\n",
      "test loss: 0.6234010992112098, test accuracy: 0.6818181872367859\n",
      "AUC ROC:  0.5327502429543245\n",
      "Kappa:  0.0\n",
      "[[105   0]\n",
      " [ 49   0]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.369691          -0.881824          -0.095656          -0.923999   \n",
      "1          -0.175875          -0.403800          -0.657709          -0.201259   \n",
      "2           0.894452          -0.189794           1.959063           0.169276   \n",
      "3           0.060782          -0.392075           0.826233          -0.048480   \n",
      "4          -1.116536          -0.923742          -1.238971          -0.919598   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0          -0.030645          -0.834931          -1.031650          -0.840942   \n",
      "1           1.691433          -0.672783          -0.119944          -0.440080   \n",
      "2          -0.403611          -1.036954           1.447615          -0.340767   \n",
      "3           1.789786          -0.552163           0.121028          -0.111355   \n",
      "4           0.313068          -1.160111           1.316032          -0.700013   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.677716            1.084098           -1.064999   \n",
      "1           0.339906            1.084098            0.504608   \n",
      "2          -0.846170           -0.515065           -0.699878   \n",
      "3           0.220614            1.084098            0.073241   \n",
      "4          -1.600210            1.084098           -1.072155   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -1.156623  \n",
      "1            0.931676  \n",
      "2            0.032355  \n",
      "3            1.176257  \n",
      "4            1.270095  \n",
      "(690, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (20, 20), 'learning_rate_init': 0.01, 'max_iter': 300}, que permiten obtener un Accuracy de 65.76% y un Kappa del 14.26\n",
      "Model: \"model_67\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_67 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_199 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_200 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_201 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 701\n",
      "Trainable params: 701\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 300\n",
      "173/173 [==============================] - 0s 83us/step\n",
      "test loss: 0.7070098405628535, test accuracy: 0.6358381509780884\n",
      "AUC ROC:  0.5863678804855276\n",
      "Kappa:  0.0721886439090832\n",
      "[[96 23]\n",
      " [40 14]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.369691          -0.881824          -0.095656          -0.923999   \n",
      "1          -0.175875          -0.403800          -0.657709          -0.201259   \n",
      "2           0.894452          -0.189794           1.959063           0.169276   \n",
      "3           0.060782          -0.392075           0.826233          -0.048480   \n",
      "4          -1.116536          -0.923742          -1.238971          -0.919598   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0          -0.030645          -0.834931          -1.031650          -0.840942   \n",
      "1           1.691433          -0.672783          -0.119944          -0.440080   \n",
      "2          -0.403611          -1.036954           1.447615          -0.340767   \n",
      "3           1.789786          -0.552163           0.121028          -0.111355   \n",
      "4           0.313068          -1.160111           1.316032          -0.700013   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.677716            1.084098           -1.064999   \n",
      "1           0.339906            1.084098            0.504608   \n",
      "2          -0.846170           -0.515065           -0.699878   \n",
      "3           0.220614            1.084098            0.073241   \n",
      "4          -1.600210            1.084098           -1.072155   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -1.156623  \n",
      "1            0.931676  \n",
      "2            0.032355  \n",
      "3            1.176257  \n",
      "4            1.270095  \n",
      "(687, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.004, 'max_iter': 10}, que permiten obtener un Accuracy de 70.87% y un Kappa del 3.94\n",
      "Model: \"model_68\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_68 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_202 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_203 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_204 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,351\n",
      "Trainable params: 1,351\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 10\n",
      "172/172 [==============================] - 0s 64us/step\n",
      "test loss: 0.6291286058204119, test accuracy: 0.6569767594337463\n",
      "AUC ROC:  0.5501823370857777\n",
      "Kappa:  -0.010756972111553686\n",
      "[[109  10]\n",
      " [ 49   4]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.369691          -0.881824          -0.095656          -0.923999   \n",
      "1          -0.175875          -0.403800          -0.657709          -0.201259   \n",
      "2           0.894452          -0.189794           1.959063           0.169276   \n",
      "3           0.060782          -0.392075           0.826233          -0.048480   \n",
      "4          -1.116536          -0.923742          -1.238971          -0.919598   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0          -0.030645          -0.834931          -1.031650          -0.840942   \n",
      "1           1.691433          -0.672783          -0.119944          -0.440080   \n",
      "2          -0.403611          -1.036954           1.447615          -0.340767   \n",
      "3           1.789786          -0.552163           0.121028          -0.111355   \n",
      "4           0.313068          -1.160111           1.316032          -0.700013   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.677716            1.084098           -1.064999   \n",
      "1           0.339906            1.084098            0.504608   \n",
      "2          -0.846170           -0.515065           -0.699878   \n",
      "3           0.220614            1.084098            0.073241   \n",
      "4          -1.600210            1.084098           -1.072155   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -1.156623  \n",
      "1            0.931676  \n",
      "2            0.032355  \n",
      "3            1.176257  \n",
      "4            1.270095  \n",
      "(618, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30,), 'learning_rate_init': 0.002, 'max_iter': 200}, que permiten obtener un Accuracy de 66.52% y un Kappa del 24.43\n",
      "Model: \"model_69\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_69 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_205 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_206 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 421\n",
      "Trainable params: 421\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 200\n",
      "155/155 [==============================] - 0s 64us/step\n",
      "test loss: 0.6416249836644818, test accuracy: 0.6129032373428345\n",
      "AUC ROC:  0.6015183615819208\n",
      "Kappa:  0.11546509416016737\n",
      "[[77 19]\n",
      " [41 18]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.369691          -0.881824          -0.095656          -0.923999   \n",
      "1          -0.175875          -0.403800          -0.657709          -0.201259   \n",
      "2           0.894452          -0.189794           1.959063           0.169276   \n",
      "3           0.060782          -0.392075           0.826233          -0.048480   \n",
      "4          -1.116536          -0.923742          -1.238971          -0.919598   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0          -0.030645          -0.834931          -1.031650          -0.840942   \n",
      "1           1.691433          -0.672783          -0.119944          -0.440080   \n",
      "2          -0.403611          -1.036954           1.447615          -0.340767   \n",
      "3           1.789786          -0.552163           0.121028          -0.111355   \n",
      "4           0.313068          -1.160111           1.316032          -0.700013   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.677716            1.084098           -1.064999   \n",
      "1           0.339906            1.084098            0.504608   \n",
      "2          -0.846170           -0.515065           -0.699878   \n",
      "3           0.220614            1.084098            0.073241   \n",
      "4          -1.600210            1.084098           -1.072155   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -1.156623  \n",
      "1            0.931676  \n",
      "2            0.032355  \n",
      "3            1.176257  \n",
      "4            1.270095  \n",
      "(615, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (500) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (20, 10), 'learning_rate_init': 0.002, 'max_iter': 500}, que permiten obtener un Accuracy de 70.93% y un Kappa del 31.29\n",
      "Model: \"model_70\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_70 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_207 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_208 (Dense)            (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_209 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 481\n",
      "Trainable params: 481\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 500\n",
      "154/154 [==============================] - 0s 45us/step\n",
      "test loss: 0.6087145596355586, test accuracy: 0.6688311696052551\n",
      "AUC ROC:  0.6798469387755102\n",
      "Kappa:  0.15892053973013498\n",
      "[[92  6]\n",
      " [45 11]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.369691          -0.881824          -0.095656          -0.923999   \n",
      "1          -0.175875          -0.403800          -0.657709          -0.201259   \n",
      "2           0.894452          -0.189794           1.959063           0.169276   \n",
      "3           0.060782          -0.392075           0.826233          -0.048480   \n",
      "4          -1.116536          -0.923742          -1.238971          -0.919598   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0          -0.030645          -0.834931          -1.031650          -0.840942   \n",
      "1           1.691433          -0.672783          -0.119944          -0.440080   \n",
      "2          -0.403611          -1.036954           1.447615          -0.340767   \n",
      "3           1.789786          -0.552163           0.121028          -0.111355   \n",
      "4           0.313068          -1.160111           1.316032          -0.700013   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.677716            1.084098           -1.064999   \n",
      "1           0.339906            1.084098            0.504608   \n",
      "2          -0.846170           -0.515065           -0.699878   \n",
      "3           0.220614            1.084098            0.073241   \n",
      "4          -1.600210            1.084098           -1.072155   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -1.156623  \n",
      "1            0.931676  \n",
      "2            0.032355  \n",
      "3            1.176257  \n",
      "4            1.270095  \n",
      "(690, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (10, 10), 'learning_rate_init': 0.006, 'max_iter': 20}, que permiten obtener un Accuracy de 64.80% y un Kappa del 9.32\n",
      "Model: \"model_71\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_71 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_210 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_211 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_212 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 251\n",
      "Trainable params: 251\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 20\n",
      "173/173 [==============================] - 0s 40us/step\n",
      "test loss: 0.6285514490452805, test accuracy: 0.6531791687011719\n",
      "AUC ROC:  0.6325196163905841\n",
      "Kappa:  0.16236281471917358\n",
      "[[95 16]\n",
      " [44 18]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Gramma y Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           2.134448          -0.644769          -0.741425          -0.871269   \n",
      "1          -0.027862           1.690489           0.055411          -0.561004   \n",
      "2          -0.377201           2.125071          -0.304151           2.056775   \n",
      "3          -0.590231           0.143657          -0.601072           1.282246   \n",
      "4          -0.233782           0.378000           1.366350           0.941326   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           0.189298          -0.623423          -0.529336           0.357204   \n",
      "1           0.495091          -0.306711           0.886175          -0.561042   \n",
      "2          -0.998633          -0.428416           1.821160          -1.229826   \n",
      "3          -0.578812          -0.045810           1.094766           0.365693   \n",
      "4          -0.898589          -0.324641           1.117103           0.022453   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.325100            0.746582           -0.476439   \n",
      "1           1.431598            0.158038            0.235945   \n",
      "2          -0.732612           -1.686385           -0.297109   \n",
      "3           1.431598           -0.618841            0.417565   \n",
      "4           1.431598           -0.507270            0.477771   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.217416  \n",
      "1            1.435645  \n",
      "2            0.515440  \n",
      "3            0.135593  \n",
      "4           -0.297988  \n",
      "(579, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 30, 30), 'learning_rate_init': 0.01, 'max_iter': 100}, que permiten obtener un Accuracy de 67.28% y un Kappa del 30.32\n",
      "Model: \"model_72\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_72 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_213 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_214 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_215 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_216 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 2,281\n",
      "Trainable params: 2,281\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 100\n",
      "145/145 [==============================] - 0s 41us/step\n",
      "test loss: 2.5417338700130068, test accuracy: 0.6137930750846863\n",
      "AUC ROC:  0.5913078149920254\n",
      "Kappa:  0.21011673151750976\n",
      "[[56 32]\n",
      " [24 33]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Gramma y Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           2.134448          -0.644769          -0.741425          -0.871269   \n",
      "1          -0.027862           1.690489           0.055411          -0.561004   \n",
      "2          -0.377201           2.125071          -0.304151           2.056775   \n",
      "3          -0.590231           0.143657          -0.601072           1.282246   \n",
      "4          -0.233782           0.378000           1.366350           0.941326   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           0.189298          -0.623423          -0.529336           0.357204   \n",
      "1           0.495091          -0.306711           0.886175          -0.561042   \n",
      "2          -0.998633          -0.428416           1.821160          -1.229826   \n",
      "3          -0.578812          -0.045810           1.094766           0.365693   \n",
      "4          -0.898589          -0.324641           1.117103           0.022453   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.325100            0.746582           -0.476439   \n",
      "1           1.431598            0.158038            0.235945   \n",
      "2          -0.732612           -1.686385           -0.297109   \n",
      "3           1.431598           -0.618841            0.417565   \n",
      "4           1.431598           -0.507270            0.477771   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.217416  \n",
      "1            1.435645  \n",
      "2            0.515440  \n",
      "3            0.135593  \n",
      "4           -0.297988  \n",
      "(576, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (10, 10), 'learning_rate_init': 0.002, 'max_iter': 100}, que permiten obtener un Accuracy de 71.30% y un Kappa del 24.50\n",
      "Model: \"model_73\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_73 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_217 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_218 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_219 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 251\n",
      "Trainable params: 251\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 100\n",
      "144/144 [==============================] - 0s 35us/step\n",
      "test loss: 0.6480367713504367, test accuracy: 0.6388888955116272\n",
      "AUC ROC:  0.600334448160535\n",
      "Kappa:  0.08771929824561409\n",
      "[[83  9]\n",
      " [43  9]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Gramma y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           2.134448          -0.644769          -0.741425          -0.871269   \n",
      "1          -0.027862           1.690489           0.055411          -0.561004   \n",
      "2          -0.377201           2.125071          -0.304151           2.056775   \n",
      "3          -0.590231           0.143657          -0.601072           1.282246   \n",
      "4          -0.233782           0.378000           1.366350           0.941326   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           0.189298          -0.623423          -0.529336           0.357204   \n",
      "1           0.495091          -0.306711           0.886175          -0.561042   \n",
      "2          -0.998633          -0.428416           1.821160          -1.229826   \n",
      "3          -0.578812          -0.045810           1.094766           0.365693   \n",
      "4          -0.898589          -0.324641           1.117103           0.022453   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.325100            0.746582           -0.476439   \n",
      "1           1.431598            0.158038            0.235945   \n",
      "2          -0.732612           -1.686385           -0.297109   \n",
      "3           1.431598           -0.618841            0.417565   \n",
      "4           1.431598           -0.507270            0.477771   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.217416  \n",
      "1            1.435645  \n",
      "2            0.515440  \n",
      "3            0.135593  \n",
      "4           -0.297988  \n",
      "(651, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (10, 10), 'learning_rate_init': 0.01, 'max_iter': 50}, que permiten obtener un Accuracy de 67.42% y un Kappa del 25.69\n",
      "Model: \"model_74\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_74 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_220 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_221 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_222 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 251\n",
      "Trainable params: 251\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 50\n",
      "163/163 [==============================] - 0s 49us/step\n",
      "test loss: 0.7894706199505578, test accuracy: 0.5950919985771179\n",
      "AUC ROC:  0.5855987055016181\n",
      "Kappa:  0.1474084640989064\n",
      "[[67 36]\n",
      " [30 30]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Hotel Marrakech y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.580880          -0.990276           0.508438           0.658218   \n",
      "1          -0.886270           1.680400           1.708314           0.223308   \n",
      "2          -0.133384           0.811454          -1.227033          -0.981611   \n",
      "3          -0.551071           0.259668          -1.073758          -0.208557   \n",
      "4          -1.234016          -0.118636          -1.227033          -0.725200   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           1.668751          -0.289029          -0.581158           1.476860   \n",
      "1           0.928680          -0.795001           0.621735          -0.834425   \n",
      "2          -1.054713          -1.027505          -0.745868           0.198884   \n",
      "3           0.002105          -0.655399          -0.056661          -1.206025   \n",
      "4          -1.112673          -0.992824          -0.809889          -1.428072   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0           0.248923            1.428276           -0.870636   \n",
      "1          -0.922785           -0.428866           -1.169994   \n",
      "2           0.508413           -1.114895           -0.312476   \n",
      "3          -1.133355           -1.217881           -0.812411   \n",
      "4          -1.315307           -1.379943           -0.903603   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -0.174217  \n",
      "1           -0.427540  \n",
      "2            1.844563  \n",
      "3            1.844563  \n",
      "4            1.844563  \n",
      "(579, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (10, 10), 'learning_rate_init': 0.009, 'max_iter': 1000}, que permiten obtener un Accuracy de 65.90% y un Kappa del 28.78\n",
      "Model: \"model_75\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_75 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_223 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_224 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_225 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 251\n",
      "Trainable params: 251\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 1000\n",
      "145/145 [==============================] - 0s 71us/step\n",
      "test loss: 0.6882458423746043, test accuracy: 0.634482741355896\n",
      "AUC ROC:  0.6715686274509804\n",
      "Kappa:  0.24471744471744472\n",
      "[[59 26]\n",
      " [27 33]]\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import display, Markdown, Latex\n",
    "import itertools\n",
    "three_combs = list(itertools.combinations(range(len(companies)), 3))\n",
    "for comb in three_combs:\n",
    "    title = '## '\n",
    "    for no in comb[:-1]:\n",
    "        title+=companies[no]\n",
    "        title+=' y '\n",
    "    title+=companies[comb[-1]]\n",
    "    display(Markdown(title))\n",
    "    X = df_n_ps_std_ch[comb[0]]\n",
    "    y = df_n_ps[comb[0]]['chosen']\n",
    "    for no in comb[1:]:\n",
    "        X= X.append(df_n_ps_std_ch[no])\n",
    "        y= y.append(df_n_ps[no]['chosen'])\n",
    "    \n",
    "\n",
    "    print(X.head())\n",
    "    print(X.shape)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "    mlp = MLPClassifier(hidden_layer_sizes=(30,30,30))\n",
    "\n",
    "    activation_vec = ['logistic', 'relu', 'tanh']\n",
    "    max_iter_vec = [10, 20, 50, 75, 100, 200, 300, 400, 500, 1000, 2000]\n",
    "    hidden_layer_sizes_vec = [(10,), (20,), (30,), (10, 10), (20, 20), (30, 30), (20, 10), \n",
    "                              (10, 10, 10), (20, 20, 20), (30, 30, 30), (30, 20, 10)]\n",
    "    learning_rate_init_vec = [0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.02]\n",
    "    batch_size_vec = [10, 20, 40, 60, 80, 100, 150]\n",
    "\n",
    "\n",
    "    np.random.seed(1234)\n",
    "    parametros = {'activation': activation_vec,\n",
    "                  'max_iter':max_iter_vec,\n",
    "                  'hidden_layer_sizes': hidden_layer_sizes_vec,\n",
    "                  'learning_rate_init': learning_rate_init_vec#,\n",
    "                  #'batch_size': batch_size_vec\n",
    "                  }\n",
    "    scoring = {'kappa':make_scorer(cohen_kappa_score), 'accuracy':'accuracy'}\n",
    "    grid = GridSearchCV(mlp, param_grid=parametros, cv=5, scoring=scoring, refit='accuracy', n_jobs=-1, iid=True)\n",
    "\n",
    "    grid.fit(X_train, y_train)\n",
    "\n",
    "    print(\"Los parámetros del mejor modelo fueron {0}, que permiten obtener un Accuracy de {1:.2f}% y un Kappa del {2:.2f}\".format(\n",
    "        grid.best_params_, grid.best_score_*100, grid.cv_results_['mean_test_kappa'][grid.best_index_]*100))\n",
    "\n",
    "    n0=X_train.shape[1]\n",
    "    ### hidden_layer_sizes\n",
    "    ns = []\n",
    "    for i in range (len(grid.best_params_['hidden_layer_sizes'])):\n",
    "        ns.append(grid.best_params_['hidden_layer_sizes'][i])\n",
    "\n",
    "    ns.append(1)\n",
    "    lr = grid.best_params_['learning_rate_init']\n",
    "    epochs = grid.best_params_['max_iter']\n",
    "\n",
    "    input_tensor = Input(shape = (n0,))\n",
    "    hidden_outputs = [input_tensor]\n",
    "    if grid.best_params_['activation'] == 'logistic':\n",
    "        grid.best_params_['activation'] = 'sigmoid'\n",
    "    for i in range (len(ns)-1):\n",
    "        hidden_outputs.append(Dense(ns[i], activation = grid.best_params_['activation'])(hidden_outputs[i]))\n",
    "    classification_output = Dense(ns[-1], activation = 'sigmoid')(hidden_outputs[-1])\n",
    "\n",
    "    model = Model([input_tensor], [classification_output])\n",
    "    weights = model.get_weights()\n",
    "\n",
    "    print(model.summary())\n",
    "\n",
    "    model.set_weights(weights)\n",
    "    adam = keras.optimizers.Adam(lr=lr)\n",
    "    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    history = model.fit(X_train, y_train, epochs=epochs, validation_data=(X_test, y_test), verbose=0, \n",
    "                callbacks=[\n",
    "                    keras.callbacks.ReduceLROnPlateau(\n",
    "                        monitor='val_accuracy', factor=0.5, patience=10, min_delta=0.01, verbose=0\n",
    "                    )\n",
    "                ]\n",
    "             )\n",
    "\n",
    "    acc = history.history['accuracy']\n",
    "    val_acc = history.history['val_accuracy']\n",
    "    loss = history.history['loss']\n",
    "    val_loss = history.history['val_loss']\n",
    "\n",
    "    epochs = range(len(acc))\n",
    "\n",
    "    print(\"epochs: \"+str(len(acc)))\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(X_test, y_test)\n",
    "    print(\"test loss: {}, test accuracy: {}\".format(test_loss, test_acc))\n",
    "\n",
    "    y_pred = model.predict(X_test)\n",
    "    print(\"AUC ROC: \",roc_auc_score(y_test, y_pred))\n",
    "\n",
    "    y_pred = list(map(lambda i: int(i>=0.5), y_pred))\n",
    "    print(\"Kappa: \",cohen_kappa_score(y_test, y_pred))\n",
    "\n",
    "    print(confusion_matrix(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4-comp groups combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Gramma y Hotel Marrakech"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(930, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (10, 10), 'learning_rate_init': 0.007, 'max_iter': 20}, que permiten obtener un Accuracy de 69.58% y un Kappa del 3.22\n",
      "Model: \"model_91\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_91 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_276 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_277 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_278 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 251\n",
      "Trainable params: 251\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 20\n",
      "233/233 [==============================] - 0s 30us/step\n",
      "test loss: 0.6367101014427873, test accuracy: 0.6824034452438354\n",
      "AUC ROC:  0.5125204137180185\n",
      "Kappa:  -0.06524156678611148\n",
      "[[159   8]\n",
      " [ 66   0]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Gramma y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(1005, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (30, 20, 10), 'learning_rate_init': 0.002, 'max_iter': 500}, que permiten obtener un Accuracy de 69.46% y un Kappa del 11.36\n",
      "Model: \"model_92\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_92 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_279 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_280 (Dense)            (None, 20)                620       \n",
      "_________________________________________________________________\n",
      "dense_281 (Dense)            (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_282 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 1,231\n",
      "Trainable params: 1,231\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 500\n",
      "252/252 [==============================] - 0s 44us/step\n",
      "test loss: 0.7708528051300655, test accuracy: 0.6111111044883728\n",
      "AUC ROC:  0.5152439024390243\n",
      "Kappa:  0.043680297397769574\n",
      "[[135  29]\n",
      " [ 69  19]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Gramma y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(1002, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (75) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20, 10), 'learning_rate_init': 0.002, 'max_iter': 75}, que permiten obtener un Accuracy de 71.11% y un Kappa del 6.64\n",
      "Model: \"model_93\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_93 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_283 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_284 (Dense)            (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_285 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 481\n",
      "Trainable params: 481\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 75\n",
      "251/251 [==============================] - 0s 32us/step\n",
      "test loss: 0.5970159234753643, test accuracy: 0.7211155295372009\n",
      "AUC ROC:  0.5729415512024207\n",
      "Kappa:  0.08679833679833693\n",
      "[[173   9]\n",
      " [ 61   8]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(933, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20, 20), 'learning_rate_init': 0.009, 'max_iter': 10}, que permiten obtener un Accuracy de 68.81% y un Kappa del 13.69\n",
      "Model: \"model_94\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_94 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_286 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_287 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_288 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 701\n",
      "Trainable params: 701\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 10\n",
      "234/234 [==============================] - 0s 43us/step\n",
      "test loss: 0.7897408105369307, test accuracy: 0.5769230723381042\n",
      "AUC ROC:  0.5008487654320988\n",
      "Kappa:  0.01830663615560635\n",
      "[[116  28]\n",
      " [ 71  19]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(930, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (20, 20, 20), 'learning_rate_init': 0.003, 'max_iter': 50}, que permiten obtener un Accuracy de 70.44% y un Kappa del 17.85\n",
      "Model: \"model_95\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_95 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_289 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_290 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_291 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_292 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 1,121\n",
      "Trainable params: 1,121\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 50\n",
      "233/233 [==============================] - 0s 43us/step\n",
      "test loss: 0.7573998828814265, test accuracy: 0.6266094446182251\n",
      "AUC ROC:  0.5691480562448304\n",
      "Kappa:  0.08914850595371826\n",
      "[[124  31]\n",
      " [ 56  22]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(1005, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (10, 10), 'learning_rate_init': 0.005, 'max_iter': 500}, que permiten obtener un Accuracy de 68.39% y un Kappa del 6.74\n",
      "Model: \"model_96\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_96 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_293 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_294 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_295 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 251\n",
      "Trainable params: 251\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 500\n",
      "252/252 [==============================] - 0s 43us/step\n",
      "test loss: 0.6699253538298229, test accuracy: 0.6230158805847168\n",
      "AUC ROC:  0.6050182670434963\n",
      "Kappa:  0.07852193995381074\n",
      "[[136  27]\n",
      " [ 68  21]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma y Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(894, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (20, 10), 'learning_rate_init': 0.004, 'max_iter': 20}, que permiten obtener un Accuracy de 67.31% y un Kappa del 8.05\n",
      "Model: \"model_97\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_97 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_296 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_297 (Dense)            (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dense_298 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 481\n",
      "Trainable params: 481\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 20\n",
      "224/224 [==============================] - 0s 45us/step\n",
      "test loss: 0.7365582415035793, test accuracy: 0.5714285969734192\n",
      "AUC ROC:  0.5238095238095237\n",
      "Kappa:  -0.032258064516129004\n",
      "[[116  24]\n",
      " [ 72  12]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma y Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(891, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'logistic', 'hidden_layer_sizes': (30,), 'learning_rate_init': 0.009, 'max_iter': 10}, que permiten obtener un Accuracy de 68.71% y un Kappa del 1.29\n",
      "Model: \"model_98\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_98 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_299 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_300 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 421\n",
      "Trainable params: 421\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 10\n",
      "223/223 [==============================] - 0s 31us/step\n",
      "test loss: 0.6403322003347457, test accuracy: 0.681614339351654\n",
      "AUC ROC:  0.5250185322461083\n",
      "Kappa:  0.0\n",
      "[[152   0]\n",
      " [ 71   0]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(966, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (75) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (10,), 'learning_rate_init': 0.007, 'max_iter': 75}, que permiten obtener un Accuracy de 68.51% y un Kappa del 17.09\n",
      "Model: \"model_99\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_99 (InputLayer)        (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_301 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_302 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 141\n",
      "Trainable params: 141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 75\n",
      "242/242 [==============================] - 0s 33us/step\n",
      "test loss: 0.6589670949730991, test accuracy: 0.6487603187561035\n",
      "AUC ROC:  0.576627486437613\n",
      "Kappa:  0.07250428352421323\n",
      "[[145  13]\n",
      " [ 72  12]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Hotel Marrakech y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(894, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (75) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (20,), 'learning_rate_init': 0.006, 'max_iter': 75}, que permiten obtener un Accuracy de 65.82% y un Kappa del 18.93\n",
      "Model: \"model_100\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_100 (InputLayer)       (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_303 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_304 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 281\n",
      "Trainable params: 281\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 75\n",
      "224/224 [==============================] - 0s 134us/step\n",
      "test loss: 0.6292154107775006, test accuracy: 0.6696428656578064\n",
      "AUC ROC:  0.6475869809203142\n",
      "Kappa:  0.23535381492757634\n",
      "[[118  25]\n",
      " [ 49  32]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma y Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.369691          -0.881824          -0.095656          -0.923999   \n",
      "1          -0.175875          -0.403800          -0.657709          -0.201259   \n",
      "2           0.894452          -0.189794           1.959063           0.169276   \n",
      "3           0.060782          -0.392075           0.826233          -0.048480   \n",
      "4          -1.116536          -0.923742          -1.238971          -0.919598   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0          -0.030645          -0.834931          -1.031650          -0.840942   \n",
      "1           1.691433          -0.672783          -0.119944          -0.440080   \n",
      "2          -0.403611          -1.036954           1.447615          -0.340767   \n",
      "3           1.789786          -0.552163           0.121028          -0.111355   \n",
      "4           0.313068          -1.160111           1.316032          -0.700013   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.677716            1.084098           -1.064999   \n",
      "1           0.339906            1.084098            0.504608   \n",
      "2          -0.846170           -0.515065           -0.699878   \n",
      "3           0.220614            1.084098            0.073241   \n",
      "4          -1.600210            1.084098           -1.072155   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -1.156623  \n",
      "1            0.931676  \n",
      "2            0.032355  \n",
      "3            1.176257  \n",
      "4            1.270095  \n",
      "(834, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'logistic', 'hidden_layer_sizes': (10, 10), 'learning_rate_init': 0.007, 'max_iter': 1000}, que permiten obtener un Accuracy de 65.44% y un Kappa del 7.01\n",
      "Model: \"model_101\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_101 (InputLayer)       (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_305 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_306 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_307 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 251\n",
      "Trainable params: 251\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 1000\n",
      "209/209 [==============================] - 0s 48us/step\n",
      "test loss: 0.6531446550451397, test accuracy: 0.6459330320358276\n",
      "AUC ROC:  0.5562562562562563\n",
      "Kappa:  0.052328431372549056\n",
      "[[128   7]\n",
      " [ 67   7]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma y Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.369691          -0.881824          -0.095656          -0.923999   \n",
      "1          -0.175875          -0.403800          -0.657709          -0.201259   \n",
      "2           0.894452          -0.189794           1.959063           0.169276   \n",
      "3           0.060782          -0.392075           0.826233          -0.048480   \n",
      "4          -1.116536          -0.923742          -1.238971          -0.919598   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0          -0.030645          -0.834931          -1.031650          -0.840942   \n",
      "1           1.691433          -0.672783          -0.119944          -0.440080   \n",
      "2          -0.403611          -1.036954           1.447615          -0.340767   \n",
      "3           1.789786          -0.552163           0.121028          -0.111355   \n",
      "4           0.313068          -1.160111           1.316032          -0.700013   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.677716            1.084098           -1.064999   \n",
      "1           0.339906            1.084098            0.504608   \n",
      "2          -0.846170           -0.515065           -0.699878   \n",
      "3           0.220614            1.084098            0.073241   \n",
      "4          -1.600210            1.084098           -1.072155   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -1.156623  \n",
      "1            0.931676  \n",
      "2            0.032355  \n",
      "3            1.176257  \n",
      "4            1.270095  \n",
      "(831, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (20, 20, 20), 'learning_rate_init': 0.005, 'max_iter': 20}, que permiten obtener un Accuracy de 69.50% y un Kappa del 11.65\n",
      "Model: \"model_102\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_102 (InputLayer)       (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_308 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_309 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_310 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_311 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 1,121\n",
      "Trainable params: 1,121\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 20\n",
      "208/208 [==============================] - 0s 312us/step\n",
      "test loss: 0.642515079333232, test accuracy: 0.6490384340286255\n",
      "AUC ROC:  0.6360435339308579\n",
      "Kappa:  0.12995645198258088\n",
      "[[114  28]\n",
      " [ 45  21]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.369691          -0.881824          -0.095656          -0.923999   \n",
      "1          -0.175875          -0.403800          -0.657709          -0.201259   \n",
      "2           0.894452          -0.189794           1.959063           0.169276   \n",
      "3           0.060782          -0.392075           0.826233          -0.048480   \n",
      "4          -1.116536          -0.923742          -1.238971          -0.919598   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0          -0.030645          -0.834931          -1.031650          -0.840942   \n",
      "1           1.691433          -0.672783          -0.119944          -0.440080   \n",
      "2          -0.403611          -1.036954           1.447615          -0.340767   \n",
      "3           1.789786          -0.552163           0.121028          -0.111355   \n",
      "4           0.313068          -1.160111           1.316032          -0.700013   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.677716            1.084098           -1.064999   \n",
      "1           0.339906            1.084098            0.504608   \n",
      "2          -0.846170           -0.515065           -0.699878   \n",
      "3           0.220614            1.084098            0.073241   \n",
      "4          -1.600210            1.084098           -1.072155   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -1.156623  \n",
      "1            0.931676  \n",
      "2            0.032355  \n",
      "3            1.176257  \n",
      "4            1.270095  \n",
      "(906, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.007, 'max_iter': 500}, que permiten obtener un Accuracy de 67.16% y un Kappa del 26.77\n",
      "Model: \"model_103\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_103 (InputLayer)       (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_312 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_313 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_314 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,351\n",
      "Trainable params: 1,351\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 500\n",
      "227/227 [==============================] - 0s 79us/step\n",
      "test loss: 0.8290954975304624, test accuracy: 0.6299559473991394\n",
      "AUC ROC:  0.6125451729478576\n",
      "Kappa:  0.14844587352625938\n",
      "[[113  36]\n",
      " [ 48  30]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Hotel Marrakech y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.369691          -0.881824          -0.095656          -0.923999   \n",
      "1          -0.175875          -0.403800          -0.657709          -0.201259   \n",
      "2           0.894452          -0.189794           1.959063           0.169276   \n",
      "3           0.060782          -0.392075           0.826233          -0.048480   \n",
      "4          -1.116536          -0.923742          -1.238971          -0.919598   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0          -0.030645          -0.834931          -1.031650          -0.840942   \n",
      "1           1.691433          -0.672783          -0.119944          -0.440080   \n",
      "2          -0.403611          -1.036954           1.447615          -0.340767   \n",
      "3           1.789786          -0.552163           0.121028          -0.111355   \n",
      "4           0.313068          -1.160111           1.316032          -0.700013   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.677716            1.084098           -1.064999   \n",
      "1           0.339906            1.084098            0.504608   \n",
      "2          -0.846170           -0.515065           -0.699878   \n",
      "3           0.220614            1.084098            0.073241   \n",
      "4          -1.600210            1.084098           -1.072155   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -1.156623  \n",
      "1            0.931676  \n",
      "2            0.032355  \n",
      "3            1.176257  \n",
      "4            1.270095  \n",
      "(834, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.005, 'max_iter': 500}, que permiten obtener un Accuracy de 64.96% y un Kappa del 23.85\n",
      "Model: \"model_104\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_104 (InputLayer)       (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_315 (Dense)            (None, 30)                390       \n",
      "_________________________________________________________________\n",
      "dense_316 (Dense)            (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_317 (Dense)            (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 1,351\n",
      "Trainable params: 1,351\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 500\n",
      "209/209 [==============================] - 0s 75us/step\n",
      "test loss: 0.6627522056752985, test accuracy: 0.6267942786216736\n",
      "AUC ROC:  0.6368855761482675\n",
      "Kappa:  0.1464921465968586\n",
      "[[103  33]\n",
      " [ 45  28]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Gramma y Hotel Marrakech y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           2.134448          -0.644769          -0.741425          -0.871269   \n",
      "1          -0.027862           1.690489           0.055411          -0.561004   \n",
      "2          -0.377201           2.125071          -0.304151           2.056775   \n",
      "3          -0.590231           0.143657          -0.601072           1.282246   \n",
      "4          -0.233782           0.378000           1.366350           0.941326   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           0.189298          -0.623423          -0.529336           0.357204   \n",
      "1           0.495091          -0.306711           0.886175          -0.561042   \n",
      "2          -0.998633          -0.428416           1.821160          -1.229826   \n",
      "3          -0.578812          -0.045810           1.094766           0.365693   \n",
      "4          -0.898589          -0.324641           1.117103           0.022453   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.325100            0.746582           -0.476439   \n",
      "1           1.431598            0.158038            0.235945   \n",
      "2          -0.732612           -1.686385           -0.297109   \n",
      "3           1.431598           -0.618841            0.417565   \n",
      "4           1.431598           -0.507270            0.477771   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.217416  \n",
      "1            1.435645  \n",
      "2            0.515440  \n",
      "3            0.135593  \n",
      "4           -0.297988  \n",
      "(795, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'logistic', 'hidden_layer_sizes': (10,), 'learning_rate_init': 0.003, 'max_iter': 10}, que permiten obtener un Accuracy de 63.26% y un Kappa del 2.21\n",
      "Model: \"model_105\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_105 (InputLayer)       (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_318 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_319 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 141\n",
      "Trainable params: 141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 10\n",
      "199/199 [==============================] - 0s 55us/step\n",
      "test loss: 0.6593509022913986, test accuracy: 0.6180904507637024\n",
      "AUC ROC:  0.5637641047477113\n",
      "Kappa:  0.0158771473191045\n",
      "[[122   0]\n",
      " [ 76   1]]\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import display, Markdown, Latex\n",
    "import itertools\n",
    "three_combs = list(itertools.combinations(range(len(companies)), 4))\n",
    "for comb in three_combs:\n",
    "    title = '## '\n",
    "    for no in comb[:-1]:\n",
    "        title+=companies[no]\n",
    "        title+=' y '\n",
    "    title+=companies[comb[-1]]\n",
    "    display(Markdown(title))\n",
    "    X = df_n_ps_std_ch[comb[0]]\n",
    "    y = df_n_ps[comb[0]]['chosen']\n",
    "    for no in comb[1:]:\n",
    "        X= X.append(df_n_ps_std_ch[no])\n",
    "        y= y.append(df_n_ps[no]['chosen'])\n",
    "    \n",
    "\n",
    "    print(X.head())\n",
    "    print(X.shape)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "    mlp = MLPClassifier(hidden_layer_sizes=(30,30,30))\n",
    "\n",
    "    activation_vec = ['logistic', 'relu', 'tanh']\n",
    "    max_iter_vec = [10, 20, 50, 75, 100, 200, 300, 400, 500, 1000, 2000]\n",
    "    hidden_layer_sizes_vec = [(10,), (20,), (30,), (10, 10), (20, 20), (30, 30), (20, 10), \n",
    "                              (10, 10, 10), (20, 20, 20), (30, 30, 30), (30, 20, 10)]\n",
    "    learning_rate_init_vec = [0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.02]\n",
    "    batch_size_vec = [10, 20, 40, 60, 80, 100, 150]\n",
    "\n",
    "\n",
    "    np.random.seed(1234)\n",
    "    parametros = {'activation': activation_vec,\n",
    "                  'max_iter':max_iter_vec,\n",
    "                  'hidden_layer_sizes': hidden_layer_sizes_vec,\n",
    "                  'learning_rate_init': learning_rate_init_vec#,\n",
    "                  #'batch_size': batch_size_vec\n",
    "                  }\n",
    "    scoring = {'kappa':make_scorer(cohen_kappa_score), 'accuracy':'accuracy'}\n",
    "    grid = GridSearchCV(mlp, param_grid=parametros, cv=5, scoring=scoring, refit='accuracy', n_jobs=-1, iid=True)\n",
    "\n",
    "    grid.fit(X_train, y_train)\n",
    "\n",
    "    print(\"Los parámetros del mejor modelo fueron {0}, que permiten obtener un Accuracy de {1:.2f}% y un Kappa del {2:.2f}\".format(\n",
    "        grid.best_params_, grid.best_score_*100, grid.cv_results_['mean_test_kappa'][grid.best_index_]*100))\n",
    "\n",
    "    n0=X_train.shape[1]\n",
    "    ### hidden_layer_sizes\n",
    "    ns = []\n",
    "    for i in range (len(grid.best_params_['hidden_layer_sizes'])):\n",
    "        ns.append(grid.best_params_['hidden_layer_sizes'][i])\n",
    "\n",
    "    ns.append(1)\n",
    "    lr = grid.best_params_['learning_rate_init']\n",
    "    epochs = grid.best_params_['max_iter']\n",
    "\n",
    "    input_tensor = Input(shape = (n0,))\n",
    "    hidden_outputs = [input_tensor]\n",
    "    if grid.best_params_['activation'] == 'logistic':\n",
    "        grid.best_params_['activation'] = 'sigmoid'\n",
    "    for i in range (len(ns)-1):\n",
    "        hidden_outputs.append(Dense(ns[i], activation = grid.best_params_['activation'])(hidden_outputs[i]))\n",
    "    classification_output = Dense(ns[-1], activation = 'sigmoid')(hidden_outputs[-1])\n",
    "\n",
    "    model = Model([input_tensor], [classification_output])\n",
    "    weights = model.get_weights()\n",
    "\n",
    "    print(model.summary())\n",
    "\n",
    "    model.set_weights(weights)\n",
    "    adam = keras.optimizers.Adam(lr=lr)\n",
    "    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    history = model.fit(X_train, y_train, epochs=epochs, validation_data=(X_test, y_test), verbose=0, \n",
    "                callbacks=[\n",
    "                    keras.callbacks.ReduceLROnPlateau(\n",
    "                        monitor='val_accuracy', factor=0.5, patience=10, min_delta=0.01, verbose=0\n",
    "                    )\n",
    "                ]\n",
    "             )\n",
    "\n",
    "    acc = history.history['accuracy']\n",
    "    val_acc = history.history['val_accuracy']\n",
    "    loss = history.history['loss']\n",
    "    val_loss = history.history['val_loss']\n",
    "\n",
    "    epochs = range(len(acc))\n",
    "\n",
    "    print(\"epochs: \"+str(len(acc)))\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(X_test, y_test)\n",
    "    print(\"test loss: {}, test accuracy: {}\".format(test_loss, test_acc))\n",
    "\n",
    "    y_pred = model.predict(X_test)\n",
    "    print(\"AUC ROC: \",roc_auc_score(y_test, y_pred))\n",
    "\n",
    "    y_pred = list(map(lambda i: int(i>=0.5), y_pred))\n",
    "    print(\"Kappa: \",cohen_kappa_score(y_test, y_pred))\n",
    "\n",
    "    print(confusion_matrix(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5-comp groups combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Gramma y Hotel Marrakech y Specialized"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(1149, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (10, 10, 10), 'learning_rate_init': 0.007, 'max_iter': 20}, que permiten obtener un Accuracy de 67.48% y un Kappa del 6.59\n",
      "Model: \"model_106\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_106 (InputLayer)       (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_320 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_321 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_322 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_323 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 361\n",
      "Trainable params: 361\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 20\n",
      "288/288 [==============================] - 0s 21us/step\n",
      "test loss: 0.7196702228652107, test accuracy: 0.5798611044883728\n",
      "AUC ROC:  0.4887234042553191\n",
      "Kappa:  -0.05805197959679376\n",
      "[[154  34]\n",
      " [ 87  13]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Gramma y Hotel Marrakech y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(1146, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (300) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (10,), 'learning_rate_init': 0.002, 'max_iter': 300}, que permiten obtener un Accuracy de 68.68% y un Kappa del 13.41\n",
      "Model: \"model_107\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_107 (InputLayer)       (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_324 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_325 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 141\n",
      "Trainable params: 141\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 300\n",
      "287/287 [==============================] - 0s 108us/step\n",
      "test loss: 0.6104007429777537, test accuracy: 0.7108014225959778\n",
      "AUC ROC:  0.4922705314009662\n",
      "Kappa:  -0.02056467160790021\n",
      "[[204   3]\n",
      " [ 80   0]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Gramma y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(1221, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (20, 20, 20), 'learning_rate_init': 0.004, 'max_iter': 10}, que permiten obtener un Accuracy de 66.78% y un Kappa del 1.70\n",
      "Model: \"model_108\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_108 (InputLayer)       (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_326 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_327 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_328 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_329 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 1,121\n",
      "Trainable params: 1,121\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 10\n",
      "306/306 [==============================] - 0s 20us/step\n",
      "test loss: 0.621280312538147, test accuracy: 0.6633986830711365\n",
      "AUC ROC:  0.5602397399431126\n",
      "Kappa:  -0.02617698769290877\n",
      "[[198  16]\n",
      " [ 87   5]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Club De Banqueros y Empresarios y Hotel Marrakech y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(1149, 12)\n",
      "Los parámetros del mejor modelo fueron {'activation': 'tanh', 'hidden_layer_sizes': (20, 20), 'learning_rate_init': 0.007, 'max_iter': 1000}, que permiten obtener un Accuracy de 66.78% y un Kappa del 26.40\n",
      "Model: \"model_109\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_109 (InputLayer)       (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_330 (Dense)            (None, 20)                260       \n",
      "_________________________________________________________________\n",
      "dense_331 (Dense)            (None, 20)                420       \n",
      "_________________________________________________________________\n",
      "dense_332 (Dense)            (None, 1)                 21        \n",
      "=================================================================\n",
      "Total params: 701\n",
      "Trainable params: 701\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 1000\n",
      "288/288 [==============================] - 0s 298us/step\n",
      "test loss: 0.7321343157026503, test accuracy: 0.6354166865348816\n",
      "AUC ROC:  0.6264130135097877\n",
      "Kappa:  0.15915915915915912\n",
      "[[144  51]\n",
      " [ 54  39]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Arte Francés y Gramma y Hotel Marrakech y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0           1.713160           0.038243           2.143431           0.466379   \n",
      "1           1.315247           0.387677           1.394548          -0.542923   \n",
      "2           1.737505          -1.440944          -1.114255           0.326790   \n",
      "3           1.737506          -1.127088          -0.672464           0.505231   \n",
      "4           1.737506           1.455410           0.333584           1.122628   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0           2.480688           1.533552          -0.564788          -0.236267   \n",
      "1          -0.194326           0.100084           0.191568           1.691930   \n",
      "2          -1.157732          -1.261521          -1.130503           1.585017   \n",
      "3          -0.830959          -1.046716          -0.912922           1.082754   \n",
      "4           1.139134          -0.481280           0.129896           0.915597   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.737672            0.470152           -0.935468   \n",
      "1          -0.164059            0.599296           -0.175780   \n",
      "2          -1.367743           -1.287569           -0.805298   \n",
      "3          -0.919242           -0.761758           -1.023346   \n",
      "4           0.333032           -0.289806           -0.236639   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0            0.074931  \n",
      "1            0.267042  \n",
      "2           -1.676436  \n",
      "3           -1.220716  \n",
      "4           -0.281891  \n",
      "(1110, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (10, 10, 10), 'learning_rate_init': 0.001, 'max_iter': 100}, que permiten obtener un Accuracy de 66.95% y un Kappa del 12.34\n",
      "Model: \"model_110\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_110 (InputLayer)       (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_333 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_334 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_335 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_336 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 361\n",
      "Trainable params: 361\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 100\n",
      "278/278 [==============================] - 0s 36us/step\n",
      "test loss: 0.6524442815952164, test accuracy: 0.6546762585639954\n",
      "AUC ROC:  0.5173992673992673\n",
      "Kappa:  0.031218237258603065\n",
      "[[177   5]\n",
      " [ 91   5]]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Club De Banqueros y Empresarios y Gramma y Hotel Marrakech y Specialized y Urban Place"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   chromagramfiles_1  chromagramfiles_2  chromagramfiles_3  chromagramfiles_4  \\\n",
      "0          -0.369691          -0.881824          -0.095656          -0.923999   \n",
      "1          -0.175875          -0.403800          -0.657709          -0.201259   \n",
      "2           0.894452          -0.189794           1.959063           0.169276   \n",
      "3           0.060782          -0.392075           0.826233          -0.048480   \n",
      "4          -1.116536          -0.923742          -1.238971          -0.919598   \n",
      "\n",
      "   chromagramfiles_5  chromagramfiles_6  chromagramfiles_7  chromagramfiles_8  \\\n",
      "0          -0.030645          -0.834931          -1.031650          -0.840942   \n",
      "1           1.691433          -0.672783          -0.119944          -0.440080   \n",
      "2          -0.403611          -1.036954           1.447615          -0.340767   \n",
      "3           1.789786          -0.552163           0.121028          -0.111355   \n",
      "4           0.313068          -1.160111           1.316032          -0.700013   \n",
      "\n",
      "   chromagramfiles_9  chromagramfiles_10  chromagramfiles_11  \\\n",
      "0          -0.677716            1.084098           -1.064999   \n",
      "1           0.339906            1.084098            0.504608   \n",
      "2          -0.846170           -0.515065           -0.699878   \n",
      "3           0.220614            1.084098            0.073241   \n",
      "4          -1.600210            1.084098           -1.072155   \n",
      "\n",
      "   chromagramfiles_12  \n",
      "0           -1.156623  \n",
      "1            0.931676  \n",
      "2            0.032355  \n",
      "3            1.176257  \n",
      "4            1.270095  \n",
      "(1050, 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (75) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Los parámetros del mejor modelo fueron {'activation': 'relu', 'hidden_layer_sizes': (10, 10), 'learning_rate_init': 0.003, 'max_iter': 75}, que permiten obtener un Accuracy de 66.71% y un Kappa del 5.60\n",
      "Model: \"model_111\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_111 (InputLayer)       (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_337 (Dense)            (None, 10)                130       \n",
      "_________________________________________________________________\n",
      "dense_338 (Dense)            (None, 10)                110       \n",
      "_________________________________________________________________\n",
      "dense_339 (Dense)            (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 251\n",
      "Trainable params: 251\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "epochs: 75\n",
      "263/263 [==============================] - 0s 59us/step\n",
      "test loss: 0.6787851892496697, test accuracy: 0.6159695982933044\n",
      "AUC ROC:  0.5606389391199518\n",
      "Kappa:  0.06675332888311125\n",
      "[[152   6]\n",
      " [ 95  10]]\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import display, Markdown, Latex\n",
    "import itertools\n",
    "three_combs = list(itertools.combinations(range(len(companies)), 5))\n",
    "for comb in three_combs:\n",
    "    title = '## '\n",
    "    for no in comb[:-1]:\n",
    "        title+=companies[no]\n",
    "        title+=' y '\n",
    "    title+=companies[comb[-1]]\n",
    "    display(Markdown(title))\n",
    "    X = df_n_ps_std_ch[comb[0]]\n",
    "    y = df_n_ps[comb[0]]['chosen']\n",
    "    for no in comb[1:]:\n",
    "        X= X.append(df_n_ps_std_ch[no])\n",
    "        y= y.append(df_n_ps[no]['chosen'])\n",
    "    \n",
    "\n",
    "    print(X.head())\n",
    "    print(X.shape)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "    mlp = MLPClassifier(hidden_layer_sizes=(30,30,30))\n",
    "\n",
    "    activation_vec = ['logistic', 'relu', 'tanh']\n",
    "    max_iter_vec = [10, 20, 50, 75, 100, 200, 300, 400, 500, 1000, 2000]\n",
    "    hidden_layer_sizes_vec = [(10,), (20,), (30,), (10, 10), (20, 20), (30, 30), (20, 10), \n",
    "                              (10, 10, 10), (20, 20, 20), (30, 30, 30), (30, 20, 10)]\n",
    "    learning_rate_init_vec = [0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, 0.01, 0.02]\n",
    "    batch_size_vec = [10, 20, 40, 60, 80, 100, 150]\n",
    "\n",
    "\n",
    "    np.random.seed(1234)\n",
    "    parametros = {'activation': activation_vec,\n",
    "                  'max_iter':max_iter_vec,\n",
    "                  'hidden_layer_sizes': hidden_layer_sizes_vec,\n",
    "                  'learning_rate_init': learning_rate_init_vec#,\n",
    "                  #'batch_size': batch_size_vec\n",
    "                  }\n",
    "    scoring = {'kappa':make_scorer(cohen_kappa_score), 'accuracy':'accuracy'}\n",
    "    grid = GridSearchCV(mlp, param_grid=parametros, cv=5, scoring=scoring, refit='accuracy', n_jobs=-1, iid=True)\n",
    "\n",
    "    grid.fit(X_train, y_train)\n",
    "\n",
    "    print(\"Los parámetros del mejor modelo fueron {0}, que permiten obtener un Accuracy de {1:.2f}% y un Kappa del {2:.2f}\".format(\n",
    "        grid.best_params_, grid.best_score_*100, grid.cv_results_['mean_test_kappa'][grid.best_index_]*100))\n",
    "\n",
    "    n0=X_train.shape[1]\n",
    "    ### hidden_layer_sizes\n",
    "    ns = []\n",
    "    for i in range (len(grid.best_params_['hidden_layer_sizes'])):\n",
    "        ns.append(grid.best_params_['hidden_layer_sizes'][i])\n",
    "\n",
    "    ns.append(1)\n",
    "    lr = grid.best_params_['learning_rate_init']\n",
    "    epochs = grid.best_params_['max_iter']\n",
    "\n",
    "    input_tensor = Input(shape = (n0,))\n",
    "    hidden_outputs = [input_tensor]\n",
    "    if grid.best_params_['activation'] == 'logistic':\n",
    "        grid.best_params_['activation'] = 'sigmoid'\n",
    "    for i in range (len(ns)-1):\n",
    "        hidden_outputs.append(Dense(ns[i], activation = grid.best_params_['activation'])(hidden_outputs[i]))\n",
    "    classification_output = Dense(ns[-1], activation = 'sigmoid')(hidden_outputs[-1])\n",
    "\n",
    "    model = Model([input_tensor], [classification_output])\n",
    "    weights = model.get_weights()\n",
    "\n",
    "    print(model.summary())\n",
    "\n",
    "    model.set_weights(weights)\n",
    "    adam = keras.optimizers.Adam(lr=lr)\n",
    "    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    history = model.fit(X_train, y_train, epochs=epochs, validation_data=(X_test, y_test), verbose=0, \n",
    "                callbacks=[\n",
    "                    keras.callbacks.ReduceLROnPlateau(\n",
    "                        monitor='val_accuracy', factor=0.5, patience=10, min_delta=0.01, verbose=0\n",
    "                    )\n",
    "                ]\n",
    "             )\n",
    "\n",
    "    acc = history.history['accuracy']\n",
    "    val_acc = history.history['val_accuracy']\n",
    "    loss = history.history['loss']\n",
    "    val_loss = history.history['val_loss']\n",
    "\n",
    "    epochs = range(len(acc))\n",
    "\n",
    "    print(\"epochs: \"+str(len(acc)))\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(X_test, y_test)\n",
    "    print(\"test loss: {}, test accuracy: {}\".format(test_loss, test_acc))\n",
    "\n",
    "    y_pred = model.predict(X_test)\n",
    "    print(\"AUC ROC: \",roc_auc_score(y_test, y_pred))\n",
    "\n",
    "    y_pred = list(map(lambda i: int(i>=0.5), y_pred))\n",
    "    print(\"Kappa: \",cohen_kappa_score(y_test, y_pred))\n",
    "\n",
    "    print(confusion_matrix(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}