Testador.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Código utilizado para experimentação e testes com diferentes métodos e parâmetros para reconhecer padrões de valência e contexto narrativo(assunto) nas letras de músicas</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import BaseDados\n",
    "import string\n",
    "import collections\n",
    "import numpy as np\n",
    "import math\n",
    "import json\n",
    "import datetime\n",
    "from __future__ import print_function\n",
    "import warnings\n",
    "from time import time\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.decomposition import NMF, LatentDirichletAllocation\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.metrics import f1_score,confusion_matrix\n",
    "from nltk.corpus import floresta,stopwords\n",
    "from nltk import DefaultTagger,UnigramTagger,BigramTagger,pos_tag,word_tokenize\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import StratifiedKFold,permutation_test_score\n",
    "import sklearn.base\n",
    "\n",
    "warnings.filterwarnings('ignore');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def StopWords(incluir = []):\n",
    "    stp = stopwords.words('portuguese');\n",
    "    [stp.append(e) for e in string.punctuation];\n",
    "    [stp.append(e) for e in incluir];\n",
    "    return stp\n",
    "\n",
    "#Retira somente a parte principal do part-of-speech (se é verbo, subs, adj, etc.)\n",
    "def simplify_tag(t):\n",
    "    if t is None: return '.'\n",
    "    if \"+\" in t:\n",
    "         return simplify_tag(t[t.index(\"+\")+1:])\n",
    "    if \"-\" in t:\n",
    "         return simplify_tag(t[:t.index(\"-\")])\n",
    "    else:\n",
    "        return t\n",
    "\n",
    "#Treina o NLTK para português\n",
    "train = floresta.tagged_sents()\n",
    "tagger1 = UnigramTagger(train)\n",
    "tagger2 = BigramTagger(train, backoff=tagger1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Converte em PoS e os vectoriza, para cada letra\n",
    "def LetrasToPartOfSpeech(Letras):\n",
    "    ret = []\n",
    "    for l in [l for l in Letras]:\n",
    "        tagged = tagger2.tag(word_tokenize(l));\n",
    "        tagged = [(w.lower(), simplify_tag(t)) for (w,t) in tagged];\n",
    "        tagged = [(w,t) for (w,t) in tagged if (t not in string.punctuation)];\n",
    "        tags = [t for (w,t) in tagged ];\n",
    "        _letra_tags_ = ' '.join(tags);\n",
    "        ret.append(_letra_tags_);\n",
    "    return ret\n",
    "\n",
    "\n",
    "def CountMatrix(lista_texto):\n",
    "    ret = []\n",
    "    all_pos = [e[0] for e in collections.Counter((' '.join(lista_texto).split(' '))).items()]\n",
    "    for l in lista_texto:\n",
    "        Cnt = collections.Counter(l.split(' '))\n",
    "        ret.append([Cnt[f]/len(list(Cnt.elements())) for f in all_pos])\n",
    "    return ret\n",
    "\n",
    "#Converte as letras em uma matriz representativa\n",
    "def RepresentarLetras(Letras,\n",
    "                      Binario,\n",
    "                      PoS,\n",
    "                      RemoveStopWords, \n",
    "                      n_features, \n",
    "                      n_components):\n",
    "    if RemoveStopWords:\n",
    "        stopwords = StopWords(['oh','la','lá','ah','alô','aí','nena']);\n",
    "    else:\n",
    "        stopwords = [];\n",
    "    \n",
    "    tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=2,\n",
    "                                       max_features=n_features,\n",
    "                                       stop_words=stopwords)\n",
    "    tfidf = tfidf_vectorizer.fit_transform(Letras)\n",
    "    \n",
    "    nmf = NMF(n_components=n_components, random_state=3,\n",
    "              alpha=.1, l1_ratio=.5).fit(tfidf)\n",
    "    W = nmf.fit_transform(tfidf)\n",
    "    \n",
    "    if Binario:\n",
    "        W = (W > 0).astype(float)\n",
    "\n",
    "    if PoS:\n",
    "        partofspeech = LetrasToPartOfSpeech(Letras)\n",
    "        partofspeech_columns = CountMatrix(partofspeech);\n",
    "        W = np.c_[W,partofspeech_columns]\n",
    "    \n",
    "    return W\n",
    "\n",
    "#Testa o modelo de ML, registra os resultados (inclusive com os grupos separados G5 G4 e G3)\n",
    "def TestarParametros(Dados,\n",
    "                     idx_g5,\n",
    "                     idx_g4,\n",
    "                     idx_g3,\n",
    "                     Teste,\n",
    "                     Target, \n",
    "                     method,\n",
    "                     hyperP,\n",
    "                     kfold,\n",
    "                     Binario,\n",
    "                     PoS,\n",
    "                     RemoveStopWords, \n",
    "                     n_features, \n",
    "                     n_components):\n",
    "    if method=='SVC':\n",
    "        s_main = SVC(C=hyperP);\n",
    "    elif method=='KNN':\n",
    "        s_main = KNeighborsClassifier(n_neighbors=hyperP);\n",
    "    elif method=='RD':\n",
    "        s_main = RandomForestClassifier(n_estimators=hyperP*10, max_depth=2,\n",
    "                                  random_state=2);\n",
    "    elif method=='GNB':\n",
    "        s_main =  GaussianNB();\n",
    "    \n",
    "    Letras = [l.Letra for l in Dados];\n",
    "    \n",
    "    W = RepresentarLetras(Letras=Letras,Binario=Binario,PoS=PoS,RemoveStopWords=RemoveStopWords,n_features=n_features,n_components=n_components);\n",
    "    \n",
    "    #Ground-Truth separados por grupos de controle\n",
    "    if Target=='Assunto':\n",
    "        vetor_spec = [l.Assunto for l in Dados]\n",
    "    elif Target=='Valência':\n",
    "        vetor_spec = [l.Valencia for l in Dados]\n",
    "    \n",
    "    f1scores = np.array([]); \n",
    "    f1scores_g5 = np.array([]); #F1 scores obtidos pelo grupo g5\n",
    "    f1scores_g4 = np.array([]); #F1 scores obtidos pelo grupo g4\n",
    "    f1scores_g3 = np.array([]); #F1 scores obtidos pelo grupo g3\n",
    "    \n",
    "    W = np.array(W);\n",
    "    vetor_spec = np.array(vetor_spec);\n",
    "    cfn_matrices = np.zeros((len(set(vetor_spec)),len(set(vetor_spec))));\n",
    "    cfn_matrices_g5 = np.zeros((len(set(vetor_spec)),len(set(vetor_spec))));\n",
    "    cfn_matrices_g4 = np.zeros((len(set(vetor_spec)),len(set(vetor_spec))));\n",
    "    cfn_matrices_g3 = np.zeros((len(set(vetor_spec)),len(set(vetor_spec))));\n",
    "    \n",
    "    #Verificação por grupo#\n",
    "    \n",
    "    ######################################################################  G5\n",
    "    scores_g5 = np.array([]);\n",
    "    kfold_g5 = math.trunc(len(idx_g5)/10);\n",
    "    skf = StratifiedKFold(kfold_g5);\n",
    "    for invl, test_index in skf.split(np.zeros(len(idx_g5)), vetor_spec[idx_g5]):\n",
    "        s = sklearn.base.clone(s_main);\n",
    "        W_local = W;\n",
    "        idx_g5_test = idx_g5[test_index];\n",
    "        W_except_g5 = [x for i,x in enumerate(W_local) if i not in idx_g5_test];\n",
    "        vetor_spec_except_g5 = [x for i,x in enumerate(vetor_spec) if i not in idx_g5_test];\n",
    "        \n",
    "        #Normalização\n",
    "        if (not Binario) or (PoS):\n",
    "            norm = StandardScaler();\n",
    "            norm.fit(W_except_g5);\n",
    "            W_local = norm.transform(W_local);\n",
    "        \n",
    "        #Treino Algoritmo\n",
    "        s.fit(W_except_g5,vetor_spec_except_g5)\n",
    "        #Predição\n",
    "        y_true = vetor_spec[idx_g5_test];\n",
    "        y_pred = s.predict(W_local[idx_g5_test]);\n",
    "        #Cálculo de F1 e Confusion Matrix\n",
    "        f1_local = f1_score(y_true, y_pred, average='macro');\n",
    "        f1scores_g5 =np.append(f1scores_g5,f1_local);      \n",
    "        scores_g5 =np.append(scores_g5,s.score(W_local[idx_g5_test],vetor_spec[idx_g5_test]));    \n",
    "        cfn_matrix = confusion_matrix(y_true, y_pred);\n",
    "        cfn_matrices_g5 += cfn_matrix;\n",
    "    \n",
    "    score_g5 = scores_g5.mean();\n",
    "    ######################################################################  G4\n",
    "    scores_g4 = np.array([]);\n",
    "    kfold_g4 = math.trunc(len(idx_g4)/10);\n",
    "    skf = StratifiedKFold(kfold_g4);\n",
    "    for invl, test_index in skf.split(np.zeros(len(idx_g4)), vetor_spec[idx_g4]):\n",
    "        s = sklearn.base.clone(s_main);\n",
    "        W_local = W;\n",
    "        idx_g4_test = idx_g4[test_index];\n",
    "        W_except_g4 = [x for i,x in enumerate(W_local) if i not in idx_g4_test];\n",
    "        vetor_spec_except_g4 = [x for i,x in enumerate(vetor_spec) if i not in idx_g4_test];\n",
    "        \n",
    "        #Normalização\n",
    "        if (not Binario) or (PoS):\n",
    "            norm = StandardScaler();\n",
    "            norm.fit(W_except_g4);\n",
    "            W_local = norm.transform(W_local);\n",
    "        \n",
    "        #Treino Algoritmo\n",
    "        s.fit(W_except_g4,vetor_spec_except_g4);\n",
    "        #Predição\n",
    "        y_true = vetor_spec[idx_g4_test];\n",
    "        y_pred = s.predict(W_local[idx_g4_test]);\n",
    "        #Cálculo de F1 e Confusion Matrix\n",
    "        f1_local = f1_score(y_true, y_pred, average='macro');\n",
    "        f1scores_g4 =np.append(f1scores_g4, f1_local);\n",
    "        scores_g4 =np.append(scores_g4,s.score(W_local[idx_g4_test],vetor_spec[idx_g4_test]));            \n",
    "        cfn_matrix = confusion_matrix(y_true, y_pred);\n",
    "        cfn_matrices_g4 += cfn_matrix;\n",
    "        \n",
    "    score_g4 = scores_g4.mean();\n",
    "    ######################################################################  G3\n",
    "    scores_g3 = np.array([]);\n",
    "    kfold_g3 = math.trunc(len(idx_g3)/10);\n",
    "    skf = StratifiedKFold(kfold_g3);\n",
    "    for invl, test_index in skf.split(np.zeros(len(idx_g3)), vetor_spec[idx_g3]):\n",
    "        s = sklearn.base.clone(s_main);\n",
    "        W_local = W;\n",
    "        idx_g3_test = idx_g3[test_index];\n",
    "        W_except_g3 = [x for i,x in enumerate(W_local) if i not in idx_g3_test];\n",
    "        vetor_spec_except_g3 = [x for i,x in enumerate(vetor_spec) if i not in idx_g3_test];\n",
    "        \n",
    "        #Normalização\n",
    "        if (not Binario) or (PoS):\n",
    "            norm = StandardScaler();\n",
    "            norm.fit(W_except_g3);\n",
    "            W_local = norm.transform(W_local);\n",
    "        \n",
    "        #Treino Algoritmo\n",
    "        s.fit(W_except_g3,vetor_spec_except_g3)\n",
    "        #Predição\n",
    "        y_true = vetor_spec[idx_g3_test];\n",
    "        y_pred = s.predict(W_local[idx_g3_test]);\n",
    "        #Cálculo de F1 e Confusion Matrix\n",
    "        f1_local = f1_score(y_true, y_pred, average='macro');\n",
    "        f1scores_g3 =np.append(f1scores_g3,f1_local);\n",
    "        scores_g3 =np.append(scores_g3,s.score(W_local[idx_g3_test],vetor_spec[idx_g3_test]));           \n",
    "        cfn_matrix = confusion_matrix(y_true, y_pred);\n",
    "        cfn_matrices_g3 += cfn_matrix;\n",
    "    \n",
    "        \n",
    "    score_g3 = scores_g3.mean();\n",
    "    ######################################################################  Principal\n",
    "    scores = np.array([]);\n",
    "    \n",
    "    skf = StratifiedKFold(kfold)\n",
    "    for train_index, test_index in skf.split(W, vetor_spec):\n",
    "        s = sklearn.base.clone(s_main)\n",
    "        \n",
    "        #Normalização\n",
    "        if (not Binario) or (PoS):\n",
    "            norm = StandardScaler();\n",
    "            norm.fit(W);\n",
    "            W = norm.transform(W);\n",
    "        \n",
    "        #Treino Algoritmo\n",
    "        s.fit(W[train_index],vetor_spec[train_index])\n",
    "        \n",
    "        #Predição\n",
    "        y_true = vetor_spec[test_index];\n",
    "        y_pred = s.predict(W[test_index]);\n",
    "        \n",
    "        #Cálculo de F1 e Confusion Matrix\n",
    "        f1_local = f1_score(y_true, y_pred, average='macro');\n",
    "        \n",
    "        f1scores = np.append(f1scores,f1_local);    \n",
    "        scores =np.append(scores,s.score(W[test_index],vetor_spec[test_index]));    \n",
    "        cfn_matrix = confusion_matrix(y_true, y_pred);\n",
    "        cfn_matrices += cfn_matrix;\n",
    "    \n",
    "    \n",
    "    score = scores.mean();\n",
    "    f1score = f1scores.mean();\n",
    "    f1score_g5 = f1scores_g5.mean();\n",
    "    f1score_g4 = f1scores_g4.mean();\n",
    "    f1score_g3 = f1scores_g3.mean();\n",
    "    f1scores = ';'.join([json.dumps(f1scores.tolist()),\n",
    "                    json.dumps(f1scores_g5.tolist()),\n",
    "                    json.dumps(f1scores_g4.tolist()),\n",
    "                    json.dumps(f1scores_g3.tolist())]);\n",
    "    \n",
    "    BaseDados.InserirTeste(Teste=Teste,\n",
    "                           Target=Target,\n",
    "                           method=method,\n",
    "                           hyperP=hyperP,\n",
    "                           Binario=Binario,\n",
    "                           PoS=PoS,\n",
    "                           RemoveStopWords=RemoveStopWords, \n",
    "                           features=n_features, \n",
    "                           components=n_components,\n",
    "                           f1score=f1score,\n",
    "                           f1score_g5=f1score_g5,\n",
    "                           f1score_g4=f1score_g4,\n",
    "                           f1score_g3=f1score_g3,\n",
    "                           f1scores=f1scores,\n",
    "                           score=score,\n",
    "                           score_g5=score_g5,\n",
    "                           score_g4=score_g4,\n",
    "                           score_g3=score_g3,\n",
    "                           cfn_matrix=json.dumps(cfn_matrices.tolist()));\n",
    "    \n",
    "    return [f1score,f1score_g5,f1score_g4,f1score_g3,cfn_matrices,cfn_matrices_g5,cfn_matrices_g4,cfn_matrices_g3]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Realiza a varredura de diversos parâmetros e modelos\n",
    "def VarrerParametrosTestes(Dados,\n",
    "                             idx_g5,\n",
    "                             idx_g4,\n",
    "                             idx_g3,\n",
    "                             Teste,\n",
    "                             Target, \n",
    "                             kfold):\n",
    "    methods = ['SVC','RD','KNN','GNB'];\n",
    "\n",
    "    trials = [];\n",
    "    for method in [m for m in methods if m != 'GNB']:\n",
    "        for binario in [0,1]:\n",
    "            for pos in [0,1]:\n",
    "                for sw in [0,1]:\n",
    "                    for hyperP in [(e*2)-1 for e in range(1,10)]:\n",
    "                        for feat in [e*4 for e in range(12,32)]:\n",
    "                            for comp in [e*3 for e in range(3,10)]:\n",
    "                                trials.append([method,hyperP,binario,pos,sw,feat,comp])\n",
    "\n",
    "    for method in [m for m in methods if m == 'GNB']:\n",
    "        for binario in [0,1]:\n",
    "            for pos in [0,1]:\n",
    "                for sw in [0,1]:\n",
    "                    for feat in [e*4 for e in range(12,32)]:\n",
    "                        for comp in [e*3 for e in range(3,10)]:\n",
    "                            trials.append([method,0,binario,pos,sw,feat,comp])\n",
    "    \n",
    "    qtde = len(trials)\n",
    "    ultima_pct = 0;\n",
    "    for i in range(0,qtde):\n",
    "        TestarParametros(Dados,\n",
    "                         np.array(idx_g5),\n",
    "                         np.array(idx_g4),\n",
    "                         np.array(idx_g3),\n",
    "                         Teste,\n",
    "                         Target, \n",
    "                         trials[i][0],\n",
    "                         trials[i][1],\n",
    "                         kfold,\n",
    "                         trials[i][2],\n",
    "                         trials[i][3],\n",
    "                         trials[i][4], \n",
    "                         trials[i][5], \n",
    "                         trials[i][6]);\n",
    "        if (math.floor(10*i/qtde) > ultima_pct):\n",
    "            ultima_pct = math.floor(10*i/qtde)\n",
    "            print(datetime.datetime.now(),' - ',i,'/',qtde,' - ',ultima_pct*10,'%')\n",
    "    \n",
    "    print('- FIM -');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Código que executa as varreduras de parâmetros e modelos para diferentes grupos de letras\n",
    "\n",
    "#Cod_Letra,Letra,Total,Valencia,Valencia_QTD_MC,Valencia_QTD_MC_2,Assunto,Assunto_QTD_MC,Assunto_QTD_MC_2\n",
    "\n",
    "Teste = 'Todos'\n",
    "print(Teste)\n",
    "dados = [l for l in BaseDados.GetLetraClassificacoes_MostCommon() if l.Assunto_QTD_MC != l.Assunto_QTD_MC_2]\n",
    "idx_g5 = [i for i, d in enumerate(dados) if d.Assunto_QTD_MC >= 5]; #Indice das letras G5\n",
    "idx_g4 = [i for i, d in enumerate(dados) if d.Assunto_QTD_MC == 4]; #Indice das letras G4\n",
    "idx_g3 = [i for i, d in enumerate(dados) if d.Assunto_QTD_MC == 3]; #Indice das letras G3\n",
    "kfold = 12;\n",
    "VarrerParametrosTestes(Dados=dados,\n",
    "                             idx_g5=idx_g5,\n",
    "                             idx_g4=idx_g4,\n",
    "                             idx_g3=idx_g3,\n",
    "                            Teste=Teste,\n",
    "                            Target='Assunto', \n",
    "                            kfold=kfold);\n",
    "\n",
    "Teste = 'Todos'\n",
    "print(Teste)\n",
    "dados = [l for l in BaseDados.GetLetraClassificacoes_MostCommon() if l.Valencia_QTD_MC != l.Valencia_QTD_MC_2]\n",
    "idx_g5 = [i for i, d in enumerate(dados) if d.Valencia_QTD_MC >= 5];\n",
    "idx_g4 = [i for i, d in enumerate(dados) if d.Valencia_QTD_MC == 4];\n",
    "idx_g3 = [i for i, d in enumerate(dados) if d.Valencia_QTD_MC == 3];\n",
    "kfold = 12;\n",
    "VarrerParametrosTestes(Dados=dados,\n",
    "                             idx_g5=idx_g5,\n",
    "                             idx_g4=idx_g4,\n",
    "                             idx_g3=idx_g3,\n",
    "                            Teste=Teste,\n",
    "                            Target='Valência', \n",
    "                            kfold=kfold);\n",
    "\n",
    "#Teste = 'Relacionamentos e Reflexões'\n",
    "#print(Teste)\n",
    "#dados = [l for l in BaseDados.GetLetraClassificacoes_MostCommon() if l.Assunto_QTD_MC != l.Assunto_QTD_MC_2 and l.Assunto in [1,2]]\n",
    "#idx_g5 = [i for i, d in enumerate(dados) if d.Assunto_QTD_MC >= 5];\n",
    "#idx_g4 = [i for i, d in enumerate(dados) if d.Assunto_QTD_MC == 4];\n",
    "#idx_g3 = [i for i, d in enumerate(dados) if d.Assunto_QTD_MC == 3];\n",
    "#kfold = 12;\n",
    "#VarrerParametrosTestes(Dados=dados,\n",
    "#                             idx_g5=idx_g5,\n",
    "#                             idx_g4=idx_g4,\n",
    "#                             idx_g3=idx_g3,\n",
    "#                            Teste=Teste,\n",
    "#                            Target='Assunto', \n",
    "#                            kfold=kfold);\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cria a base que registrara os resultados ML\n",
    "BaseDados.CreateBaseTestes()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}