Normal_active_learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 1 Build on word-level text to generate a fixed-length vector for each sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "np.random.seed(2019)\n",
    "\n",
    "from numpy import genfromtxt\n",
    "\n",
    "import random as r\n",
    "r.seed(2019)\n",
    "\n",
    "import sys\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import f1_score,accuracy_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "torch.manual_seed(2019)\n",
    "torch.cuda.manual_seed_all(2019)\n",
    "\n",
    "import os\n",
    "os.environ['PYTHONHASHSEED'] = str(2019)\n",
    "\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from util import d, here\n",
    "\n",
    "import pandas as pd\n",
    "from argparse import ArgumentParser\n",
    "\n",
    "import random, tqdm, sys, math, gzip\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "\n",
    "torch.backends.cudnn.benchmark = False\n",
    "\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "import gc\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import ShuffleSplit,StratifiedKFold\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "# import fine_tuned\n",
    "import inference\n",
    "import time\n",
    "\n",
    "import shutil"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 1 Built-in Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "## show size of variables\n",
    "def sizeof_fmt(num, suffix='B'):\n",
    "    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:\n",
    "        if abs(num) < 1024.0:\n",
    "            return \"%3.1f%s%s\" % (num, unit, suffix)\n",
    "        num /= 1024.0\n",
    "    return \"%.1f%s%s\" % (num, 'Yi', suffix)\n",
    "\n",
    "## random over sampling instances belong to minority class in case of imbalanced dataset\n",
    "def resample(X,Y,random_state=2019):\n",
    "    X_reshape = np.reshape(X, (X.shape[0], X.shape[1]*X.shape[2]))\n",
    "    rus = RandomOverSampler(random_state=random_state)\n",
    "    X_res, y_res = rus.fit_resample(X_reshape, Y)\n",
    "    X_back = np.reshape(X_res, (X_res.shape[0],X.shape[1],X.shape[2]))\n",
    "    \n",
    "    return X_back, y_res\n",
    "\n",
    "## seed the initial data\n",
    "def initial_seed_dataset(n_initial, Y,random_state):\n",
    "    \n",
    "    np.random.seed(random_state)\n",
    "    \n",
    "    df = pd.DataFrame()\n",
    "    df['label'] = Y\n",
    "\n",
    "    Samplesize = n_initial  #number of samples that you want       \n",
    "    initial_samples = df.groupby('label', as_index=False).apply(lambda array: array.loc[np.random.choice(array.index, Samplesize, False),:])\n",
    "\n",
    "    permutation = [index[1] for index in initial_samples.index.tolist()]\n",
    "    \n",
    "    print ('initial random chosen samples', permutation)\n",
    "    \n",
    "    return permutation\n",
    "\n",
    "## compute the entropy\n",
    "def compute_entropy(output):\n",
    "    entropy = -(output[:,0]*np.log2(output[:,0]) + output[:,1]*np.log2(output[:,1]))\n",
    "    return entropy\n",
    "\n",
    "\n",
    "## sample the candidates for labelling\n",
    "def sample_candidates(selection_method,permutation,y_prob,num_candidate,y_pred,dist):\n",
    "    \n",
    "    print('*'*20,'selection method ',selection_method,'*'*20)\n",
    "    \n",
    "        \n",
    "        \n",
    "    if selection_method == 'uncertainty':\n",
    "       \n",
    "        candidate_index = []\n",
    "        \n",
    "        for index in dist.argsort():\n",
    "            if len(candidate_index) == num_candidate:\n",
    "                break\n",
    "            if index not in permutation:\n",
    "                candidate_index.append(index)\n",
    "\n",
    "        permutation = permutation+candidate_index\n",
    "        \n",
    "    if selection_method == 'certainty':\n",
    "        entropy = compute_entropy(np.array(y_prob))\n",
    "        candidate_index = []\n",
    "        \n",
    "        \n",
    "        for index in entropy.argsort()[:]:\n",
    "            if len(candidate_index) == num_candidate:\n",
    "                break\n",
    "            if index not in permutation:\n",
    "                candidate_index.append(index)\n",
    "\n",
    "        permutation = permutation+candidate_index\n",
    "        \n",
    "    if selection_method == 'mostConfident':\n",
    "        entropy = compute_entropy(np.array(y_prob))\n",
    "        candidate_index = []\n",
    "        \n",
    "        \n",
    "        for index in np.argsort(np.array(y_prob)[:,1])[::-1]:\n",
    "            if len(candidate_index) == num_candidate:\n",
    "                break\n",
    "            if index not in permutation:\n",
    "                candidate_index.append(index)\n",
    "\n",
    "        permutation = permutation+candidate_index\n",
    "        \n",
    "    print('*'*20,'num of training set ',len(permutation),'*'*20)   \n",
    "        \n",
    "    return permutation\n",
    "\n",
    "## normalization instance\n",
    "class Normalize(object):\n",
    "    \n",
    "    def normalize(self, X_train, X_val):\n",
    "        self.scaler = MinMaxScaler()\n",
    "        X_train = self.scaler.fit_transform(X_train)\n",
    "        X_val   = self.scaler.transform(X_val)\n",
    "       \n",
    "        return (X_train, X_val) \n",
    "    \n",
    "    def inverse(self, X_train, X_val):\n",
    "        X_train = self.scaler.inverse_transform(X_train)\n",
    "        X_val   = self.scaler.inverse_transform(X_val)\n",
    "    \n",
    "        return (X_train, X_val) \n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "## read data from directory\n",
    "def read_data(path):\n",
    "\n",
    "    corpora = []\n",
    "    for filename in os.listdir(path):\n",
    "\n",
    "        df_temp = pd.read_csv(path+filename)\n",
    "\n",
    "        corpora.append(df_temp.text.tolist())\n",
    "\n",
    "    class_one_len = len(corpora[0])\n",
    "    class_two_len = len(corpora[1])\n",
    "\n",
    "    return corpora, class_one_len, class_two_len\n",
    "\n",
    "## construct training text dataset for fine tuning language model, save in directory data/\n",
    "def construct_train_set(dataset,permutation):\n",
    "    print('constructing new text training set.....')\n",
    "    corpora, class_one_len, class_two_len = read_data('./corpus_data/'+dataset+'/')\n",
    "\n",
    "    texts = np.array(corpora[0]+corpora[1])\n",
    "    labels = np.array([0]*class_one_len+[1]*class_two_len)\n",
    "\n",
    "#     X_train, X_test, y_train, y_test = train_test_split(texts[permutation], labels[permutation], test_size=0.0, random_state=2019)\n",
    "    X_train, y_train = texts[permutation], labels[permutation]\n",
    "    \n",
    "    train_df = pd.DataFrame({'id':range(len(X_train)),'label':y_train,'alpha':['a']*len(X_train),'text':X_train})\n",
    "#     dev_df = pd.DataFrame({'id':range(len(X_test)),'label':y_test,'alpha':['a']*len(X_test),'text':X_test})\n",
    "    \n",
    "    train_df.to_csv('data/train.tsv', sep='\\t', index=False, header=False)\n",
    "#     dev_df.to_csv('data/dev.tsv', sep='\\t', index=False, header=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 2. Import Inferred embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "## choose the bert-like models, options are roberta-base, disilbert-base-uncased, gpt2, xlnet-base-cased, albert-base-v2\n",
    "text_rep = 'roberta-base'\n",
    "\n",
    "\n",
    "## input dataset \n",
    "dataset = 'ProtonPumpInhibitors'\n",
    "dir_neg = dataset + '_neg.csv'\n",
    "dir_pos = dataset + '_pos.csv'\n",
    "\n",
    "representations_neg = genfromtxt('./%s_data/'%(text_rep)+dir_neg, delimiter=',')\n",
    "representations_pos = genfromtxt('./%s_data/'%(text_rep)+dir_pos, delimiter=',')\n",
    "\n",
    "ulti_representations = np.concatenate((representations_neg,representations_pos),axis=0)\n",
    "labels = np.array([0]*len(representations_neg)+[1]*len(representations_pos))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 3. Active Learning Process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_CLS = 2\n",
    "\n",
    "def active_process(arg):\n",
    "    \"\"\"\n",
    "    Active learning procedure for normal active learning.\n",
    "    \"\"\"\n",
    "    \n",
    "    ## gridsearch for hyper-parameters tuning for SVM\n",
    "    if arg.loop%arg.gridsearch_interval==0:\n",
    "        gridsearch = True\n",
    "    else:\n",
    "        gridsearch = False\n",
    "    \n",
    "    if arg.initial: \n",
    "        ## the first round, initialize seed data\n",
    "        permutation = initial_seed_dataset(arg.n_seedsamples,arg.LABEL_emb,arg.initial_random_seed)\n",
    "       \n",
    "        X_train = arg.TEXT_emb[permutation]\n",
    "        Y_train = arg.LABEL_emb[permutation]\n",
    "\n",
    "        X_val = arg.TEXT_emb\n",
    "        Y_val = arg.LABEL_emb\n",
    "        \n",
    "        bin_count = np.bincount(Y_train)\n",
    "        unique = np.unique(Y_train)\n",
    "        print (\n",
    "        'initial training set size:',\n",
    "        Y_train.shape[0],\n",
    "        'unique(labels):',\n",
    "        unique,\n",
    "        'label counts:',\n",
    "        bin_count\n",
    "        )\n",
    "\n",
    "        print('Number of training examples ', len(Y_train))\n",
    "        \n",
    "    else:\n",
    "        \n",
    "        permutation = arg.permutation\n",
    "       \n",
    "        \n",
    "        X_train = arg.TEXT_emb[permutation]\n",
    "        Y_train = arg.LABEL_emb[permutation]\n",
    "\n",
    "        X_val = arg.TEXT_emb\n",
    "        Y_val = arg.LABEL_emb\n",
    "        \n",
    "        bin_count = np.bincount(Y_train)\n",
    "        unique = np.unique(Y_train)\n",
    "        print (\n",
    "        'training set size:',\n",
    "        Y_train.shape[0],\n",
    "        'unique(labels):',\n",
    "        unique,\n",
    "        'label counts:',\n",
    "        bin_count\n",
    "        )\n",
    "\n",
    "        print('Number of training examples ', len(Y_train))\n",
    "\n",
    "\n",
    "    evaluation= { 'Confusion matrix':[]}\n",
    "    \n",
    "    normalizer = Normalize()\n",
    "    X_train, X_val = normalizer.normalize(X_train, X_val) \n",
    "        \n",
    "    \n",
    "    if gridsearch == True:\n",
    "       ## gridsearch for finding best hyper-parameters\n",
    "        print('start gridsearch ...')\n",
    "        parameters = [\n",
    "                    {'kernel': ['linear'],\n",
    "                     'C': [ 0.01, 0.1, 1,10]}]\n",
    "\n",
    "        cv = StratifiedKFold(n_splits=5,random_state=arg.initial_random_seed)\n",
    "        svc = SVC(probability=True,random_state=2019,class_weight='balanced',max_iter=10000)\n",
    "        classifier = GridSearchCV(svc, parameters, cv=cv,scoring='accuracy',n_jobs=8,verbose = 0)\n",
    "        classifier.fit(X_train, Y_train)\n",
    "        print('best parameters is ', classifier.best_params_)\n",
    "        best_params_ = classifier.best_params_\n",
    "\n",
    "        \n",
    "    else:\n",
    "        ## hyper-parameters from last loop   \n",
    "        best_params_ = arg.best_params_\n",
    "        kernel = best_params_['kernel']\n",
    "        C = best_params_['C']\n",
    "        \n",
    "        classifier = SVC(probability=True,random_state=2019,class_weight='balanced',C=C,kernel=kernel,max_iter=10000)\n",
    "        classifier.fit(X_train, Y_train)\n",
    "        print('best parameters is ', classifier)\n",
    "        \n",
    "        \n",
    "    ## evaluation\n",
    "        \n",
    "    y_pred_prob = classifier.predict_proba(X_val)\n",
    "#     y_pred = np.argmax(y_pred_prob, axis=1)\n",
    "    y_pred = classifier.predict(X_val)\n",
    "    y_true = Y_val\n",
    "    \n",
    "    numerators = classifier.decision_function(X_val)\n",
    "    try:\n",
    "        w_norm = np.linalg.norm(classifier.best_estimator_.coef_)\n",
    "    except Exception:\n",
    "        w_norm = np.linalg.norm(classifier.coef_)\n",
    "\n",
    "    dist = abs(numerators) / w_norm\n",
    "   \n",
    "\n",
    "    y_true_remain = np.delete(np.array(y_true),permutation)\n",
    "    y_pred_remain = np.delete(np.array(y_pred),permutation)\n",
    "        \n",
    "    print(classification_report(y_true_remain,y_pred_remain,labels=[0,1]))\n",
    "   \n",
    "    tn, fp, fn, tp = confusion_matrix(y_true_remain, y_pred_remain, labels=[0,1]).ravel()\n",
    "    print('TP_H',bin_count[1],' TN_H',bin_count[0], ' TP_M',tp, ' TN_M',tn, ' FP_M', fp, ' FN_M',fn)\n",
    "    acc = accuracy_score(y_true_remain, y_pred_remain)\n",
    "    f_score = f1_score(y_true_remain,y_pred_remain,average='macro')\n",
    "                \n",
    "    evaluation['Confusion matrix'].append({'TP_H':bin_count[1],'TN_H':bin_count[0], 'TP_M':tp, 'TN_M':tn, 'FP_M': fp, 'FN_M':fn})\n",
    "\n",
    "    \n",
    "    return evaluation, y_pred_prob, y_pred,permutation, best_params_,dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing  0 th loops---------------\n",
      "initial random chosen samples [476, 707, 770, 587, 468, 1026, 1891, 1549, 1101, 1390]\n",
      "initial training set size: 10 unique(labels): [0 1] label counts: [5 5]\n",
      "Number of training examples  10\n",
      "start gridsearch ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:296: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n",
      "  FutureWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  {'C': 0.01, 'kernel': 'linear'}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.70      0.83      0.76       995\n",
      "           1       0.79      0.65      0.72       995\n",
      "\n",
      "    accuracy                           0.74      1990\n",
      "   macro avg       0.75      0.74      0.74      1990\n",
      "weighted avg       0.75      0.74      0.74      1990\n",
      "\n",
      "TP_H 5  TN_H 5  TP_M 650  TN_M 824  FP_M 171  FN_M 345\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  20 ********************\n",
      "processing  1 th loops---------------\n",
      "training set size: 20 unique(labels): [0 1] label counts: [11  9]\n",
      "Number of training examples  20\n",
      "best parameters is  SVC(C=0.01, break_ties=False, cache_size=200, class_weight='balanced',\n",
      "    coef0=0.0, decision_function_shape='ovr', degree=3, gamma='scale',\n",
      "    kernel='linear', max_iter=10000, probability=True, random_state=2019,\n",
      "    shrinking=True, tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.60      0.94      0.73       989\n",
      "           1       0.86      0.36      0.51       991\n",
      "\n",
      "    accuracy                           0.65      1980\n",
      "   macro avg       0.73      0.65      0.62      1980\n",
      "weighted avg       0.73      0.65      0.62      1980\n",
      "\n",
      "TP_H 9  TN_H 11  TP_M 359  TN_M 932  FP_M 57  FN_M 632\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  30 ********************\n",
      "processing  2 th loops---------------\n",
      "training set size: 30 unique(labels): [0 1] label counts: [11 19]\n",
      "Number of training examples  30\n",
      "best parameters is  SVC(C=0.01, break_ties=False, cache_size=200, class_weight='balanced',\n",
      "    coef0=0.0, decision_function_shape='ovr', degree=3, gamma='scale',\n",
      "    kernel='linear', max_iter=10000, probability=True, random_state=2019,\n",
      "    shrinking=True, tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.61      0.97      0.75       989\n",
      "           1       0.94      0.38      0.54       981\n",
      "\n",
      "    accuracy                           0.68      1970\n",
      "   macro avg       0.77      0.68      0.65      1970\n",
      "weighted avg       0.77      0.68      0.65      1970\n",
      "\n",
      "TP_H 19  TN_H 11  TP_M 375  TN_M 963  FP_M 26  FN_M 606\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  40 ********************\n",
      "processing  3 th loops---------------\n",
      "training set size: 40 unique(labels): [0 1] label counts: [12 28]\n",
      "Number of training examples  40\n",
      "best parameters is  SVC(C=0.01, break_ties=False, cache_size=200, class_weight='balanced',\n",
      "    coef0=0.0, decision_function_shape='ovr', degree=3, gamma='scale',\n",
      "    kernel='linear', max_iter=10000, probability=True, random_state=2019,\n",
      "    shrinking=True, tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.89      0.84       988\n",
      "           1       0.88      0.77      0.82       972\n",
      "\n",
      "    accuracy                           0.83      1960\n",
      "   macro avg       0.84      0.83      0.83      1960\n",
      "weighted avg       0.84      0.83      0.83      1960\n",
      "\n",
      "TP_H 28  TN_H 12  TP_M 744  TN_M 884  FP_M 104  FN_M 228\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  50 ********************\n",
      "processing  4 th loops---------------\n",
      "training set size: 50 unique(labels): [0 1] label counts: [15 35]\n",
      "Number of training examples  50\n",
      "best parameters is  SVC(C=0.01, break_ties=False, cache_size=200, class_weight='balanced',\n",
      "    coef0=0.0, decision_function_shape='ovr', degree=3, gamma='scale',\n",
      "    kernel='linear', max_iter=10000, probability=True, random_state=2019,\n",
      "    shrinking=True, tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.87      0.82       985\n",
      "           1       0.85      0.75      0.80       965\n",
      "\n",
      "    accuracy                           0.81      1950\n",
      "   macro avg       0.81      0.81      0.81      1950\n",
      "weighted avg       0.81      0.81      0.81      1950\n",
      "\n",
      "TP_H 35  TN_H 15  TP_M 722  TN_M 857  FP_M 128  FN_M 243\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  60 ********************\n",
      "processing  5 th loops---------------\n",
      "training set size: 60 unique(labels): [0 1] label counts: [18 42]\n",
      "Number of training examples  60\n",
      "best parameters is  SVC(C=0.01, break_ties=False, cache_size=200, class_weight='balanced',\n",
      "    coef0=0.0, decision_function_shape='ovr', degree=3, gamma='scale',\n",
      "    kernel='linear', max_iter=10000, probability=True, random_state=2019,\n",
      "    shrinking=True, tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.72      0.94      0.81       982\n",
      "           1       0.91      0.62      0.74       958\n",
      "\n",
      "    accuracy                           0.78      1940\n",
      "   macro avg       0.81      0.78      0.77      1940\n",
      "weighted avg       0.81      0.78      0.78      1940\n",
      "\n",
      "TP_H 42  TN_H 18  TP_M 595  TN_M 920  FP_M 62  FN_M 363\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  70 ********************\n",
      "processing  6 th loops---------------\n",
      "training set size: 70 unique(labels): [0 1] label counts: [20 50]\n",
      "Number of training examples  70\n",
      "best parameters is  SVC(C=0.01, break_ties=False, cache_size=200, class_weight='balanced',\n",
      "    coef0=0.0, decision_function_shape='ovr', degree=3, gamma='scale',\n",
      "    kernel='linear', max_iter=10000, probability=True, random_state=2019,\n",
      "    shrinking=True, tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.71      0.98      0.82       980\n",
      "           1       0.96      0.58      0.72       950\n",
      "\n",
      "    accuracy                           0.78      1930\n",
      "   macro avg       0.83      0.78      0.77      1930\n",
      "weighted avg       0.83      0.78      0.77      1930\n",
      "\n",
      "TP_H 50  TN_H 20  TP_M 550  TN_M 957  FP_M 23  FN_M 400\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  80 ********************\n",
      "processing  7 th loops---------------\n",
      "training set size: 80 unique(labels): [0 1] label counts: [23 57]\n",
      "Number of training examples  80\n",
      "best parameters is  SVC(C=0.01, break_ties=False, cache_size=200, class_weight='balanced',\n",
      "    coef0=0.0, decision_function_shape='ovr', degree=3, gamma='scale',\n",
      "    kernel='linear', max_iter=10000, probability=True, random_state=2019,\n",
      "    shrinking=True, tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.94      0.85       977\n",
      "           1       0.92      0.73      0.81       943\n",
      "\n",
      "    accuracy                           0.84      1920\n",
      "   macro avg       0.85      0.84      0.83      1920\n",
      "weighted avg       0.85      0.84      0.83      1920\n",
      "\n",
      "TP_H 57  TN_H 23  TP_M 688  TN_M 919  FP_M 58  FN_M 255\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  90 ********************\n",
      "processing  8 th loops---------------\n",
      "training set size: 90 unique(labels): [0 1] label counts: [26 64]\n",
      "Number of training examples  90\n",
      "best parameters is  SVC(C=0.01, break_ties=False, cache_size=200, class_weight='balanced',\n",
      "    coef0=0.0, decision_function_shape='ovr', degree=3, gamma='scale',\n",
      "    kernel='linear', max_iter=10000, probability=True, random_state=2019,\n",
      "    shrinking=True, tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.76      0.96      0.84       974\n",
      "           1       0.94      0.68      0.79       936\n",
      "\n",
      "    accuracy                           0.82      1910\n",
      "   macro avg       0.85      0.82      0.82      1910\n",
      "weighted avg       0.85      0.82      0.82      1910\n",
      "\n",
      "TP_H 64  TN_H 26  TP_M 634  TN_M 933  FP_M 41  FN_M 302\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  100 ********************\n",
      "processing  9 th loops---------------\n",
      "training set size: 100 unique(labels): [0 1] label counts: [27 73]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training examples  100\n",
      "best parameters is  SVC(C=0.01, break_ties=False, cache_size=200, class_weight='balanced',\n",
      "    coef0=0.0, decision_function_shape='ovr', degree=3, gamma='scale',\n",
      "    kernel='linear', max_iter=10000, probability=True, random_state=2019,\n",
      "    shrinking=True, tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.95      0.87       973\n",
      "           1       0.94      0.74      0.83       927\n",
      "\n",
      "    accuracy                           0.85      1900\n",
      "   macro avg       0.87      0.85      0.85      1900\n",
      "weighted avg       0.86      0.85      0.85      1900\n",
      "\n",
      "TP_H 73  TN_H 27  TP_M 683  TN_M 929  FP_M 44  FN_M 244\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  110 ********************\n",
      "processing  10 th loops---------------\n",
      "training set size: 110 unique(labels): [0 1] label counts: [31 79]\n",
      "Number of training examples  110\n",
      "start gridsearch ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:296: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n",
      "  FutureWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  {'C': 0.1, 'kernel': 'linear'}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.83      0.87       969\n",
      "           1       0.84      0.90      0.87       921\n",
      "\n",
      "    accuracy                           0.87      1890\n",
      "   macro avg       0.87      0.87      0.87      1890\n",
      "weighted avg       0.87      0.87      0.87      1890\n",
      "\n",
      "TP_H 79  TN_H 31  TP_M 833  TN_M 808  FP_M 161  FN_M 88\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  120 ********************\n",
      "processing  11 th loops---------------\n",
      "training set size: 120 unique(labels): [0 1] label counts: [39 81]\n",
      "Number of training examples  120\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.89      0.89       961\n",
      "           1       0.89      0.89      0.89       919\n",
      "\n",
      "    accuracy                           0.89      1880\n",
      "   macro avg       0.89      0.89      0.89      1880\n",
      "weighted avg       0.89      0.89      0.89      1880\n",
      "\n",
      "TP_H 81  TN_H 39  TP_M 816  TN_M 857  FP_M 104  FN_M 103\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  130 ********************\n",
      "processing  12 th loops---------------\n",
      "training set size: 130 unique(labels): [0 1] label counts: [44 86]\n",
      "Number of training examples  130\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.89      0.89       956\n",
      "           1       0.88      0.89      0.89       914\n",
      "\n",
      "    accuracy                           0.89      1870\n",
      "   macro avg       0.89      0.89      0.89      1870\n",
      "weighted avg       0.89      0.89      0.89      1870\n",
      "\n",
      "TP_H 86  TN_H 44  TP_M 817  TN_M 848  FP_M 108  FN_M 97\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  140 ********************\n",
      "processing  13 th loops---------------\n",
      "training set size: 140 unique(labels): [0 1] label counts: [49 91]\n",
      "Number of training examples  140\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.87      0.88       951\n",
      "           1       0.86      0.90      0.88       909\n",
      "\n",
      "    accuracy                           0.88      1860\n",
      "   macro avg       0.88      0.88      0.88      1860\n",
      "weighted avg       0.88      0.88      0.88      1860\n",
      "\n",
      "TP_H 91  TN_H 49  TP_M 816  TN_M 823  FP_M 128  FN_M 93\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  150 ********************\n",
      "processing  14 th loops---------------\n",
      "training set size: 150 unique(labels): [0 1] label counts: [55 95]\n",
      "Number of training examples  150\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.88      0.88       945\n",
      "           1       0.87      0.88      0.87       905\n",
      "\n",
      "    accuracy                           0.88      1850\n",
      "   macro avg       0.88      0.88      0.88      1850\n",
      "weighted avg       0.88      0.88      0.88      1850\n",
      "\n",
      "TP_H 95  TN_H 55  TP_M 795  TN_M 827  FP_M 118  FN_M 110\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  160 ********************\n",
      "processing  15 th loops---------------\n",
      "training set size: 160 unique(labels): [0 1] label counts: [62 98]\n",
      "Number of training examples  160\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.92      0.90       938\n",
      "           1       0.91      0.87      0.89       902\n",
      "\n",
      "    accuracy                           0.89      1840\n",
      "   macro avg       0.90      0.89      0.89      1840\n",
      "weighted avg       0.90      0.89      0.89      1840\n",
      "\n",
      "TP_H 98  TN_H 62  TP_M 784  TN_M 862  FP_M 76  FN_M 118\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  170 ********************\n",
      "processing  16 th loops---------------\n",
      "training set size: 170 unique(labels): [0 1] label counts: [ 67 103]\n",
      "Number of training examples  170\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.93      0.90       933\n",
      "           1       0.92      0.87      0.89       897\n",
      "\n",
      "    accuracy                           0.90      1830\n",
      "   macro avg       0.90      0.90      0.90      1830\n",
      "weighted avg       0.90      0.90      0.90      1830\n",
      "\n",
      "TP_H 103  TN_H 67  TP_M 779  TN_M 867  FP_M 66  FN_M 118\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  180 ********************\n",
      "processing  17 th loops---------------\n",
      "training set size: 180 unique(labels): [0 1] label counts: [ 69 111]\n",
      "Number of training examples  180\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.93      0.91       931\n",
      "           1       0.93      0.88      0.91       889\n",
      "\n",
      "    accuracy                           0.91      1820\n",
      "   macro avg       0.91      0.91      0.91      1820\n",
      "weighted avg       0.91      0.91      0.91      1820\n",
      "\n",
      "TP_H 111  TN_H 69  TP_M 786  TN_M 870  FP_M 61  FN_M 103\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  190 ********************\n",
      "processing  18 th loops---------------\n",
      "training set size: 190 unique(labels): [0 1] label counts: [ 74 116]\n",
      "Number of training examples  190\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.93      0.91       926\n",
      "           1       0.92      0.88      0.90       884\n",
      "\n",
      "    accuracy                           0.90      1810\n",
      "   macro avg       0.90      0.90      0.90      1810\n",
      "weighted avg       0.90      0.90      0.90      1810\n",
      "\n",
      "TP_H 116  TN_H 74  TP_M 779  TN_M 857  FP_M 69  FN_M 105\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  200 ********************\n",
      "processing  19 th loops---------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training set size: 200 unique(labels): [0 1] label counts: [ 78 122]\n",
      "Number of training examples  200\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.93      0.91       922\n",
      "           1       0.92      0.88      0.90       878\n",
      "\n",
      "    accuracy                           0.90      1800\n",
      "   macro avg       0.91      0.90      0.90      1800\n",
      "weighted avg       0.91      0.90      0.90      1800\n",
      "\n",
      "TP_H 122  TN_H 78  TP_M 772  TN_M 856  FP_M 66  FN_M 106\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  210 ********************\n",
      "processing  20 th loops---------------\n",
      "training set size: 210 unique(labels): [0 1] label counts: [ 79 131]\n",
      "Number of training examples  210\n",
      "start gridsearch ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:296: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n",
      "  FutureWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  {'C': 1, 'kernel': 'linear'}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.89      0.91       921\n",
      "           1       0.89      0.92      0.90       869\n",
      "\n",
      "    accuracy                           0.91      1790\n",
      "   macro avg       0.91      0.91      0.91      1790\n",
      "weighted avg       0.91      0.91      0.91      1790\n",
      "\n",
      "TP_H 131  TN_H 79  TP_M 798  TN_M 822  FP_M 99  FN_M 71\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  220 ********************\n",
      "processing  21 th loops---------------\n",
      "training set size: 220 unique(labels): [0 1] label counts: [ 84 136]\n",
      "Number of training examples  220\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.91      0.91       916\n",
      "           1       0.90      0.91      0.91       864\n",
      "\n",
      "    accuracy                           0.91      1780\n",
      "   macro avg       0.91      0.91      0.91      1780\n",
      "weighted avg       0.91      0.91      0.91      1780\n",
      "\n",
      "TP_H 136  TN_H 84  TP_M 789  TN_M 830  FP_M 86  FN_M 75\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  230 ********************\n",
      "processing  22 th loops---------------\n",
      "training set size: 230 unique(labels): [0 1] label counts: [ 90 140]\n",
      "Number of training examples  230\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.91      0.92       910\n",
      "           1       0.91      0.92      0.91       860\n",
      "\n",
      "    accuracy                           0.91      1770\n",
      "   macro avg       0.91      0.91      0.91      1770\n",
      "weighted avg       0.91      0.91      0.91      1770\n",
      "\n",
      "TP_H 140  TN_H 90  TP_M 789  TN_M 829  FP_M 81  FN_M 71\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  240 ********************\n",
      "processing  23 th loops---------------\n",
      "training set size: 240 unique(labels): [0 1] label counts: [ 94 146]\n",
      "Number of training examples  240\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.92      0.92       906\n",
      "           1       0.91      0.91      0.91       854\n",
      "\n",
      "    accuracy                           0.91      1760\n",
      "   macro avg       0.91      0.91      0.91      1760\n",
      "weighted avg       0.91      0.91      0.91      1760\n",
      "\n",
      "TP_H 146  TN_H 94  TP_M 775  TN_M 831  FP_M 75  FN_M 79\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  250 ********************\n",
      "processing  24 th loops---------------\n",
      "training set size: 250 unique(labels): [0 1] label counts: [ 97 153]\n",
      "Number of training examples  250\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.92      0.92       903\n",
      "           1       0.92      0.92      0.92       847\n",
      "\n",
      "    accuracy                           0.92      1750\n",
      "   macro avg       0.92      0.92      0.92      1750\n",
      "weighted avg       0.92      0.92      0.92      1750\n",
      "\n",
      "TP_H 153  TN_H 97  TP_M 780  TN_M 831  FP_M 72  FN_M 67\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  260 ********************\n",
      "processing  25 th loops---------------\n",
      "training set size: 260 unique(labels): [0 1] label counts: [101 159]\n",
      "Number of training examples  260\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.92      0.93       899\n",
      "           1       0.92      0.93      0.92       841\n",
      "\n",
      "    accuracy                           0.92      1740\n",
      "   macro avg       0.92      0.92      0.92      1740\n",
      "weighted avg       0.92      0.92      0.92      1740\n",
      "\n",
      "TP_H 159  TN_H 101  TP_M 779  TN_M 828  FP_M 71  FN_M 62\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  270 ********************\n",
      "processing  26 th loops---------------\n",
      "training set size: 270 unique(labels): [0 1] label counts: [107 163]\n",
      "Number of training examples  270\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.92      0.93       893\n",
      "           1       0.92      0.93      0.92       837\n",
      "\n",
      "    accuracy                           0.92      1730\n",
      "   macro avg       0.92      0.92      0.92      1730\n",
      "weighted avg       0.92      0.92      0.92      1730\n",
      "\n",
      "TP_H 163  TN_H 107  TP_M 776  TN_M 821  FP_M 72  FN_M 61\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  280 ********************\n",
      "processing  27 th loops---------------\n",
      "training set size: 280 unique(labels): [0 1] label counts: [111 169]\n",
      "Number of training examples  280\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.91      0.92       889\n",
      "           1       0.91      0.93      0.92       831\n",
      "\n",
      "    accuracy                           0.92      1720\n",
      "   macro avg       0.92      0.92      0.92      1720\n",
      "weighted avg       0.92      0.92      0.92      1720\n",
      "\n",
      "TP_H 169  TN_H 111  TP_M 770  TN_M 809  FP_M 80  FN_M 61\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  290 ********************\n",
      "processing  28 th loops---------------\n",
      "training set size: 290 unique(labels): [0 1] label counts: [118 172]\n",
      "Number of training examples  290\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.91      0.92       882\n",
      "           1       0.91      0.93      0.92       828\n",
      "\n",
      "    accuracy                           0.92      1710\n",
      "   macro avg       0.92      0.92      0.92      1710\n",
      "weighted avg       0.92      0.92      0.92      1710\n",
      "\n",
      "TP_H 172  TN_H 118  TP_M 767  TN_M 806  FP_M 76  FN_M 61\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  300 ********************\n",
      "processing  29 th loops---------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training set size: 300 unique(labels): [0 1] label counts: [124 176]\n",
      "Number of training examples  300\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.91      0.92       876\n",
      "           1       0.91      0.93      0.92       824\n",
      "\n",
      "    accuracy                           0.92      1700\n",
      "   macro avg       0.92      0.92      0.92      1700\n",
      "weighted avg       0.92      0.92      0.92      1700\n",
      "\n",
      "TP_H 176  TN_H 124  TP_M 769  TN_M 798  FP_M 78  FN_M 55\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  310 ********************\n",
      "processing  30 th loops---------------\n",
      "training set size: 310 unique(labels): [0 1] label counts: [131 179]\n",
      "Number of training examples  310\n",
      "start gridsearch ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:296: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n",
      "  FutureWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  {'C': 1, 'kernel': 'linear'}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.92      0.93       869\n",
      "           1       0.92      0.93      0.93       821\n",
      "\n",
      "    accuracy                           0.93      1690\n",
      "   macro avg       0.93      0.93      0.93      1690\n",
      "weighted avg       0.93      0.93      0.93      1690\n",
      "\n",
      "TP_H 179  TN_H 131  TP_M 766  TN_M 800  FP_M 69  FN_M 55\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  320 ********************\n",
      "processing  31 th loops---------------\n",
      "training set size: 320 unique(labels): [0 1] label counts: [139 181]\n",
      "Number of training examples  320\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.93      0.93       861\n",
      "           1       0.92      0.93      0.93       819\n",
      "\n",
      "    accuracy                           0.93      1680\n",
      "   macro avg       0.93      0.93      0.93      1680\n",
      "weighted avg       0.93      0.93      0.93      1680\n",
      "\n",
      "TP_H 181  TN_H 139  TP_M 762  TN_M 798  FP_M 63  FN_M 57\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  330 ********************\n",
      "processing  32 th loops---------------\n",
      "training set size: 330 unique(labels): [0 1] label counts: [146 184]\n",
      "Number of training examples  330\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.93      0.93       854\n",
      "           1       0.93      0.93      0.93       816\n",
      "\n",
      "    accuracy                           0.93      1670\n",
      "   macro avg       0.93      0.93      0.93      1670\n",
      "weighted avg       0.93      0.93      0.93      1670\n",
      "\n",
      "TP_H 184  TN_H 146  TP_M 760  TN_M 796  FP_M 58  FN_M 56\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  340 ********************\n",
      "processing  33 th loops---------------\n",
      "training set size: 340 unique(labels): [0 1] label counts: [152 188]\n",
      "Number of training examples  340\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.93      0.93       848\n",
      "           1       0.93      0.93      0.93       812\n",
      "\n",
      "    accuracy                           0.93      1660\n",
      "   macro avg       0.93      0.93      0.93      1660\n",
      "weighted avg       0.93      0.93      0.93      1660\n",
      "\n",
      "TP_H 188  TN_H 152  TP_M 757  TN_M 791  FP_M 57  FN_M 55\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  350 ********************\n",
      "processing  34 th loops---------------\n",
      "training set size: 350 unique(labels): [0 1] label counts: [158 192]\n",
      "Number of training examples  350\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.94      0.94       842\n",
      "           1       0.94      0.94      0.94       808\n",
      "\n",
      "    accuracy                           0.94      1650\n",
      "   macro avg       0.94      0.94      0.94      1650\n",
      "weighted avg       0.94      0.94      0.94      1650\n",
      "\n",
      "TP_H 192  TN_H 158  TP_M 756  TN_M 793  FP_M 49  FN_M 52\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  360 ********************\n",
      "processing  35 th loops---------------\n",
      "training set size: 360 unique(labels): [0 1] label counts: [162 198]\n",
      "Number of training examples  360\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.94      0.94       838\n",
      "           1       0.94      0.94      0.94       802\n",
      "\n",
      "    accuracy                           0.94      1640\n",
      "   macro avg       0.94      0.94      0.94      1640\n",
      "weighted avg       0.94      0.94      0.94      1640\n",
      "\n",
      "TP_H 198  TN_H 162  TP_M 754  TN_M 790  FP_M 48  FN_M 48\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  370 ********************\n",
      "processing  36 th loops---------------\n",
      "training set size: 370 unique(labels): [0 1] label counts: [167 203]\n",
      "Number of training examples  370\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.94      0.94       833\n",
      "           1       0.94      0.94      0.94       797\n",
      "\n",
      "    accuracy                           0.94      1630\n",
      "   macro avg       0.94      0.94      0.94      1630\n",
      "weighted avg       0.94      0.94      0.94      1630\n",
      "\n",
      "TP_H 203  TN_H 167  TP_M 753  TN_M 784  FP_M 49  FN_M 44\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  380 ********************\n",
      "processing  37 th loops---------------\n",
      "training set size: 380 unique(labels): [0 1] label counts: [173 207]\n",
      "Number of training examples  380\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.95      0.95       827\n",
      "           1       0.94      0.95      0.95       793\n",
      "\n",
      "    accuracy                           0.95      1620\n",
      "   macro avg       0.95      0.95      0.95      1620\n",
      "weighted avg       0.95      0.95      0.95      1620\n",
      "\n",
      "TP_H 207  TN_H 173  TP_M 750  TN_M 783  FP_M 44  FN_M 43\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  390 ********************\n",
      "processing  38 th loops---------------\n",
      "training set size: 390 unique(labels): [0 1] label counts: [179 211]\n",
      "Number of training examples  390\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.95      0.95       821\n",
      "           1       0.94      0.95      0.94       789\n",
      "\n",
      "    accuracy                           0.95      1610\n",
      "   macro avg       0.95      0.95      0.95      1610\n",
      "weighted avg       0.95      0.95      0.95      1610\n",
      "\n",
      "TP_H 211  TN_H 179  TP_M 747  TN_M 776  FP_M 45  FN_M 42\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  400 ********************\n",
      "processing  39 th loops---------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training set size: 400 unique(labels): [0 1] label counts: [186 214]\n",
      "Number of training examples  400\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.95      0.95       814\n",
      "           1       0.95      0.95      0.95       786\n",
      "\n",
      "    accuracy                           0.95      1600\n",
      "   macro avg       0.95      0.95      0.95      1600\n",
      "weighted avg       0.95      0.95      0.95      1600\n",
      "\n",
      "TP_H 214  TN_H 186  TP_M 743  TN_M 772  FP_M 42  FN_M 43\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  410 ********************\n",
      "processing  40 th loops---------------\n",
      "training set size: 410 unique(labels): [0 1] label counts: [191 219]\n",
      "Number of training examples  410\n",
      "start gridsearch ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:296: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n",
      "  FutureWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  {'C': 1, 'kernel': 'linear'}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.95      0.95       809\n",
      "           1       0.95      0.95      0.95       781\n",
      "\n",
      "    accuracy                           0.95      1590\n",
      "   macro avg       0.95      0.95      0.95      1590\n",
      "weighted avg       0.95      0.95      0.95      1590\n",
      "\n",
      "TP_H 219  TN_H 191  TP_M 740  TN_M 772  FP_M 37  FN_M 41\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  420 ********************\n",
      "processing  41 th loops---------------\n",
      "training set size: 420 unique(labels): [0 1] label counts: [198 222]\n",
      "Number of training examples  420\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.96      0.95       802\n",
      "           1       0.96      0.95      0.95       778\n",
      "\n",
      "    accuracy                           0.95      1580\n",
      "   macro avg       0.95      0.95      0.95      1580\n",
      "weighted avg       0.95      0.95      0.95      1580\n",
      "\n",
      "TP_H 222  TN_H 198  TP_M 738  TN_M 768  FP_M 34  FN_M 40\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  430 ********************\n",
      "processing  42 th loops---------------\n",
      "training set size: 430 unique(labels): [0 1] label counts: [204 226]\n",
      "Number of training examples  430\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.96      0.96       796\n",
      "           1       0.96      0.95      0.95       774\n",
      "\n",
      "    accuracy                           0.96      1570\n",
      "   macro avg       0.96      0.96      0.96      1570\n",
      "weighted avg       0.96      0.96      0.96      1570\n",
      "\n",
      "TP_H 226  TN_H 204  TP_M 735  TN_M 765  FP_M 31  FN_M 39\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  440 ********************\n",
      "processing  43 th loops---------------\n",
      "training set size: 440 unique(labels): [0 1] label counts: [208 232]\n",
      "Number of training examples  440\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.96      0.96       792\n",
      "           1       0.96      0.95      0.96       768\n",
      "\n",
      "    accuracy                           0.96      1560\n",
      "   macro avg       0.96      0.96      0.96      1560\n",
      "weighted avg       0.96      0.96      0.96      1560\n",
      "\n",
      "TP_H 232  TN_H 208  TP_M 732  TN_M 763  FP_M 29  FN_M 36\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  450 ********************\n",
      "processing  44 th loops---------------\n",
      "training set size: 450 unique(labels): [0 1] label counts: [213 237]\n",
      "Number of training examples  450\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.97      0.96       787\n",
      "           1       0.97      0.95      0.96       763\n",
      "\n",
      "    accuracy                           0.96      1550\n",
      "   macro avg       0.96      0.96      0.96      1550\n",
      "weighted avg       0.96      0.96      0.96      1550\n",
      "\n",
      "TP_H 237  TN_H 213  TP_M 728  TN_M 761  FP_M 26  FN_M 35\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  460 ********************\n",
      "processing  45 th loops---------------\n",
      "training set size: 460 unique(labels): [0 1] label counts: [218 242]\n",
      "Number of training examples  460\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.97      0.96       782\n",
      "           1       0.97      0.96      0.96       758\n",
      "\n",
      "    accuracy                           0.96      1540\n",
      "   macro avg       0.96      0.96      0.96      1540\n",
      "weighted avg       0.96      0.96      0.96      1540\n",
      "\n",
      "TP_H 242  TN_H 218  TP_M 724  TN_M 758  FP_M 24  FN_M 34\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  470 ********************\n",
      "processing  46 th loops---------------\n",
      "training set size: 470 unique(labels): [0 1] label counts: [221 249]\n",
      "Number of training examples  470\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.97      0.97       779\n",
      "           1       0.97      0.96      0.96       751\n",
      "\n",
      "    accuracy                           0.96      1530\n",
      "   macro avg       0.96      0.96      0.96      1530\n",
      "weighted avg       0.96      0.96      0.96      1530\n",
      "\n",
      "TP_H 249  TN_H 221  TP_M 720  TN_M 756  FP_M 23  FN_M 31\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  480 ********************\n",
      "processing  47 th loops---------------\n",
      "training set size: 480 unique(labels): [0 1] label counts: [226 254]\n",
      "Number of training examples  480\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.97      0.97       774\n",
      "           1       0.97      0.96      0.97       746\n",
      "\n",
      "    accuracy                           0.97      1520\n",
      "   macro avg       0.97      0.97      0.97      1520\n",
      "weighted avg       0.97      0.97      0.97      1520\n",
      "\n",
      "TP_H 254  TN_H 226  TP_M 716  TN_M 753  FP_M 21  FN_M 30\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  490 ********************\n",
      "processing  48 th loops---------------\n",
      "training set size: 490 unique(labels): [0 1] label counts: [230 260]\n",
      "Number of training examples  490\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.97      0.97       770\n",
      "           1       0.97      0.96      0.97       740\n",
      "\n",
      "    accuracy                           0.97      1510\n",
      "   macro avg       0.97      0.97      0.97      1510\n",
      "weighted avg       0.97      0.97      0.97      1510\n",
      "\n",
      "TP_H 260  TN_H 230  TP_M 714  TN_M 750  FP_M 20  FN_M 26\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  500 ********************\n",
      "processing  49 th loops---------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training set size: 500 unique(labels): [0 1] label counts: [237 263]\n",
      "Number of training examples  500\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.97      0.97       763\n",
      "           1       0.97      0.96      0.97       737\n",
      "\n",
      "    accuracy                           0.97      1500\n",
      "   macro avg       0.97      0.97      0.97      1500\n",
      "weighted avg       0.97      0.97      0.97      1500\n",
      "\n",
      "TP_H 263  TN_H 237  TP_M 711  TN_M 743  FP_M 20  FN_M 26\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  510 ********************\n",
      "processing  50 th loops---------------\n",
      "training set size: 510 unique(labels): [0 1] label counts: [242 268]\n",
      "Number of training examples  510\n",
      "start gridsearch ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:296: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n",
      "  FutureWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  {'C': 1, 'kernel': 'linear'}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.97      0.97       758\n",
      "           1       0.97      0.97      0.97       732\n",
      "\n",
      "    accuracy                           0.97      1490\n",
      "   macro avg       0.97      0.97      0.97      1490\n",
      "weighted avg       0.97      0.97      0.97      1490\n",
      "\n",
      "TP_H 268  TN_H 242  TP_M 708  TN_M 738  FP_M 20  FN_M 24\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  520 ********************\n",
      "processing  51 th loops---------------\n",
      "training set size: 520 unique(labels): [0 1] label counts: [247 273]\n",
      "Number of training examples  520\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.97      0.97       753\n",
      "           1       0.97      0.97      0.97       727\n",
      "\n",
      "    accuracy                           0.97      1480\n",
      "   macro avg       0.97      0.97      0.97      1480\n",
      "weighted avg       0.97      0.97      0.97      1480\n",
      "\n",
      "TP_H 273  TN_H 247  TP_M 704  TN_M 734  FP_M 19  FN_M 23\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  530 ********************\n",
      "processing  52 th loops---------------\n",
      "training set size: 530 unique(labels): [0 1] label counts: [251 279]\n",
      "Number of training examples  530\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.98      0.97       749\n",
      "           1       0.97      0.97      0.97       721\n",
      "\n",
      "    accuracy                           0.97      1470\n",
      "   macro avg       0.97      0.97      0.97      1470\n",
      "weighted avg       0.97      0.97      0.97      1470\n",
      "\n",
      "TP_H 279  TN_H 251  TP_M 700  TN_M 731  FP_M 18  FN_M 21\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  540 ********************\n",
      "processing  53 th loops---------------\n",
      "training set size: 540 unique(labels): [0 1] label counts: [259 281]\n",
      "Number of training examples  540\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.98      0.98       741\n",
      "           1       0.98      0.97      0.97       719\n",
      "\n",
      "    accuracy                           0.97      1460\n",
      "   macro avg       0.97      0.97      0.97      1460\n",
      "weighted avg       0.97      0.97      0.97      1460\n",
      "\n",
      "TP_H 281  TN_H 259  TP_M 699  TN_M 724  FP_M 17  FN_M 20\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  550 ********************\n",
      "processing  54 th loops---------------\n",
      "training set size: 550 unique(labels): [0 1] label counts: [266 284]\n",
      "Number of training examples  550\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.98      0.98       734\n",
      "           1       0.98      0.97      0.98       716\n",
      "\n",
      "    accuracy                           0.98      1450\n",
      "   macro avg       0.98      0.98      0.98      1450\n",
      "weighted avg       0.98      0.98      0.98      1450\n",
      "\n",
      "TP_H 284  TN_H 266  TP_M 697  TN_M 718  FP_M 16  FN_M 19\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  560 ********************\n",
      "processing  55 th loops---------------\n",
      "training set size: 560 unique(labels): [0 1] label counts: [275 285]\n",
      "Number of training examples  560\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.98      0.98       725\n",
      "           1       0.98      0.97      0.98       715\n",
      "\n",
      "    accuracy                           0.98      1440\n",
      "   macro avg       0.98      0.98      0.98      1440\n",
      "weighted avg       0.98      0.98      0.98      1440\n",
      "\n",
      "TP_H 285  TN_H 275  TP_M 696  TN_M 711  FP_M 14  FN_M 19\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  570 ********************\n",
      "processing  56 th loops---------------\n",
      "training set size: 570 unique(labels): [0 1] label counts: [278 292]\n",
      "Number of training examples  570\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       722\n",
      "           1       0.98      0.97      0.98       708\n",
      "\n",
      "    accuracy                           0.98      1430\n",
      "   macro avg       0.98      0.98      0.98      1430\n",
      "weighted avg       0.98      0.98      0.98      1430\n",
      "\n",
      "TP_H 292  TN_H 278  TP_M 690  TN_M 708  FP_M 14  FN_M 18\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  580 ********************\n",
      "processing  57 th loops---------------\n",
      "training set size: 580 unique(labels): [0 1] label counts: [285 295]\n",
      "Number of training examples  580\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.98      0.98       715\n",
      "           1       0.98      0.97      0.98       705\n",
      "\n",
      "    accuracy                           0.98      1420\n",
      "   macro avg       0.98      0.98      0.98      1420\n",
      "weighted avg       0.98      0.98      0.98      1420\n",
      "\n",
      "TP_H 295  TN_H 285  TP_M 687  TN_M 701  FP_M 14  FN_M 18\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  590 ********************\n",
      "processing  58 th loops---------------\n",
      "training set size: 590 unique(labels): [0 1] label counts: [289 301]\n",
      "Number of training examples  590\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       711\n",
      "           1       0.98      0.98      0.98       699\n",
      "\n",
      "    accuracy                           0.98      1410\n",
      "   macro avg       0.98      0.98      0.98      1410\n",
      "weighted avg       0.98      0.98      0.98      1410\n",
      "\n",
      "TP_H 301  TN_H 289  TP_M 682  TN_M 698  FP_M 13  FN_M 17\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  600 ********************\n",
      "processing  59 th loops---------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training set size: 600 unique(labels): [0 1] label counts: [293 307]\n",
      "Number of training examples  600\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       707\n",
      "           1       0.98      0.98      0.98       693\n",
      "\n",
      "    accuracy                           0.98      1400\n",
      "   macro avg       0.98      0.98      0.98      1400\n",
      "weighted avg       0.98      0.98      0.98      1400\n",
      "\n",
      "TP_H 307  TN_H 293  TP_M 677  TN_M 694  FP_M 13  FN_M 16\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  610 ********************\n",
      "processing  60 th loops---------------\n",
      "training set size: 610 unique(labels): [0 1] label counts: [298 312]\n",
      "Number of training examples  610\n",
      "start gridsearch ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:296: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n",
      "  FutureWarning\n",
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  {'C': 10, 'kernel': 'linear'}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       702\n",
      "           1       0.98      0.98      0.98       688\n",
      "\n",
      "    accuracy                           0.98      1390\n",
      "   macro avg       0.98      0.98      0.98      1390\n",
      "weighted avg       0.98      0.98      0.98      1390\n",
      "\n",
      "TP_H 312  TN_H 298  TP_M 672  TN_M 690  FP_M 12  FN_M 16\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  620 ********************\n",
      "processing  61 th loops---------------\n",
      "training set size: 620 unique(labels): [0 1] label counts: [305 315]\n",
      "Number of training examples  620\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=10, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       695\n",
      "           1       0.98      0.98      0.98       685\n",
      "\n",
      "    accuracy                           0.98      1380\n",
      "   macro avg       0.98      0.98      0.98      1380\n",
      "weighted avg       0.98      0.98      0.98      1380\n",
      "\n",
      "TP_H 315  TN_H 305  TP_M 669  TN_M 683  FP_M 12  FN_M 16\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  630 ********************\n",
      "processing  62 th loops---------------\n",
      "training set size: 630 unique(labels): [0 1] label counts: [309 321]\n",
      "Number of training examples  630\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=10, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       691\n",
      "           1       0.98      0.98      0.98       679\n",
      "\n",
      "    accuracy                           0.98      1370\n",
      "   macro avg       0.98      0.98      0.98      1370\n",
      "weighted avg       0.98      0.98      0.98      1370\n",
      "\n",
      "TP_H 321  TN_H 309  TP_M 663  TN_M 679  FP_M 12  FN_M 16\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  640 ********************\n",
      "processing  63 th loops---------------\n",
      "training set size: 640 unique(labels): [0 1] label counts: [317 323]\n",
      "Number of training examples  640\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=10, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       683\n",
      "           1       0.98      0.98      0.98       677\n",
      "\n",
      "    accuracy                           0.98      1360\n",
      "   macro avg       0.98      0.98      0.98      1360\n",
      "weighted avg       0.98      0.98      0.98      1360\n",
      "\n",
      "TP_H 323  TN_H 317  TP_M 661  TN_M 671  FP_M 12  FN_M 16\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  650 ********************\n",
      "processing  64 th loops---------------\n",
      "training set size: 650 unique(labels): [0 1] label counts: [323 327]\n",
      "Number of training examples  650\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=10, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       677\n",
      "           1       0.98      0.98      0.98       673\n",
      "\n",
      "    accuracy                           0.98      1350\n",
      "   macro avg       0.98      0.98      0.98      1350\n",
      "weighted avg       0.98      0.98      0.98      1350\n",
      "\n",
      "TP_H 327  TN_H 323  TP_M 657  TN_M 665  FP_M 12  FN_M 16\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  660 ********************\n",
      "processing  65 th loops---------------\n",
      "training set size: 660 unique(labels): [0 1] label counts: [327 333]\n",
      "Number of training examples  660\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=10, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       673\n",
      "           1       0.98      0.98      0.98       667\n",
      "\n",
      "    accuracy                           0.98      1340\n",
      "   macro avg       0.98      0.98      0.98      1340\n",
      "weighted avg       0.98      0.98      0.98      1340\n",
      "\n",
      "TP_H 333  TN_H 327  TP_M 652  TN_M 661  FP_M 12  FN_M 15\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  670 ********************\n",
      "processing  66 th loops---------------\n",
      "training set size: 670 unique(labels): [0 1] label counts: [334 336]\n",
      "Number of training examples  670\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=10, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       666\n",
      "           1       0.98      0.98      0.98       664\n",
      "\n",
      "    accuracy                           0.98      1330\n",
      "   macro avg       0.98      0.98      0.98      1330\n",
      "weighted avg       0.98      0.98      0.98      1330\n",
      "\n",
      "TP_H 336  TN_H 334  TP_M 650  TN_M 654  FP_M 12  FN_M 14\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  680 ********************\n",
      "processing  67 th loops---------------\n",
      "training set size: 680 unique(labels): [0 1] label counts: [340 340]\n",
      "Number of training examples  680\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=10, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       660\n",
      "           1       0.98      0.98      0.98       660\n",
      "\n",
      "    accuracy                           0.98      1320\n",
      "   macro avg       0.98      0.98      0.98      1320\n",
      "weighted avg       0.98      0.98      0.98      1320\n",
      "\n",
      "TP_H 340  TN_H 340  TP_M 646  TN_M 648  FP_M 12  FN_M 14\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  690 ********************\n",
      "processing  68 th loops---------------\n",
      "training set size: 690 unique(labels): [0 1] label counts: [347 343]\n",
      "Number of training examples  690\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=10, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       653\n",
      "           1       0.98      0.98      0.98       657\n",
      "\n",
      "    accuracy                           0.98      1310\n",
      "   macro avg       0.98      0.98      0.98      1310\n",
      "weighted avg       0.98      0.98      0.98      1310\n",
      "\n",
      "TP_H 343  TN_H 347  TP_M 643  TN_M 641  FP_M 12  FN_M 14\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  700 ********************\n",
      "processing  69 th loops---------------\n",
      "training set size: 700 unique(labels): [0 1] label counts: [355 345]\n",
      "Number of training examples  700\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=10, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       645\n",
      "           1       0.98      0.98      0.98       655\n",
      "\n",
      "    accuracy                           0.98      1300\n",
      "   macro avg       0.98      0.98      0.98      1300\n",
      "weighted avg       0.98      0.98      0.98      1300\n",
      "\n",
      "TP_H 345  TN_H 355  TP_M 641  TN_M 633  FP_M 12  FN_M 14\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  710 ********************\n",
      "processing  70 th loops---------------\n",
      "training set size: 710 unique(labels): [0 1] label counts: [359 351]\n",
      "Number of training examples  710\n",
      "start gridsearch ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:296: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n",
      "  FutureWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  {'C': 1, 'kernel': 'linear'}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       641\n",
      "           1       0.98      0.98      0.98       649\n",
      "\n",
      "    accuracy                           0.98      1290\n",
      "   macro avg       0.98      0.98      0.98      1290\n",
      "weighted avg       0.98      0.98      0.98      1290\n",
      "\n",
      "TP_H 351  TN_H 359  TP_M 637  TN_M 629  FP_M 12  FN_M 12\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  720 ********************\n",
      "processing  71 th loops---------------\n",
      "training set size: 720 unique(labels): [0 1] label counts: [366 354]\n",
      "Number of training examples  720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       634\n",
      "           1       0.98      0.98      0.98       646\n",
      "\n",
      "    accuracy                           0.98      1280\n",
      "   macro avg       0.98      0.98      0.98      1280\n",
      "weighted avg       0.98      0.98      0.98      1280\n",
      "\n",
      "TP_H 354  TN_H 366  TP_M 636  TN_M 622  FP_M 12  FN_M 10\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  730 ********************\n",
      "processing  72 th loops---------------\n",
      "training set size: 730 unique(labels): [0 1] label counts: [373 357]\n",
      "Number of training examples  730\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       627\n",
      "           1       0.98      0.98      0.98       643\n",
      "\n",
      "    accuracy                           0.98      1270\n",
      "   macro avg       0.98      0.98      0.98      1270\n",
      "weighted avg       0.98      0.98      0.98      1270\n",
      "\n",
      "TP_H 357  TN_H 373  TP_M 633  TN_M 615  FP_M 12  FN_M 10\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  740 ********************\n",
      "processing  73 th loops---------------\n",
      "training set size: 740 unique(labels): [0 1] label counts: [377 363]\n",
      "Number of training examples  740\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.98      0.98       623\n",
      "           1       0.98      0.99      0.98       637\n",
      "\n",
      "    accuracy                           0.98      1260\n",
      "   macro avg       0.98      0.98      0.98      1260\n",
      "weighted avg       0.98      0.98      0.98      1260\n",
      "\n",
      "TP_H 363  TN_H 377  TP_M 628  TN_M 612  FP_M 11  FN_M 9\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  750 ********************\n",
      "processing  74 th loops---------------\n",
      "training set size: 750 unique(labels): [0 1] label counts: [380 370]\n",
      "Number of training examples  750\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.98      0.99       620\n",
      "           1       0.98      0.99      0.99       630\n",
      "\n",
      "    accuracy                           0.99      1250\n",
      "   macro avg       0.99      0.99      0.99      1250\n",
      "weighted avg       0.99      0.99      0.99      1250\n",
      "\n",
      "TP_H 370  TN_H 380  TP_M 623  TN_M 610  FP_M 10  FN_M 7\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  760 ********************\n",
      "processing  75 th loops---------------\n",
      "training set size: 760 unique(labels): [0 1] label counts: [386 374]\n",
      "Number of training examples  760\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       614\n",
      "           1       0.99      0.99      0.99       626\n",
      "\n",
      "    accuracy                           0.99      1240\n",
      "   macro avg       0.99      0.99      0.99      1240\n",
      "weighted avg       0.99      0.99      0.99      1240\n",
      "\n",
      "TP_H 374  TN_H 386  TP_M 619  TN_M 605  FP_M 9  FN_M 7\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  770 ********************\n",
      "processing  76 th loops---------------\n",
      "training set size: 770 unique(labels): [0 1] label counts: [393 377]\n",
      "Number of training examples  770\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       607\n",
      "           1       0.99      0.99      0.99       623\n",
      "\n",
      "    accuracy                           0.99      1230\n",
      "   macro avg       0.99      0.99      0.99      1230\n",
      "weighted avg       0.99      0.99      0.99      1230\n",
      "\n",
      "TP_H 377  TN_H 393  TP_M 616  TN_M 598  FP_M 9  FN_M 7\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  780 ********************\n",
      "processing  77 th loops---------------\n",
      "training set size: 780 unique(labels): [0 1] label counts: [401 379]\n",
      "Number of training examples  780\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       599\n",
      "           1       0.99      0.99      0.99       621\n",
      "\n",
      "    accuracy                           0.99      1220\n",
      "   macro avg       0.99      0.99      0.99      1220\n",
      "weighted avg       0.99      0.99      0.99      1220\n",
      "\n",
      "TP_H 379  TN_H 401  TP_M 614  TN_M 591  FP_M 8  FN_M 7\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  790 ********************\n",
      "processing  78 th loops---------------\n",
      "training set size: 790 unique(labels): [0 1] label counts: [408 382]\n",
      "Number of training examples  790\n",
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       592\n",
      "           1       0.99      0.99      0.99       618\n",
      "\n",
      "    accuracy                           0.99      1210\n",
      "   macro avg       0.99      0.99      0.99      1210\n",
      "weighted avg       0.99      0.99      0.99      1210\n",
      "\n",
      "TP_H 382  TN_H 408  TP_M 611  TN_M 584  FP_M 8  FN_M 7\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  800 ********************\n",
      "processing  79 th loops---------------\n",
      "training set size: 800 unique(labels): [0 1] label counts: [413 387]\n",
      "Number of training examples  800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       587\n",
      "           1       0.99      0.99      0.99       613\n",
      "\n",
      "    accuracy                           0.99      1200\n",
      "   macro avg       0.99      0.99      0.99      1200\n",
      "weighted avg       0.99      0.99      0.99      1200\n",
      "\n",
      "TP_H 387  TN_H 413  TP_M 607  TN_M 579  FP_M 8  FN_M 6\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  810 ********************\n",
      "processing  80 th loops---------------\n",
      "training set size: 810 unique(labels): [0 1] label counts: [417 393]\n",
      "Number of training examples  810\n",
      "start gridsearch ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:296: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n",
      "  FutureWarning\n",
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  {'C': 1, 'kernel': 'linear'}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       583\n",
      "           1       0.99      0.99      0.99       607\n",
      "\n",
      "    accuracy                           0.99      1190\n",
      "   macro avg       0.99      0.99      0.99      1190\n",
      "weighted avg       0.99      0.99      0.99      1190\n",
      "\n",
      "TP_H 393  TN_H 417  TP_M 601  TN_M 575  FP_M 8  FN_M 6\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  820 ********************\n",
      "processing  81 th loops---------------\n",
      "training set size: 820 unique(labels): [0 1] label counts: [423 397]\n",
      "Number of training examples  820\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       577\n",
      "           1       0.99      0.99      0.99       603\n",
      "\n",
      "    accuracy                           0.99      1180\n",
      "   macro avg       0.99      0.99      0.99      1180\n",
      "weighted avg       0.99      0.99      0.99      1180\n",
      "\n",
      "TP_H 397  TN_H 423  TP_M 597  TN_M 569  FP_M 8  FN_M 6\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  830 ********************\n",
      "processing  82 th loops---------------\n",
      "training set size: 830 unique(labels): [0 1] label counts: [424 406]\n",
      "Number of training examples  830\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       576\n",
      "           1       0.99      0.99      0.99       594\n",
      "\n",
      "    accuracy                           0.99      1170\n",
      "   macro avg       0.99      0.99      0.99      1170\n",
      "weighted avg       0.99      0.99      0.99      1170\n",
      "\n",
      "TP_H 406  TN_H 424  TP_M 588  TN_M 568  FP_M 8  FN_M 6\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  840 ********************\n",
      "processing  83 th loops---------------\n",
      "training set size: 840 unique(labels): [0 1] label counts: [430 410]\n",
      "Number of training examples  840\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       570\n",
      "           1       0.99      0.99      0.99       590\n",
      "\n",
      "    accuracy                           0.99      1160\n",
      "   macro avg       0.99      0.99      0.99      1160\n",
      "weighted avg       0.99      0.99      0.99      1160\n",
      "\n",
      "TP_H 410  TN_H 430  TP_M 584  TN_M 562  FP_M 8  FN_M 6\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  850 ********************\n",
      "processing  84 th loops---------------\n",
      "training set size: 850 unique(labels): [0 1] label counts: [433 417]\n",
      "Number of training examples  850\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       567\n",
      "           1       0.99      0.99      0.99       583\n",
      "\n",
      "    accuracy                           0.99      1150\n",
      "   macro avg       0.99      0.99      0.99      1150\n",
      "weighted avg       0.99      0.99      0.99      1150\n",
      "\n",
      "TP_H 417  TN_H 433  TP_M 578  TN_M 559  FP_M 8  FN_M 5\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  860 ********************\n",
      "processing  85 th loops---------------\n",
      "training set size: 860 unique(labels): [0 1] label counts: [437 423]\n",
      "Number of training examples  860\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       563\n",
      "           1       0.99      0.99      0.99       577\n",
      "\n",
      "    accuracy                           0.99      1140\n",
      "   macro avg       0.99      0.99      0.99      1140\n",
      "weighted avg       0.99      0.99      0.99      1140\n",
      "\n",
      "TP_H 423  TN_H 437  TP_M 573  TN_M 555  FP_M 8  FN_M 4\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  870 ********************\n",
      "processing  86 th loops---------------\n",
      "training set size: 870 unique(labels): [0 1] label counts: [442 428]\n",
      "Number of training examples  870\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       558\n",
      "           1       0.99      0.99      0.99       572\n",
      "\n",
      "    accuracy                           0.99      1130\n",
      "   macro avg       0.99      0.99      0.99      1130\n",
      "weighted avg       0.99      0.99      0.99      1130\n",
      "\n",
      "TP_H 428  TN_H 442  TP_M 568  TN_M 550  FP_M 8  FN_M 4\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  880 ********************\n",
      "processing  87 th loops---------------\n",
      "training set size: 880 unique(labels): [0 1] label counts: [446 434]\n",
      "Number of training examples  880\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       554\n",
      "           1       0.99      0.99      0.99       566\n",
      "\n",
      "    accuracy                           0.99      1120\n",
      "   macro avg       0.99      0.99      0.99      1120\n",
      "weighted avg       0.99      0.99      0.99      1120\n",
      "\n",
      "TP_H 434  TN_H 446  TP_M 562  TN_M 546  FP_M 8  FN_M 4\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  890 ********************\n",
      "processing  88 th loops---------------\n",
      "training set size: 890 unique(labels): [0 1] label counts: [452 438]\n",
      "Number of training examples  890\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       548\n",
      "           1       0.99      0.99      0.99       562\n",
      "\n",
      "    accuracy                           0.99      1110\n",
      "   macro avg       0.99      0.99      0.99      1110\n",
      "weighted avg       0.99      0.99      0.99      1110\n",
      "\n",
      "TP_H 438  TN_H 452  TP_M 558  TN_M 541  FP_M 7  FN_M 4\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  900 ********************\n",
      "processing  89 th loops---------------\n",
      "training set size: 900 unique(labels): [0 1] label counts: [456 444]\n",
      "Number of training examples  900\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:231: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  SVC(C=1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       544\n",
      "           1       0.99      0.99      0.99       556\n",
      "\n",
      "    accuracy                           0.99      1100\n",
      "   macro avg       0.99      0.99      0.99      1100\n",
      "weighted avg       0.99      0.99      0.99      1100\n",
      "\n",
      "TP_H 444  TN_H 456  TP_M 552  TN_M 537  FP_M 7  FN_M 4\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  910 ********************\n",
      "processing  90 th loops---------------\n",
      "training set size: 910 unique(labels): [0 1] label counts: [459 451]\n",
      "Number of training examples  910\n",
      "start gridsearch ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lujinghui\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:296: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n",
      "  FutureWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best parameters is  {'C': 0.1, 'kernel': 'linear'}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       541\n",
      "           1       0.99      0.99      0.99       549\n",
      "\n",
      "    accuracy                           0.99      1090\n",
      "   macro avg       0.99      0.99      0.99      1090\n",
      "weighted avg       0.99      0.99      0.99      1090\n",
      "\n",
      "TP_H 451  TN_H 459  TP_M 545  TN_M 534  FP_M 7  FN_M 4\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  920 ********************\n",
      "processing  91 th loops---------------\n",
      "training set size: 920 unique(labels): [0 1] label counts: [463 457]\n",
      "Number of training examples  920\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       537\n",
      "           1       0.99      0.99      0.99       543\n",
      "\n",
      "    accuracy                           0.99      1080\n",
      "   macro avg       0.99      0.99      0.99      1080\n",
      "weighted avg       0.99      0.99      0.99      1080\n",
      "\n",
      "TP_H 457  TN_H 463  TP_M 539  TN_M 531  FP_M 6  FN_M 4\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  930 ********************\n",
      "processing  92 th loops---------------\n",
      "training set size: 930 unique(labels): [0 1] label counts: [468 462]\n",
      "Number of training examples  930\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       532\n",
      "           1       0.99      0.99      0.99       538\n",
      "\n",
      "    accuracy                           0.99      1070\n",
      "   macro avg       0.99      0.99      0.99      1070\n",
      "weighted avg       0.99      0.99      0.99      1070\n",
      "\n",
      "TP_H 462  TN_H 468  TP_M 535  TN_M 526  FP_M 6  FN_M 3\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  940 ********************\n",
      "processing  93 th loops---------------\n",
      "training set size: 940 unique(labels): [0 1] label counts: [475 465]\n",
      "Number of training examples  940\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       525\n",
      "           1       0.99      0.99      0.99       535\n",
      "\n",
      "    accuracy                           0.99      1060\n",
      "   macro avg       0.99      0.99      0.99      1060\n",
      "weighted avg       0.99      0.99      0.99      1060\n",
      "\n",
      "TP_H 465  TN_H 475  TP_M 532  TN_M 519  FP_M 6  FN_M 3\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  950 ********************\n",
      "processing  94 th loops---------------\n",
      "training set size: 950 unique(labels): [0 1] label counts: [478 472]\n",
      "Number of training examples  950\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       522\n",
      "           1       0.99      0.99      0.99       528\n",
      "\n",
      "    accuracy                           0.99      1050\n",
      "   macro avg       0.99      0.99      0.99      1050\n",
      "weighted avg       0.99      0.99      0.99      1050\n",
      "\n",
      "TP_H 472  TN_H 478  TP_M 525  TN_M 516  FP_M 6  FN_M 3\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  960 ********************\n",
      "processing  95 th loops---------------\n",
      "training set size: 960 unique(labels): [0 1] label counts: [482 478]\n",
      "Number of training examples  960\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       518\n",
      "           1       0.99      0.99      0.99       522\n",
      "\n",
      "    accuracy                           0.99      1040\n",
      "   macro avg       0.99      0.99      0.99      1040\n",
      "weighted avg       0.99      0.99      0.99      1040\n",
      "\n",
      "TP_H 478  TN_H 482  TP_M 519  TN_M 513  FP_M 5  FN_M 3\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  970 ********************\n",
      "processing  96 th loops---------------\n",
      "training set size: 970 unique(labels): [0 1] label counts: [484 486]\n",
      "Number of training examples  970\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       516\n",
      "           1       0.99      0.99      0.99       514\n",
      "\n",
      "    accuracy                           0.99      1030\n",
      "   macro avg       0.99      0.99      0.99      1030\n",
      "weighted avg       0.99      0.99      0.99      1030\n",
      "\n",
      "TP_H 486  TN_H 484  TP_M 511  TN_M 511  FP_M 5  FN_M 3\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  980 ********************\n",
      "processing  97 th loops---------------\n",
      "training set size: 980 unique(labels): [0 1] label counts: [488 492]\n",
      "Number of training examples  980\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.99      0.99       512\n",
      "           1       0.99      1.00      0.99       508\n",
      "\n",
      "    accuracy                           0.99      1020\n",
      "   macro avg       0.99      0.99      0.99      1020\n",
      "weighted avg       0.99      0.99      0.99      1020\n",
      "\n",
      "TP_H 492  TN_H 488  TP_M 506  TN_M 507  FP_M 5  FN_M 2\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  990 ********************\n",
      "processing  98 th loops---------------\n",
      "training set size: 990 unique(labels): [0 1] label counts: [493 497]\n",
      "Number of training examples  990\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.99      0.99       507\n",
      "           1       0.99      1.00      0.99       503\n",
      "\n",
      "    accuracy                           0.99      1010\n",
      "   macro avg       0.99      0.99      0.99      1010\n",
      "weighted avg       0.99      0.99      0.99      1010\n",
      "\n",
      "TP_H 497  TN_H 493  TP_M 501  TN_M 502  FP_M 5  FN_M 2\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  1000 ********************\n",
      "processing  99 th loops---------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training set size: 1000 unique(labels): [0 1] label counts: [499 501]\n",
      "Number of training examples  1000\n",
      "best parameters is  SVC(C=0.1, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',\n",
      "    max_iter=10000, probability=True, random_state=2019, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.99      0.99       501\n",
      "           1       0.99      1.00      0.99       499\n",
      "\n",
      "    accuracy                           0.99      1000\n",
      "   macro avg       0.99      0.99      0.99      1000\n",
      "weighted avg       0.99      0.99      0.99      1000\n",
      "\n",
      "TP_H 501  TN_H 499  TP_M 497  TN_M 496  FP_M 5  FN_M 2\n",
      "******************** selection method  uncertainty ********************\n",
      "******************** num of training set  1010 ********************\n",
      "--- 390.5867164134979 seconds ---\n"
     ]
    }
   ],
   "source": [
    "import easydict\n",
    "from argparse import ArgumentParser\n",
    "\n",
    "# Run the main function\n",
    "# selection_method = 'certainty'\n",
    "selection_method = 'uncertainty'\n",
    "# selection_method = 'mostConfident'\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    \n",
    "    df = pd.DataFrame()\n",
    "    df_permutation = pd.DataFrame()\n",
    "    ### using different random seeds to seed the initial dataset\n",
    "    random_seeds = [i for i in range(1988,1989)]\n",
    "    \n",
    "    start_time = time.time()\n",
    "    for seed in random_seeds:\n",
    "        confusion_matrice = []\n",
    "        entropy = []\n",
    "        permutations = []\n",
    "        \n",
    "        for loop in range(100):\n",
    "            print('processing ',loop,'th loops---------------')\n",
    "\n",
    "            if loop==0:\n",
    "                \n",
    "                \n",
    "                parser = ArgumentParser()\n",
    "                args = parser.parse_known_args()[0]\n",
    "                args = easydict.EasyDict({\n",
    "                        \"n_seedsamples\":5,\n",
    "                        \"initial_random_seed\":seed,\n",
    "                        \"initial\": True,\n",
    "                        \"TEXT_emb\": ulti_representations,\n",
    "                        \"LABEL_emb\": labels,\n",
    "                        \"gridsearch\":True,\n",
    "                        \"loop\":loop,\n",
    "                        \"gridsearch_interval\":10\n",
    "\n",
    "                })\n",
    "\n",
    "                evaluation,y_pred_prob,y_pred,permutation,best_params_,dist = active_process(args)\n",
    "\n",
    "\n",
    "                permutation = sample_candidates(num_candidate=10,permutation=permutation,selection_method=selection_method,y_prob=y_pred_prob,y_pred=y_pred,dist=dist)\n",
    "                permutations.append(permutation)\n",
    "                \n",
    "                confusion_matrice.append(evaluation['Confusion matrix'])\n",
    "                \n",
    "                \n",
    "        \n",
    "\n",
    "            else:\n",
    "                \n",
    "#             \n",
    "\n",
    "                parser = ArgumentParser()\n",
    "                args = parser.parse_known_args()[0]\n",
    "                args = easydict.EasyDict({\n",
    "                        \"n_seedsamples\":5,\n",
    "                        \"initial_random_seed\":seed,\n",
    "                        \"permutation\": permutation,\n",
    "                        \"initial\": False,\n",
    "                        \"TEXT_emb\": ulti_representations,\n",
    "                        \"LABEL_emb\": labels,\n",
    "                        \"best_params_\":best_params_,\n",
    "                        \"loop\":loop,\n",
    "                        \"gridsearch_interval\":10\n",
    "                })\n",
    "\n",
    "                evaluation,y_pred_prob,y_pred,permutation,best_params_,dist = active_process(args)\n",
    "\n",
    "\n",
    "                permutation = sample_candidates(num_candidate=10,permutation=permutation,selection_method=selection_method,y_prob=y_pred_prob,y_pred=y_pred,dist=dist)\n",
    "                permutations.append(permutation)\n",
    "                \n",
    "                confusion_matrice.append(evaluation['Confusion matrix'])\n",
    "                \n",
    "               \n",
    "                \n",
    "              \n",
    "            \n",
    "        df[seed] = [item[0] for item in confusion_matrice]\n",
    "        df.to_csv('./results/raw_%s_%s_%s_gridsearch_1.csv'%(text_rep,dataset,selection_method))\n",
    "\n",
    "\n",
    "        df_permutation[seed] = [permutation for permutation in permutations]\n",
    "\n",
    "        df_permutation.to_csv('./results/permutation_%s_%s_%s_result_1.csv'%(text_rep,dataset,selection_method))\n",
    "        \n",
    "        \n",
    "    print(\"--- %s seconds ---\" % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import easydict\n",
    "# from argparse import ArgumentParser\n",
    "\n",
    "# # Run the main function\n",
    "# # selection_method = 'certainty'\n",
    "# selection_method = 'uncertainty'\n",
    "# # selection_method = 'mostConfident'\n",
    "\n",
    "\n",
    "# if __name__ == \"__main__\":\n",
    "    \n",
    "#     df = pd.DataFrame()\n",
    "#     df_permutation = pd.DataFrame()\n",
    "# #     random_seeds = [1989,1990,1991,1992,1993,1994,1995,1996,1997]\n",
    "#     random_seeds = [i for i in range(1988,1998)]\n",
    "    \n",
    "#     start_time = time.time()\n",
    "#     for seed in random_seeds:\n",
    "#         confusion_matrice = []\n",
    "#         entropy = []\n",
    "#         permutations = []\n",
    "        \n",
    "#         for loop in range(100):\n",
    "#             print('processing ',loop,'th loops---------------')\n",
    "\n",
    "#             if loop==0 :\n",
    "                \n",
    "#                 dir_neg = dataset + '_neg.csv'\n",
    "#                 dir_pos = dataset + '_pos.csv'\n",
    "#                 representations_neg = genfromtxt('./%s_data/'%(text_rep)+dir_neg, delimiter=',')\n",
    "#                 representations_pos = genfromtxt('./%s_data/'%(text_rep)+dir_pos, delimiter=',')\n",
    "#                 ulti_representations = np.concatenate((representations_neg,representations_pos),axis=0)\n",
    "#                 labels = np.array([0]*len(representations_neg)+[1]*len(representations_pos))\n",
    "       \n",
    "#                 parser = ArgumentParser()\n",
    "#                 args = parser.parse_known_args()[0]\n",
    "#                 args = easydict.EasyDict({\n",
    "#                         \"n_seedsamples\":5,\n",
    "#                         \"initial_random_seed\":seed,\n",
    "#                         \"initial\": True,\n",
    "#                         \"TEXT_emb\": ulti_representations,\n",
    "#                         \"LABEL_emb\": labels,\n",
    "#                         \"gridsearch\":True,\n",
    "#                         \"loop\":loop,\n",
    "#                         \"gridsearch_interval\":10\n",
    "\n",
    "#                 })\n",
    "\n",
    "#                 evaluation,y_pred_prob,y_pred,permutation,best_params_,dist = active_process(args)\n",
    "\n",
    "\n",
    "#                 permutation = sample_candidates(num_candidate=10,permutation=permutation,selection_method=selection_method,y_prob=y_pred_prob,y_pred=y_pred,dist=dist)\n",
    "#                 permutations.append(permutation)\n",
    "                \n",
    "#                 confusion_matrice.append(evaluation['Confusion matrix'])\n",
    "                \n",
    "                \n",
    "        \n",
    "\n",
    "#             else:\n",
    "                \n",
    "               \n",
    "#                 if loop==40:\n",
    "#                     construct_train_set(dataset,permutation)\n",
    "#                     checkpoint, accs = fine_tuned.fine_tuned(loop,dataset,'roberta-base',32,1e-6)\n",
    "#                     inference.inference(dataset,checkpoint,loop)\n",
    "\n",
    "#                     dir_neg = dataset + '_tuned_neg_%s.csv'%(loop)\n",
    "#                     dir_pos = dataset + '_tuned_pos_%s.csv'%(loop)\n",
    "#                     representations_neg = genfromtxt('./%s_data/'%(text_rep)+dir_neg, delimiter=',')\n",
    "#                     representations_pos = genfromtxt('./%s_data/'%(text_rep)+dir_pos, delimiter=',')\n",
    "\n",
    "#                     ulti_representations = np.concatenate((representations_neg,representations_pos),axis=0)\n",
    "#                 elif loop%40==0 and loop>40:\n",
    "#                     construct_train_set(dataset,permutation)\n",
    "#                     checkpoint, accs = fine_tuned.fine_tuned(loop,dataset,checkpoint,64,1e-5)\n",
    "#                     inference.inference(dataset,checkpoint,loop)\n",
    "\n",
    "#                     dir_neg = dataset + '_tuned_neg_%s.csv'%(loop)\n",
    "#                     dir_pos = dataset + '_tuned_pos_%s.csv'%(loop)\n",
    "#                     representations_neg = genfromtxt('./%s_data/'%(text_rep)+dir_neg, delimiter=',')\n",
    "#                     representations_pos = genfromtxt('./%s_data/'%(text_rep)+dir_pos, delimiter=',')\n",
    "\n",
    "#                     ulti_representations = np.concatenate((representations_neg,representations_pos),axis=0)\n",
    "                    \n",
    "              \n",
    "\n",
    "#                 parser = ArgumentParser()\n",
    "#                 args = parser.parse_known_args()[0]\n",
    "#                 args = easydict.EasyDict({\n",
    "#                         \"n_seedsamples\":5,\n",
    "#                         \"initial_random_seed\":seed,\n",
    "#                         \"permutation\": permutation,\n",
    "#                         \"initial\": False,\n",
    "#                         \"TEXT_emb\": ulti_representations,\n",
    "#                         \"LABEL_emb\": labels,\n",
    "#                         \"best_params_\":best_params_,\n",
    "#                         \"loop\":loop,\n",
    "#                         \"gridsearch_interval\":10\n",
    "#                 })\n",
    "\n",
    "#                 evaluation,y_pred_prob,y_pred,permutation,best_params_,dist = active_process(args)\n",
    "\n",
    "\n",
    "#                 permutation = sample_candidates(num_candidate=10,permutation=permutation,selection_method=selection_method,y_prob=y_pred_prob,y_pred=y_pred,dist=dist)\n",
    "#                 permutations.append(permutation)\n",
    "                \n",
    "#                 confusion_matrice.append(evaluation['Confusion matrix'])\n",
    "                \n",
    "#         shutil.rmtree(\"./outputs/\")\n",
    "                \n",
    "                \n",
    "            \n",
    "#         df[seed] = [item[0] for item in confusion_matrice]\n",
    "#         df.to_csv('./results/raw_%s_%s_%s_gridsearch.csv'%(text_rep,dataset,selection_method))\n",
    "\n",
    "\n",
    "#         df_permutation[seed] = [permutation for permutation in permutations]\n",
    "\n",
    "#         df_permutation.to_csv('./results/permutation_%s_%s_%s_result.csv'%(text_rep,dataset,selection_method))\n",
    "        \n",
    "#     print(\"--- %s seconds ---\" % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}