00_C_DL_Image_Database_Randomization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IMAGE RANDOMIZATION (Training, Validation & Testing)\n",
    "### Code to generate the basic train/validation/testing image database for Keras  \n",
    "#### by Luis Soenksen\n",
    "#### Last Update: 01/08/2018"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ORIGINAL & CLAHE DATA IMAGE RANDOMIZATION (copies files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPLITING PERCENTAGES ARE: (TRAINING=60.0% / VALIDATION=20.0% / TESTING=20.0%)\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    " BASIC IMAGE DATABASE TRAIN/VALIDATION/TEST RANDOMIZATION CODE\n",
    " ---------------------------------\n",
    " by Luis R Soenksen\n",
    " Last Update: 2017/04/23\n",
    "\"\"\"\n",
    "import glob\n",
    "import cv2\n",
    "import os\n",
    "import shutil\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "import tensorflow as tf\n",
    "tf.Session(config=tf.ConfigProto(log_device_placement=True)) #To ensure activation of GPUs in TF Backend\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "\n",
    "\n",
    "#IMAGE RANDOMIZATION AND AUGMENTATION HELPER FUNCTIONS\n",
    "# Print iterations progress\n",
    "def printProgressBar (iteration, total, prefix = '', suffix = '', decimals = 1, length = 100, fill = '█'):\n",
    "    \"\"\"\n",
    "    Call in a loop to create terminal progress bar\n",
    "    @params:\n",
    "        iteration   - Required  : current iteration (Int)\n",
    "        total       - Required  : total iterations (Int)\n",
    "        prefix      - Optional  : prefix string (Str)\n",
    "        suffix      - Optional  : suffix string (Str)\n",
    "        decimals    - Optional  : positive number of decimals in percent complete (Int)\n",
    "        length      - Optional  : character length of bar (Int)\n",
    "        fill        - Optional  : bar fill character (Str)\n",
    "    \"\"\"\n",
    "    percent = (\"{0:.\" + str(decimals) + \"f}\").format(100 * (iteration / float(total)))\n",
    "    filledLength = int(length * iteration // total)\n",
    "    bar = fill * filledLength + '-' * (length - filledLength)\n",
    "    print('\\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix), end = '\\r')\n",
    "    # Print New Line on Complete\n",
    "    if iteration == total: \n",
    "        print()\n",
    "\n",
    "def data_randomization (inputpath, outputpath):\n",
    "    # Print Message specifying database\n",
    "    print('Randomized Spliting of database in:'+ inputpath)\n",
    "    \n",
    "    #Creation of required folders\n",
    "    if not os.path.isdir(outputpath):\n",
    "        os.mkdir(outputpath)\n",
    "        \n",
    "    for dirpath, dirnames, filenames in os.walk(inputpath):\n",
    "        structure = os.path.join(outputpath + 'train/', dirpath[len(inputpath):])\n",
    "        if not os.path.isdir(structure):\n",
    "            os.mkdir(structure)\n",
    "    \n",
    "    for dirpath, dirnames, filenames in os.walk(inputpath):\n",
    "        structure = os.path.join(outputpath + 'validation/', dirpath[len(inputpath):])\n",
    "        if not os.path.isdir(structure):\n",
    "            os.mkdir(structure)       \n",
    "            \n",
    "    for dirpath, dirnames, filenames in os.walk(inputpath):\n",
    "        structure = os.path.join(outputpath + 'test/', dirpath[len(inputpath):])\n",
    "        if not os.path.isdir(structure):\n",
    "            os.mkdir(structure)\n",
    "            \n",
    "    class_num = 0\n",
    "    all_file_dataframe = pd.DataFrame([])\n",
    "    all_file_list = []\n",
    "    all_train_set_df = pd.DataFrame([])\n",
    "    all_validate_set_df = pd.DataFrame([])\n",
    "    all_test_set_df = pd.DataFrame([])\n",
    "    \n",
    "    for directory in glob.iglob(inputpath + '*', recursive=True):\n",
    "        class_file_list=[]\n",
    "        df = pd.DataFrame([])\n",
    "        for filename in glob.iglob(directory + '/' +'*.png', recursive=True):\n",
    "            class_file_list.append(filename)\n",
    "        \n",
    "        class_num += 1\n",
    "        all_file_list.append(class_file_list)\n",
    "        df = pd.DataFrame({directory:class_file_list})\n",
    "        all_file_dataframe = pd.concat([all_file_dataframe, df], axis=1)\n",
    "        train, validate, test = np.split(df.sample(frac=1), [int(train_p*len(df)), int((train_p + validation_p)*len(df))])\n",
    "        all_train_set_df = pd.concat([all_train_set_df, train], axis=1)\n",
    "        all_validate_set_df = pd.concat([all_validate_set_df, validate], axis=1)\n",
    "        all_test_set_df = pd.concat([all_test_set_df, test], axis=1)\n",
    "    \n",
    "    print('Total Number of classes: '+ str(len(all_file_list)))\n",
    "    n = 0\n",
    "    l = sum([len(files) for r, d, files in os.walk(inputpath)])-1  \n",
    "    for dir_n in all_file_dataframe.columns:\n",
    "        #CREATE FULLY RANDOMIZED TRAINING SET\n",
    "        for train_file_n in list(all_train_set_df.loc[:, dir_n]):\n",
    "            if isinstance(train_file_n,str):\n",
    "                n += 1\n",
    "                printProgressBar(n , l, prefix = 'Progress:', suffix = 'Complete', length = 50)\n",
    "                shutil.copy2(train_file_n, train_file_n.replace(inputpath, outputpath + 'train/')) # Copy files to target filename is /data/test/img.png\n",
    "        \n",
    "        #CREATE FULLY RANDOMIZED VALIDATION SET\n",
    "        for val_file_n in list(all_validate_set_df.loc[:, dir_n]):\n",
    "            if isinstance(val_file_n,str):\n",
    "                n += 1\n",
    "                printProgressBar(n , l, prefix = 'Progress:', suffix = 'Complete', length = 50)\n",
    "                shutil.copy2(val_file_n, val_file_n.replace(inputpath, outputpath + 'validation/')) # Copy files to target filename is /data/test/img.png\n",
    "        #CREATE FULLY RANDOMIZED TESTING SET    \n",
    "        for test_file_n in list(all_test_set_df.loc[:, dir_n]):\n",
    "            if isinstance(test_file_n,str):\n",
    "                n += 1\n",
    "                printProgressBar(n , l, prefix = 'Progress:', suffix = 'Complete', length = 50)\n",
    "                shutil.copy2(test_file_n, test_file_n.replace(inputpath, outputpath + 'test/')) # Copy files to target filename is /data/test/img.png\n",
    "        \n",
    "    print('Total number of randomized files: '+ str(n))\n",
    "        \n",
    "        \n",
    "# SPLITING PERCENTAGES (TRAINING=60% / VALIDATION=20% / TESTING=20%)\n",
    "train_p = 0.6\n",
    "validation_p = 0.20\n",
    "test_p = (1.0 - train_p - validation_p)\n",
    "print('SPLITING PERCENTAGES ARE: (TRAINING=' + str(train_p*100) +'% / VALIDATION=' + str(validation_p*100) +'% / TESTING=' + str(test_p*100) + '%)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Randomized Spliting of database in:./data/single_lesion_database/original_data/\n",
      "Total Number of classes: 6\n",
      "Progress: |██████████████████████████████████████████████████| 100.0% Complete\n",
      "Total number of randomized files: 38277██████████████████████| 100.0% Complete\n"
     ]
    }
   ],
   "source": [
    "#BASE DATA RANDOMIZATION\n",
    "#Definition of folder tree structure for converted files for original database and Randomization\n",
    "inputpath = './data/single_lesion_database/original_data/'\n",
    "outputpath ='./data/single_lesion_database/original_data_randomized/'\n",
    "data_randomization(inputpath, outputpath)     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Randomized Spliting of database in:./data/single_lesion_database/clahe_data/\n",
      "Total Number of classes: 6\n",
      "Progress: |██████████████████████████████████████████████████| 100.0% Complete\n",
      "Total number of randomized files: 38277██████████████████████| 100.0% Complete\n"
     ]
    }
   ],
   "source": [
    "#BASE DATA RANDOMIZATION\n",
    "#Definition of folder tree structure for converted files for original database and Randomization\n",
    "inputpath = './data/single_lesion_database/clahe_data/'\n",
    "outputpath ='./data/single_lesion_database/clahe_data_randomized/'\n",
    "data_randomization(inputpath, outputpath) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "------------------"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}