binary_label_img_preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Prep the images in data_src_dir by copying them to test and validation directory to be consumped by keras flow_from_directory.\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import os\n",
    "import gc\n",
    "from shutil import copyfile\n",
    "from shutil import move\n",
    "import re\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "from utils.file import makedirs\n",
    "\n",
    "data_dir = 'D:/Downloads/amazon/'\n",
    "data_src_dir = data_dir + 'train-jpg/'\n",
    "\n",
    "# ['slash_burn', 'blooming', 'primary', 'conventional_mine', 'water', 'cultivation', 'artisinal_mine', 'habitation', 'bare_ground', 'blow_down', 'agriculture', 'road', 'selective_logging']\n",
    "target_label = 'habitation'\n",
    "\n",
    "preprocess_train_dir = data_dir + 'prep/' + target_label + '/train-jpg/'\n",
    "preprocess_validation_dir = data_dir + 'prep/' + target_label + '/validation-jpg/'\n",
    "\n",
    "validation_fraction = 0.2  # fraction of original training set to be used for validation set\n",
    "\n",
    "df = pd.read_csv(data_dir + 'train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# copy images to the positive/ and negative/ folder.  \n",
    "# this is a preprocessing step for using Keras ImageDataGenerator.flow_from_directory.\n",
    "\n",
    "def preprocessImageFileForIndex(index, target):\n",
    "    source_filename = df.iloc[index]['image_name'] + '.jpg'\n",
    "    #print(source_filename)\n",
    "    #print(df.iloc[index]['tags'])\n",
    "\n",
    "    words = df.iloc[index]['tags'].split()\n",
    "    if target in words:\n",
    "        copyfile(data_src_dir + source_filename, preprocess_train_dir + 'positive/'+ source_filename)\n",
    "    else:\n",
    "        copyfile(data_src_dir + source_filename, preprocess_train_dir + 'negative/'+ source_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "makedirs(preprocess_train_dir)\n",
    "makedirs(preprocess_train_dir + 'positive/')\n",
    "makedirs(preprocess_train_dir + 'negative/')\n",
    "makedirs(preprocess_validation_dir)\n",
    "makedirs(preprocess_validation_dir + 'positive/')\n",
    "makedirs(preprocess_validation_dir + 'negative/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "40479it [11:53, 56.77it/s]                                                                                             \n"
     ]
    }
   ],
   "source": [
    "for index, row in tqdm(df.iterrows(), miniters=1000):\n",
    "    preprocessImageFileForIndex(index, target_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Move randomized samples to validation-jpg folder\n",
    "def randomTrainValidationSplit(validation_fraction, train_dir, validation_dir):\n",
    "    list_files = os.listdir(train_dir)\n",
    "    random.shuffle(list_files) # In-place shuffle\n",
    "    validation_size = int(round(len(list_files) * validation_fraction, 0))\n",
    "    print('validation_size:' , validation_size)\n",
    "    validation_files = list_files[0:validation_size]\n",
    "    for curr_file in validation_files:\n",
    "        src = train_dir + curr_file\n",
    "        dest = validation_dir + curr_file\n",
    "        #print ('src: ' + src)\n",
    "        #print ('dest: ' + dest)\n",
    "        move(src, dest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('validation_size:', 732)\n",
      "('validation_size:', 7363)\n"
     ]
    }
   ],
   "source": [
    "randomTrainValidationSplit(validation_fraction, preprocess_train_dir + 'positive/', preprocess_validation_dir + 'positive/')\n",
    "randomTrainValidationSplit(validation_fraction, preprocess_train_dir + 'negative/', preprocess_validation_dir + 'negative/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preprocessing complete for habitation\n"
     ]
    }
   ],
   "source": [
    "print('preprocessing complete for ' + target_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}