data_utils.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import cv2\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_folder = \"./flower_photos/\"\n",
    "resized_folder = \"./resized_flower_photos/\"\n",
    "\n",
    "all_labels = sorted([f for f in os.listdir(base_folder) if not f.startswith(\".\")])\n",
    "label_mapping = {label: i for i, label in enumerate(all_labels)}\n",
    "\n",
    "label_file = \"./label.txt\"\n",
    "with open(label_file, 'wb') as thefile:\n",
    "    thefile.write(\"\\n\".join(all_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_step():\n",
    "    def resize_image(image_path, resize_width=299, resize_height=299):\n",
    "        img = Image.open(image_path).convert(\"RGB\")\n",
    "        img = img.resize(size=(resize_width, resize_height), resample=Image.BICUBIC)\n",
    "        return img\n",
    "    \n",
    "    def resize_all_images(image_folder, save_folder, resize_width=299, resize_height=299):\n",
    "        all_label_images = [f for f in os.listdir(image_folder) if f.endswith(\".jpg\")]\n",
    "        all_resized_label_images = []\n",
    "        for f in all_label_images:\n",
    "            image_path = os.path.join(image_folder, f)\n",
    "            img = resize_image(image_path, resize_width, resize_height)\n",
    "            new_image_path = os.path.join(save_folder, f)\n",
    "            img.save(new_image_path, \"JPEG\")\n",
    "            all_resized_label_images.append(new_image_path)\n",
    "        return all_resized_label_images\n",
    "    \n",
    "    for label in all_labels:\n",
    "        label_folder = os.path.join(base_folder, label)\n",
    "        resize_label_folder = os.path.join(resized_folder, label)\n",
    "        print(resize_label_folder)\n",
    "        if not os.path.isdir(resize_label_folder):\n",
    "            os.makedirs(resize_label_folder)\n",
    "\n",
    "        all_label_images = resize_all_images(label_folder, resize_label_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocess_step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_train_test(image_list, fraction=0.1):\n",
    "    random.shuffle(image_list)\n",
    "    n = int(fraction*len(image_list))\n",
    "    return image_list[n:], image_list[:n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_test_data():\n",
    "    train_images = []\n",
    "    test_images = []\n",
    "    \n",
    "    for label in all_labels:\n",
    "        label_folder = os.path.join(resized_folder, label)\n",
    "        label_train_images = [f for f in os.listdir(label_folder) if f.endswith(\".jpg\")]\n",
    "        test_folder = os.path.join(label_folder, 'test')\n",
    "        if not os.path.isdir(test_folder):\n",
    "            os.makedirs(test_folder)\n",
    "            \n",
    "        label_test_images = [f for f in os.listdir(test_folder) if f.endswith(\".jpg\")]\n",
    "        \n",
    "        if not label_test_images:\n",
    "            label_train_images, label_test_images = split_train_test(label_train_images)\n",
    "            for img in label_test_images:\n",
    "                src = os.path.join(label_folder, img)\n",
    "                dst = os.path.join(test_folder, img)\n",
    "                os.rename(src, dst)\n",
    "        for img in label_train_images:\n",
    "            img_full_path = os.path.join(label_folder, img)\n",
    "            train_images.append([img_full_path, label_mapping[label]])\n",
    "        \n",
    "        for img in label_test_images:\n",
    "            img_full_path = os.path.join(test_folder, img)\n",
    "            test_images.append([img_full_path, label_mapping[label]])\n",
    "        print(label, len(label_train_images), len(label_test_images))\n",
    "        \n",
    "    return train_images, test_images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('daisy', 570, 63)\n",
      "('dandelion', 809, 89)\n",
      "('roses', 577, 64)\n",
      "('sunflowers', 630, 69)\n",
      "('tulips', 720, 79)\n"
     ]
    }
   ],
   "source": [
    "train_images, test_images = get_train_test_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_iter(data_list, batch_size=16, shuffle=True):\n",
    "    if shuffle:\n",
    "        random.shuffle(data_list)\n",
    "    \n",
    "    while len(data_list) > 0:\n",
    "        batch_data_list, data_list = data_list[:batch_size], data_list[batch_size:]\n",
    "        yield batch_data_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}