All_Installs_3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d015ebf",
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division\n",
    "from IPython.display import clear_output\n",
    "from BashColors import C\n",
    "from TarfileFunctions import *\n",
    "\n",
    "import os, sys\n",
    "from os.path import *\n",
    "from time import perf_counter, perf_counter_ns\n",
    "\n",
    "contentPath = os.getcwd()\n",
    "genPath = join(contentPath, 'DataGenerator')\n",
    "testPath = join(contentPath, 'images')\n",
    "\n",
    "if not exists(genPath):\n",
    "    tff.extractTarfiles('DataGenerator3.tar.gz')\n",
    "if not exists(testPath):\n",
    "    tff.extractTarfiles('images.tar.gz')\n",
    "\n",
    "from time import perf_counter, sleep\n",
    "import concurrent.futures\n",
    "# %pip show beautifulsoup4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5591e55",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 231 modules installed at start\n",
    "import sys\n",
    "alreadyInstalledList=set([each.split('.')[0] for each in sys.modules.keys()])\n",
    "modul = sys.modules.keys()\n",
    "\n",
    "for idx, item in enumerate(alreadyInstalledList):\n",
    "    if item in sorted(alreadyInstalledList):\n",
    "        # print(f'{idx} {C.BIBlue}{item}{C.ColorOff}')\n",
    "        pass\n",
    "print(f'{idx} modules installed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba4e7775",
   "metadata": {},
   "outputs": [],
   "source": [
    "def printTime(strt, fin):\n",
    "    '''print execution times '''\n",
    "    totalTime = fin - strt\n",
    "    totalTime= round(totalTime, 0)\n",
    "    if totalTime < 60.0:\n",
    "        print(f'completed: {totalTime} second(s)')\n",
    "    elif totalTime >= 60.0:\n",
    "        minutes = totalTime//60\n",
    "        seconds = totalTime - (minutes * 60)\n",
    "        seconds = round(seconds, 2)\n",
    "        print(f'completed: {minutes} minutes {seconds} second(s)')\n",
    "\n",
    "def getTensorflow():\n",
    "    %pip install tensorflow\n",
    "    print()\n",
    "    sleep(1)\n",
    "    return f'installed: {C.BIBlue}tensorflow{C.ColorOff}'\n",
    "\n",
    "def getTFHub():\n",
    "    %pip install --upgrade tensorflow-hub\n",
    "    print()\n",
    "    sleep(1)\n",
    "    return f'installed: {C.BIBlue}tensorflow-hub{C.ColorOff}'\n",
    "    \n",
    "def getTFLite():\n",
    "    %pip install tflite-model-maker\n",
    "    print()\n",
    "    sleep(1)\n",
    "    return f'installed: {C.BIBlue}tflite-model-maker{C.ColorOff}'\n",
    "\n",
    "def getNumpy():\n",
    "    %pip install numpy\n",
    "    print()\n",
    "    sleep(1)\n",
    "    return f'installed: {C.BIBlue}numpy{C.ColorOff}'\n",
    "    \n",
    "def getCV2():\n",
    "    %pip install opencv-python-headless # works.\n",
    "    print()\n",
    "    sleep(1)\n",
    "    return f'installed: {C.BIBlue}opencv-python-headless{C.ColorOff}'\n",
    "    # %pip install opencv-contrib-python-headless\n",
    "\n",
    "def getMatplotlib():\n",
    "    %pip install matplotlib\n",
    "    print()\n",
    "    sleep(1)\n",
    "    return f'installed: {C.BIBlue}matplotlib{C.ColorOff}'\n",
    "    \n",
    "def getSciPi():\n",
    "    %pip install SciPy\n",
    "    print()\n",
    "    sleep(1)\n",
    "    return f'installed: {C.BIBlue}SciPy{C.ColorOff}'\n",
    "\n",
    "def getResults():\n",
    "    %pip install --upgrade results\n",
    "    print()\n",
    "    sleep(1)\n",
    "    return f'installed: {C.BIBlue}results{C.ColorOff}'\n",
    "\n",
    "def getBeautifulSoup():\n",
    "    ''' '''\n",
    "    %pip install beautifulsoup4\n",
    "    sleep(1)\n",
    "    return f'installed: {C.BIBlue}beautifulsoup4{C.ColorOff}'\n",
    "\n",
    "def get_htmlib():\n",
    "    '''USED WITH BeautifulSoup'''\n",
    "    %pip install html5lib\n",
    "    sleep(1)\n",
    "    return f'installed: {C.BIBlue}html5lib{C.ColorOff}'\n",
    "\n",
    "def get_lxml():\n",
    "    '''USED WITH BeautifulSoup'''\n",
    "    %pip install lxml\n",
    "    sleep(1)\n",
    "    return f'installed: {C.BIBlue}lxml{C.ColorOff}'\n",
    "print('ok')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c12a121e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from concurrent.futures import ThreadPoolExecutor\n",
    "\n",
    "start = perf_counter()\n",
    "with ThreadPoolExecutor(max_workers=2) as executor:\n",
    "    thread_1 = executor.submit(getTensorflow)\n",
    "    thread_1.add_done_callback\n",
    "    thread_2 = executor.submit(getTFHub)\n",
    "    thread_3 = executor.submit(getTFLite)\n",
    "    thread_4 = executor.submit(getNumpy)\n",
    "    thread_5 = executor.submit(getMatplotlib)\n",
    "    thread_6 = executor.submit(getCV2)\n",
    "    thread_7 = executor.submit(getSciPi)\n",
    "    thread_8 = executor.submit(getResults)\n",
    "    thread_9 = executor.submit(getBeautifulSoup)\n",
    "    thread_10 = executor.submit(get_htmlib)\n",
    "    thread_11 = executor.submit(get_lxml)\n",
    "finish = perf_counter()\n",
    "printTime(start,finish)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9616d67f",
   "metadata": {},
   "outputs": [],
   "source": [
    "printTime(start, finish)\n",
    "print(thread_1.result())\n",
    "print(thread_2.result())\n",
    "print(thread_3.result())\n",
    "print(thread_4.result())\n",
    "print(thread_5.result())\n",
    "print(thread_6.result())\n",
    "print(thread_7.result())\n",
    "print(thread_8.result())\n",
    "print(thread_9.result())\n",
    "print(thread_10.result())\n",
    "print(thread_11.result())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "760ab7f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%writefile CIFAR_Image.py\n",
    "# 02/07/2022\n",
    "class CIFAR_Image:\n",
    "    \"\"\" \"\"\"\n",
    "    import numpy as np\n",
    "    # def __init__(self, image=None, label=None):\n",
    "    def __init__(self):\n",
    "        \"\"\"init\"\"\"\n",
    "        # Dimensions of image for reconstruction - not necessary \n",
    "        # for this dataset, but some datasets may include images of \n",
    "        # varying sizes\n",
    "        self.cfi = CIFAR_Image\n",
    "        self.image = None\n",
    "        try: self.channels = image.shape[2]\n",
    "        except: pass\n",
    "        try: self.size = image.shape[:2]\n",
    "        except: pass\n",
    "        try: self.image = image.tobytes()\n",
    "        except: pass\n",
    "        try: self.label = None\n",
    "        except: pass\n",
    "        \n",
    "    def __iter__(self):\n",
    "         \"\"\"iter\"\"\"\n",
    "         for key in self.some_sequence:\n",
    "            yield (key, 'Value for {}'.format(key))\n",
    "\n",
    "    def getImage(self, image):\n",
    "        \"\"\"Returns the image as a numpy array. \"\"\"\n",
    "        \n",
    "        self.image = image\n",
    "        image = np.frombuffer(self.image, dtype=np.uint8)\n",
    "        return image.reshape(*self.size, self.channels)\n",
    "    \n",
    "    def persistImage(self, folder_path:str,url:str):\n",
    "        \"\"\"Saves an image to disk.\"\"\"\n",
    "        try:\n",
    "            image_content = requests.get(url).content\n",
    "        except Exception as e:\n",
    "            print(f\"ERROR - Could not download {url} - {e}\")\n",
    "\n",
    "        try:\n",
    "            image_file = io.BytesIO(image_content)\n",
    "            image = Image.open(image_file).convert('RGB')\n",
    "            file_path = os.path.join(folder_path,hashlib.sha1(image_content).hexdigest()[:10] + '.jpg')\n",
    "            with open(file_path, 'wb') as f:\n",
    "                image.save(f, \"JPEG\", quality=100) # default 85\n",
    "            print(f\"SUCCESS - saved {url} - as {file_path}\")\n",
    "        except Exception as e:\n",
    "            print(f\"ERROR - Could not save {url} - {e}\")\n",
    "# cfi = CIFAR_Image(image=None, label=None)\n",
    "cfi=CIFAR_Image()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "454f81f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from CIFAR_Image import *\n",
    "help(CIFAR_Image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b426393f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "from PIL import Image\n",
    "import hashlib\n",
    "\n",
    "def persist_image(folder_path:str,url:str):\n",
    "    try:\n",
    "        image_content = requests.get(url).content\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"ERROR - Could not download {url} - {e}\")\n",
    "\n",
    "    try:\n",
    "        image_file = io.BytesIO(image_content)\n",
    "        image = Image.open(image_file).convert('RGB')\n",
    "        file_path = os.path.join(folder_path,hashlib.sha1(image_content).hexdigest()[:10] + '.jpg')\n",
    "        with open(file_path, 'wb') as f:\n",
    "            image.save(f, \"JPEG\", quality=85)\n",
    "        print(f\"SUCCESS - saved {url} - as {file_path}\")\n",
    "    except Exception as e:\n",
    "        print(f\"ERROR - Could not save {url} - {e}\")\n",
    "print('ok')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "822a2f26",
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip show beautifulsoup4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26604268",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}