All_Installs_4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a68ecdae",
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division\n",
    "from IPython.display import clear_output\n",
    "from BashColors import C\n",
    "from TarfileFunctions import *\n",
    "\n",
    "import glob, json, os, sys\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '5'\n",
    "import pprint\n",
    "pp=pprint.PrettyPrinter()\n",
    "from os.path import *\n",
    "from time import perf_counter\n",
    "\n",
    "contentPath = os.getcwd()\n",
    "genPath = join(contentPath, 'DataGenerator')\n",
    "testPath = join(contentPath, 'images')\n",
    "jsonPath = join(contentPath, 'initialGlobList.json')\n",
    "modelPath=join(contentPath, 'Defcon4_fp16.tflite')\n",
    "imagePath=join(contentPath, '3b7d7d8a64.jpg')\n",
    "planetsPath=join(contentPath, 'planets')\n",
    "\n",
    "removePath=join(contentPath, 'Defcon4_mobilenet_v2_Dynamic.tflite')\n",
    "if exists(removePath):\n",
    "    os.remove(removePath)\n",
    "    print(f'{C.BIRed}{removePath}{C.ColorOff} removed\\n')\n",
    "    removePath=None\n",
    "\n",
    "if not exists(planetsPath):\n",
    "    tff.extractTarfiles('OriginalPlanets.tar.gz')\n",
    "\n",
    "if not exists(genPath):\n",
    "    tff.extractTarfiles('DataGenerator5.tar.gz')\n",
    "if not exists(testPath):\n",
    "    tff.extractTarfiles('images.tar.gz')\n",
    "    print()\n",
    "\n",
    "from time import perf_counter, sleep\n",
    "import concurrent.futures\n",
    "\n",
    "initialGlobList=glob.glob('**', recursive=True)\n",
    "if not 'initialGlobList.json' in initialGlobList:\n",
    "    initialGlobList.append('initialGlobList.json')\n",
    "\n",
    "with open(\"initialGlobList.json\", 'w') as f:\n",
    "    # indent=2 is not needed but makes the file human-readable\n",
    "    json.dump(initialGlobList, f, indent=1)\n",
    "\n",
    "# initialGlobList:list\n",
    "with open(\"initialGlobList.json\", 'r') as f:\n",
    "    initialGlobList = json.load(f)\n",
    "    # pprint.pprint(initialGlobList)\n",
    "\n",
    "def listNewFiles(initial=initialGlobList, deleteNew=False):\n",
    "    currentFilesGlob=glob.glob('**', recursive=True)\n",
    "    print(f'{C.BICyan}New files...{C.ColorOff}')\n",
    "    if len(initial) == len(currentFilesGlob):\n",
    "        print(f'{C.BIRed}No new files.\\nNothing to see here')\n",
    "    for fil in currentFilesGlob:\n",
    "        if not fil in initial:\n",
    "            if isdir(fil):\n",
    "                print(f'{C.BIBlue}{fil}')\n",
    "                if deleteNew:\n",
    "                    shutil.rmtree(fil)\n",
    "            elif isfile(fil):\n",
    "                print(f'{C.ColorOff}{fil}')\n",
    "                if deleteNew:\n",
    "                    os.remove(fil)\n",
    "                    \n",
    "listNewFiles()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "693896cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -U pip\n",
    "import pip\n",
    "print(pip.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25a8bb1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 233 modules installed at start\n",
    "# from BashColors import C\n",
    "import sys\n",
    "def listModules(silent=True):\n",
    "    alreadyInstalledSet=set([each.split('.')[0] for each in sys.modules.keys()])\n",
    "    modul = sys.modules.keys()\n",
    "    for idx, item in sorted(enumerate(alreadyInstalledSet)):\n",
    "        if item in sorted(alreadyInstalledSet):\n",
    "            if not silent:\n",
    "                print(f'{idx} {C.BIBlue}{item}{C.ColorOff}')\n",
    "    print(f'{idx} modules installed')\n",
    "listModules(silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7dcd36f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"initialGlobList.json\", 'r') as f:\n",
    "    initialGlobList = json.load(f)\n",
    "    f.close()\n",
    "    fileCount=0\n",
    "    for fil in initialGlobList:\n",
    "        fileCount+=1\n",
    "        # print(fil)\n",
    "print(f'{fileCount}{C.BIBlue} files in contentPath')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "985ee2bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -q -U pip\n",
    "\n",
    "def printTime(strt, fin):\n",
    "    '''print execution times '''\n",
    "    totalTime = fin - strt\n",
    "    totalTime= round(totalTime, 0)\n",
    "    if totalTime < 60.0:\n",
    "        print(f'{C.BIPurple}')\n",
    "        print(f'completed: {totalTime} second(s)')\n",
    "    elif totalTime >= 60.0:\n",
    "        minutes = totalTime//60\n",
    "        seconds = totalTime - (minutes * 60)\n",
    "        seconds = round(seconds, 2)\n",
    "        print(f'completed: {minutes} minutes {seconds} second(s)')\n",
    "\n",
    "def getTensorflow():\n",
    "    ''' '''\n",
    "    print(f'installing: {C.BIPurple}tensorflow{C.ColorOff}')\n",
    "    !pip install -q -U tensorflow\n",
    "    print(f'completed: {C.BIGreen}tensorflow{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}tensorflow{C.ColorOff}'\n",
    "\n",
    "def getTFX():\n",
    "    ''' '''\n",
    "    print(f'installing: {C.BIPurple}TFX{C.ColorOff}')\n",
    "    !pip install -q -U tfx\n",
    "    print(f'completed: {C.BIGreen}TFX{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}TFX{C.ColorOff}'\n",
    "\n",
    "def getApache():\n",
    "    ''' '''\n",
    "    print(f'installing: {C.BIPurple}apache-beam[interactive]{C.ColorOff}')\n",
    "    !pip install -q -U apache-beam[interactive] # TFX requirement\n",
    "    print(f'completed: {C.BIGreen}apache-beam[interactive]{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}apache-beam{C.ColorOff}'\n",
    "\n",
    "def getSnappy():\n",
    "    ''' '''\n",
    "    print(f'installing: {C.BIPurple}snappy{C.ColorOff}')\n",
    "    !pip install -q -U python-snappy\n",
    "    print(f'completed: {C.BIGreen}snappy{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}snappy{C.ColorOff}'\n",
    "\n",
    "def getTensorflowDatasets():\n",
    "    ''' '''\n",
    "    print(f'installing: {C.BIPurple}tensorflow-datasets{C.ColorOff}')\n",
    "    !pip install -q -U tensorflow-datasets\n",
    "    print(f'completed: {C.BIGreen}tensorflow-datasets{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}tensorflow-datasets{C.ColorOff}'\n",
    "\n",
    "def getTFLiteModelMaker():\n",
    "    ''' '''\n",
    "    # print(f'installing: {C.BIPurple}tflite-model-maker{C.ColorOff}')\n",
    "    !pip install -q -U tflite-model-maker\n",
    "    print(f'completed: {C.BIGreen}tflite-model-maker{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}tflite-model-maker{C.ColorOff}'\n",
    "\n",
    "def getTFHub():\n",
    "    ''' '''\n",
    "    print(f'installing: {C.BIPurple}tensorflow-hub{C.ColorOff}')\n",
    "    !pip install -q -U tensorflow-hub\n",
    "    print(f'completed: {C.BIGreen}tensorflow-hub{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}tensorflow-hub{C.ColorOff}'\n",
    "\n",
    "def getTFLiteSupport():\n",
    "    ''' '''\n",
    "    print(f'installing: {C.BIPurple}tflite-support{C.ColorOff}')\n",
    "    !pip install -q -U tflite-support\n",
    "    print(f'completed: {C.BIGreen}tflite-support{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}tflite-support{C.ColorOff}'\n",
    "\n",
    "def getTFLiteRuntime():\n",
    "    ''' '''\n",
    "    print(f'installing: {C.BIPurple}tflite-runtime{C.ColorOff}')\n",
    "    !pip install -q -U tflite-runtime\n",
    "    print(f'completed: {C.BIGreen}tflite-runtime{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}tflite-runtime{C.ColorOff}'\n",
    "\n",
    "def getNumpy():\n",
    "    ''' '''\n",
    "    print(f'installing: {C.BIPurple}numpy{C.ColorOff}')\n",
    "    !pip install -q -U numpy\n",
    "    print(f'completed: {C.BIGreen}numpy{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}numpy{C.ColorOff}'\n",
    "    \n",
    "def getCV2():\n",
    "    ''' '''\n",
    "    print(f'installing: {C.BIPurple}opencv-headless{C.ColorOff}')\n",
    "    !pip install -q -U opencv-python-headless # works.\n",
    "    print(f'completed: {C.BIGreen}opencv-headless{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}opencv-headless{C.ColorOff}'\n",
    "    # !pip install -q -U opencv-contrib-python-headless\n",
    "\n",
    "def getMatplotlib():\n",
    "    ''' '''\n",
    "    print(f'installing: {C.BIPurple}matplotlib{C.ColorOff}')\n",
    "    !pip install -q -U matplotlib\n",
    "    print(f'completed: {C.BIGreen}matplotlib{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}matplotlib{C.ColorOff}'\n",
    "    \n",
    "def getSciPi():\n",
    "    ''' '''\n",
    "    print(f'installing: {C.BIPurple}SciPy{C.ColorOff}')\n",
    "    !pip install -q -U SciPy\n",
    "    print(f'completed: {C.BIGreen}SciPy{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}SciPy{C.ColorOff}'\n",
    "\n",
    "def getSeaborn():\n",
    "    ''' '''\n",
    "    print(f'installing: {C.BIPurple}seaborn{C.ColorOff}')\n",
    "    !pip install -q -U seaborn\n",
    "    print(f'completed: {C.BIGreen}seaborn{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}seaborn{C.ColorOff}'\n",
    "\n",
    "def getResults():\n",
    "    ''' '''\n",
    "    print(f'installing: {C.BIPurple}results{C.ColorOff}')\n",
    "    !pip install -q -U results\n",
    "    print(f'completed: {C.BIGreen}results{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}results{C.ColorOff}'\n",
    "\n",
    "def getBeautifulSoup():\n",
    "    ''' '''\n",
    "    print(f'installing: {C.BIPurple}beautifulsoup4{C.ColorOff}')\n",
    "    !pip install -q -U beautifulsoup4\n",
    "    print(f'completed: {C.BIGreen}beautifulsoup4{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}beautifulsoup4{C.ColorOff}'\n",
    "\n",
    "def get_htmlib():\n",
    "    '''USED WITH BeautifulSoup'''\n",
    "    print(f'installing: {C.BIPurple}html5lib{C.ColorOff}')\n",
    "    !pip install -q -U html5lib\n",
    "    print(f'completed: {C.BIGreen}html5lib{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}html5lib{C.ColorOff}'\n",
    "\n",
    "def get_lxml():\n",
    "    '''USED WITH BeautifulSoup'''\n",
    "    print(f'installing: {C.BIPurple}lxml{C.ColorOff}')\n",
    "    !pip install -q -U lxml\n",
    "    print(f'completed: {C.BIGreen}lxml{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}lxml{C.ColorOff}'\n",
    "\n",
    "def getImutils():\n",
    "    '''Image processing utility '''\n",
    "    print(f'installing: {C.BIPurple}imutils{C.ColorOff}')\n",
    "    !pip install -q -U imutils\n",
    "    print(f'completed: {C.BIGreen}imutils{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}imutils{C.ColorOff}'\n",
    "\n",
    "def getPillow():\n",
    "    '''Image processing utility '''\n",
    "    print(f'installing: {C.BIPurple}pillow{C.ColorOff}')\n",
    "    !pip install -q -U pillow\n",
    "    print(f'completed: {C.BIGreen}pillow{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}pillow{C.ColorOff}'\n",
    "\n",
    "def getPSUtil():\n",
    "    '''system monitoring, profiling and limiting process resources'''\n",
    "    print(f'installing: {C.BIPurple}psutil{C.ColorOff}')\n",
    "    !pip install -q -U psutil\n",
    "    print(f'completed: {C.BIGreen}psutil{C.ColorOff}')\n",
    "    # return f'completed: {C.BIGreen}psutil{C.ColorOff}'\n",
    "print('ok')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16fc8b56",
   "metadata": {},
   "outputs": [],
   "source": [
    "from concurrent.futures import ThreadPoolExecutor\n",
    "\n",
    "start = perf_counter()\n",
    "with ThreadPoolExecutor(max_workers=4) as executor:\n",
    "    thread_0=executor.submit(getTFX)\n",
    "    # thread_1=executor.submit(getApache)\n",
    "    thread_2=executor.submit(getSnappy)\n",
    "    thread_3 = executor.submit(getCV2)\n",
    "    thread_4 = executor.submit(getMatplotlib)\n",
    "    thread_5 = executor.submit(getNumpy)\n",
    "    \n",
    "print()\n",
    "finish = perf_counter()\n",
    "print()\n",
    "printTime(start,finish)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3e96418",
   "metadata": {},
   "outputs": [],
   "source": [
    "printTime(start, finish)\n",
    "print(thread_0.result())\n",
    "# print(thread_1.result())\n",
    "print(thread_2.result())\n",
    "print(thread_3.result())\n",
    "print(thread_4.result())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22031087",
   "metadata": {},
   "outputs": [],
   "source": [
    "from BashColors import C\n",
    "import tensorflow as tf\n",
    "import tfx\n",
    "print(f'tensorflow version: {C.BIBlue}{tf.__version__}{C.ColorOff}')\n",
    "print(f'tfx version: {C.BIBlue}{tfx.__version__}{C.ColorOff}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}