Revised DeepLearning with Pretrained Embedding.ipynb

{
    "cells": [
        {
            "metadata": {
                "tags": []
            },
            "cell_type": "code",
            "source": [
                "!pip install --upgrade numpy\n",
                "!pip install --upgrade pandas\n",
                "#!pip install pyspark==2.4.5\n",
                "!pip install -U scikit-learn\n",
                "#!pip install gensim\n",
                "!pip install -U keras\n",
                "# we want tensorflow 2.3\n",
                "!pip install --upgrade tensorflow\n"
            ],
            "execution_count": 3,
            "outputs": [
                {
                    "output_type": "stream",
                    "name": "stdout",
                    "text": "Requirement already up-to-date: numpy in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (1.19.2)\nRequirement already up-to-date: pandas in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (1.1.2)\nRequirement already satisfied, skipping upgrade: numpy>=1.15.4 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from pandas) (1.19.2)\nRequirement already satisfied, skipping upgrade: pytz>=2017.2 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from pandas) (2020.1)\nRequirement already satisfied, skipping upgrade: python-dateutil>=2.7.3 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from pandas) (2.8.1)\nRequirement already satisfied, skipping upgrade: six>=1.5 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from python-dateutil>=2.7.3->pandas) (1.15.0)\nRequirement already up-to-date: scikit-learn in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (0.23.2)\nRequirement already satisfied, skipping upgrade: joblib>=0.11 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from scikit-learn) (0.16.0)\nRequirement already satisfied, skipping upgrade: scipy>=0.19.1 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from scikit-learn) (1.5.2)\nRequirement already satisfied, skipping upgrade: threadpoolctl>=2.0.0 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from scikit-learn) (2.1.0)\nRequirement already satisfied, skipping upgrade: numpy>=1.13.3 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from scikit-learn) (1.19.2)\n"
                }
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 1,
            "metadata": {
                "tags": []
            },
            "outputs": [
                {
                    "output_type": "stream",
                    "name": "stdout",
                    "text": "Tensorflow version:  2.3.0\n"
                }
            ],
            "source": [
                "import tensorflow as tf\n",
                "print(\"Tensorflow version: \", tf.__version__)\n",
                "if not tf.__version__ == '2.3.0':\n",
                "    raise ValueError('please upgrade to TensorFlow 2.3, or restart your Kernel (Kernel->Restart & Clear Output)')\n",
                "from tensorflow import keras\n"
            ]
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": [
                "#%matplotlib inline\n",
                "#import matplotlib\n",
                "#import matplotlib.pyplot as plt\n",
                "from pprint import pprint\n",
                "from time import time\n",
                "import logging\n",
                "import numpy as np\n",
                "import pandas as pd\n",
                "import string\n",
                "import re\n",
                "from datetime import datetime\n",
                "#from packaging import version\n",
                "from ibm_botocore.client import Config\n",
                "import ibm_boto3\n",
                "pd.__version__\n",
                "#from sklearn.model_selection import train_test_split"
            ],
            "execution_count": 2,
            "outputs": [
                {
                    "output_type": "execute_result",
                    "data": {
                        "text/plain": "'1.1.2'"
                    },
                    "metadata": {},
                    "execution_count": 2
                }
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 8,
            "metadata": {
                "tags": []
            },
            "outputs": [
                {
                    "output_type": "stream",
                    "name": "stdout",
                    "text": "Processing c:\\users\\adamdavis\\appdata\\local\\pip\\cache\\wheels\\56\\14\\78\\c2fc4fe36e1e7198e4bde8707cf59b838a7be6859c631142f6\\ibm_cos_sdk-2.7.0-py2.py3-none-any.whl\nProcessing c:\\users\\adamdavis\\appdata\\local\\pip\\cache\\wheels\\bf\\bf\\71\\d4b460c36adadb88a24dfd26411c8b4dfa19459d15988c648d\\ibm_cos_sdk_s3transfer-2.7.0-py2.py3-none-any.whl\nCollecting jmespath<1.0.0,>=0.7.1\n  Using cached jmespath-0.10.0-py2.py3-none-any.whl (24 kB)\nProcessing c:\\users\\adamdavis\\appdata\\local\\pip\\cache\\wheels\\54\\a5\\95\\ac2068077b0f41719c2a69a03fdfbdd76ad02cdbb0984ad1a7\\ibm_cos_sdk_core-2.7.0-py2.py3-none-any.whl\nCollecting docutils<0.16,>=0.10\n  Using cached docutils-0.15.2-py3-none-any.whl (547 kB)\nRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from ibm-cos-sdk-core==2.7.0->ibm-cos-sdk) (2.8.1)\nCollecting requests<3.0,>=2.18\n  Using cached requests-2.24.0-py2.py3-none-any.whl (61 kB)\nRequirement already satisfied: six>=1.5 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from python-dateutil<3.0.0,>=2.1->ibm-cos-sdk-core==2.7.0->ibm-cos-sdk) (1.15.0)\nCollecting idna<3,>=2.5\n  Using cached idna-2.10-py2.py3-none-any.whl (58 kB)\nCollecting chardet<4,>=3.0.2\n  Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)\nCollecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1\n  Using cached urllib3-1.25.10-py2.py3-none-any.whl (127 kB)\nCollecting certifi>=2017.4.17\n  Using cached certifi-2020.6.20-py2.py3-none-any.whl (156 kB)\nInstalling collected packages: docutils, jmespath, idna, chardet, urllib3, certifi, requests, ibm-cos-sdk-core, ibm-cos-sdk-s3transfer, ibm-cos-sdk\nSuccessfully installed certifi-2020.6.20 chardet-3.0.4 docutils-0.15.2 ibm-cos-sdk-2.7.0 ibm-cos-sdk-core-2.7.0 ibm-cos-sdk-s3transfer-2.7.0 idna-2.10 jmespath-0.10.0 requests-2.24.0 urllib3-1.25.10\nNote: you may need to restart the kernel to use updated packages.\n"
                }
            ],
            "source": [
                "pip install ibm-cos-sdk"
            ]
        },
        {
            "metadata": {
                "tags": []
            },
            "cell_type": "code",
            "source": [
                "#Get our data from IBM Cloud\n",
                "\n",
                "# @hidden_cell\n",
                "# The following code contains the credentials for a file in your IBM Cloud Object Storage.\n",
                "# You might want to remove those credentials before you share your notebook.\n",
                "credentials_news = {\n",
                "    'IAM_SERVICE_ID': 'iam-ServiceId-32e8ee67-397c-4ff1-b69b-543172331f43',\n",
                "    'IBM_API_KEY_ID': 'Rx4FR4JSAueCnnIsoevsgYgOsuh8LCXtbkFpFpC0EmVU',\n",
                "    #'ENDPOINT': 'https://s3-api.us-geo.objectstorage.service.networklayer.com',\n",
                "    'ENDPOINT':'https://s3-api.us-geo.objectstorage.softlayer.net',\n",
                "    'IBM_AUTH_ENDPOINT': 'https://iam.cloud.ibm.com/oidc/token',\n",
                "    'BUCKET': 'advanceddatasciencecapstone-donotdelete-pr-tqabpnbxebk8rm',\n",
                "    'FILE': 'dfTrueFalseNews.pkl'\n",
                "}\n",
                "\n",
                "def download_file_cos(credentials,local_file_name,key):  \n",
                "    cos = ibm_boto3.client(service_name='s3',\n",
                "    ibm_api_key_id=credentials['IBM_API_KEY_ID'],\n",
                "    ibm_service_instance_id=credentials['IAM_SERVICE_ID'],\n",
                "    ibm_auth_endpoint=credentials['IBM_AUTH_ENDPOINT'],\n",
                "    config=Config(signature_version='oauth'),\n",
                "    endpoint_url=credentials['ENDPOINT'])\n",
                "    try:\n",
                "        res=cos.download_file(Bucket=credentials['BUCKET'],Key=key,Filename=local_file_name)\n",
                "    except Exception as e:\n",
                "        print(Exception, e)\n",
                "    else:\n",
                "        print('File Downloaded')\n",
                "\n",
                "def upload_file_cos(credentials,local_file_name,key):  \n",
                "    cos = ibm_boto3.client(service_name='s3',\n",
                "    ibm_api_key_id=credentials['IBM_API_KEY_ID'],\n",
                "    ibm_service_instance_id=credentials['IAM_SERVICE_ID'],\n",
                "    ibm_auth_endpoint=credentials['IBM_AUTH_ENDPOINT'],\n",
                "    config=Config(signature_version='oauth'),\n",
                "    endpoint_url=credentials['ENDPOINT'])\n",
                "    try:\n",
                "        res=cos.upload_file(Filename=local_file_name, Bucket=credentials['BUCKET'],Key=key)\n",
                "    except Exception as e:\n",
                "        print(Exception, e)\n",
                "    else:\n",
                "        print(' File Uploaded')\n",
                "        \n",
                "dfNews = download_file_cos(credentials_news, \"dfTrueFalseNews.pkl\", \"dfTrueFalseNews.pkl\")\n",
                "#dfNews = download_file_cos(credentials_news, \"conf.yml\", \"conf.yml\")\n",
                "#dfNews = download_file_cos(credentials_news, \"pipeline.py\", \"pipeline.py\")\n",
                "#dfNews = download_file_cos(credentials_news, \"text_preprocessing.py\", \"text_preprocessing.py\")\n",
                "#dfNews = download_file_cos(credentials_news, \"RNN_model.py\", \"RNN_model.py\")\n"
            ],
            "execution_count": 11,
            "outputs": [
                {
                    "output_type": "stream",
                    "name": "stdout",
                    "text": "File Downloaded\n"
                }
            ]
        },
        {
            "metadata": {
                "tags": []
            },
            "cell_type": "code",
            "source": [
                "dfNewsTemp = pd.read_pickle('dfTrueFalseNews.pkl')\n",
                "#dfTrueFalseNews_tokenized = pd.read_pickle('dfTrueFalseNews_tokenized.pkl')\n",
                "#dfNews['truthvalue'] = pd.Categorical(dfNews['truthvalue'])\n",
                "type(dfNewsTemp)\n",
                "\n",
                "x = dfNewsTemp['text'].values\n",
                "y = dfNewsTemp['truthvalue'].values\n",
                "print (dfNewsTemp.shape, dfNewsTemp.columns, '\\n', dfNewsTemp.dtypes, type(x), type(y))"
            ],
            "execution_count": 3,
            "outputs": [
                {
                    "output_type": "stream",
                    "name": "stdout",
                    "text": "(1126, 3) Index(['text', 'source', 'truthvalue'], dtype='object') \n text          object\nsource        object\ntruthvalue    object\ndtype: object <class 'numpy.ndarray'> <class 'numpy.ndarray'>\n"
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": [
                "# Get the pretrained GloVe embedding.\n",
                "#!curl http://nlp.stanford.edu/data/glove.6B.zip --output glove.6B.zip -L\n",
                "download_file_cos(credentials_news, \"glove.6B.zip\", \"glove.6B.zip\")\n",
                "#!ls -al\n",
                "!unzip glove.6B.zip glove.6B.300d.txt\n",
                "#!ls -al\n",
                "#upload_file_cos(credentials_news, \"glove.6B.zip\", \"glove.6B.zip\")\n",
                "!rm glove.6B.zip\n",
                "!ls -al"
            ],
            "execution_count": 8,
            "outputs": [
                {
                    "output_type": "error",
                    "ename": "KeyboardInterrupt",
                    "evalue": "",
                    "traceback": [
                        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
                        "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
                        "\u001b[1;32m<ipython-input-8-319207893e41>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# Get the pretrained GloVe embedding.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;31m#!curl http://nlp.stanford.edu/data/glove.6B.zip --output glove.6B.zip -L\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mdownload_file_cos\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcredentials_news\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"glove.6B.zip\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"glove.6B.zip\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[1;31m#!ls -al\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mget_ipython\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msystem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'unzip glove.6B.zip glove.6B.300d.txt'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;32m<ipython-input-7-bf67012742df>\u001b[0m in \u001b[0;36mdownload_file_cos\u001b[1;34m(credentials, local_file_name, key)\u001b[0m\n\u001b[0;32m     22\u001b[0m     endpoint_url=credentials['ENDPOINT'])\n\u001b[0;32m     23\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 24\u001b[1;33m         \u001b[0mres\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdownload_file\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mBucket\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcredentials\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'BUCKET'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mKey\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mFilename\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlocal_file_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     25\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     26\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mException\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\site-packages\\ibm_boto3\\s3\\inject.py\u001b[0m in \u001b[0;36mdownload_file\u001b[1;34m(self, Bucket, Key, Filename, ExtraArgs, Callback, Config)\u001b[0m\n\u001b[0;32m    168\u001b[0m     \"\"\"\n\u001b[0;32m    169\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mS3Transfer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mConfig\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtransfer\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 170\u001b[1;33m         return transfer.download_file(\n\u001b[0m\u001b[0;32m    171\u001b[0m             \u001b[0mbucket\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBucket\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mKey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfilename\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mFilename\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    172\u001b[0m             extra_args=ExtraArgs, callback=Callback)\n",
                        "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\site-packages\\ibm_boto3\\s3\\transfer.py\u001b[0m in \u001b[0;36mdownload_file\u001b[1;34m(self, bucket, key, filename, extra_args, callback)\u001b[0m\n\u001b[0;32m    305\u001b[0m             bucket, key, filename, extra_args, subscribers)\n\u001b[0;32m    306\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 307\u001b[1;33m             \u001b[0mfuture\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    308\u001b[0m         \u001b[1;31m# This is for backwards compatibility where when retries are\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    309\u001b[0m         \u001b[1;31m# exceeded we need to throw the same error from ibm_boto3 instead of\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\site-packages\\ibm_s3transfer\\futures.py\u001b[0m in \u001b[0;36mresult\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    107\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    108\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcancel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 109\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    110\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    111\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mcancel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\site-packages\\ibm_s3transfer\\futures.py\u001b[0m in \u001b[0;36mresult\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    104\u001b[0m             \u001b[1;31m# however if a KeyboardInterrupt is raised we want want to exit\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    105\u001b[0m             \u001b[1;31m# out of this and propogate the exception.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 106\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_coordinator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    107\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    108\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcancel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\site-packages\\ibm_s3transfer\\futures.py\u001b[0m in \u001b[0;36mresult\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    258\u001b[0m         \u001b[1;31m# possible value integer value, which is on the scale of billions of\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    259\u001b[0m         \u001b[1;31m# years...\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 260\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_done_event\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mMAXINT\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    261\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    262\u001b[0m         \u001b[1;31m# Once done waiting, raise an exception if present or return the\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\threading.py\u001b[0m in \u001b[0;36mwait\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    556\u001b[0m             \u001b[0msignaled\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_flag\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    557\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0msignaled\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 558\u001b[1;33m                 \u001b[0msignaled\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    559\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0msignaled\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    560\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\threading.py\u001b[0m in \u001b[0;36mwait\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    300\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m    \u001b[1;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    301\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 302\u001b[1;33m                 \u001b[0mwaiter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    303\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    304\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
                    ]
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": [
                "# Package for data wrangling\n",
                "import pandas as pd \n",
                "# Package for array math\n",
                "import numpy as np \n",
                "# Package for system path traversal\n",
                "import os\n",
                "# Package for working with dates\n",
                "from datetime import date\n",
                "# K fold analysis package\n",
                "from sklearn.model_selection import KFold\n",
                "# Import the main analysis pipeline\n",
                "from pipeline import Pipeline\n",
                "# Tensor creation class\n",
                "from text_preprocessing import TextToTensor\n"
            ],
            "execution_count": 4,
            "outputs": []
        },
        {
            "cell_type": "code",
            "execution_count": 5,
            "metadata": {},
            "outputs": [],
            "source": [
                "conf = {'k_fold': True,\n",
                "         'save_results': True,\n",
                "         'batch_size': 256,\n",
                "         'epochs': 7,\n",
                "         'max_len': 20}"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 33,
            "metadata": {},
            "outputs": [
                {
                    "output_type": "execute_result",
                    "data": {
                        "text/plain": "<module 'pipeline' from 'c:\\\\Users\\\\AdamDavis\\\\Documents\\\\Projects\\\\Data-Science-Advanced-Capstone\\\\pipeline.py'>"
                    },
                    "metadata": {},
                    "execution_count": 33
                }
            ],
            "source": [
                "import importlib\n",
                "import pipeline\n",
                "importlib.reload(pipeline)"
            ]
        },
        {
            "metadata": {
                "tags": []
            },
            "cell_type": "code",
            "source": [
                "# Modified, based on https://github.com/Eligijus112/twitter-genuine-tweets/blob/master/master.py\n",
                "# We're not going to use any stop words. The pretrained embedding has a lot of words.\n",
                "stop_words = []\n",
                "#try:\n",
                "#    stop_words = pd.read_csv('data/stop_words.txt', sep='\\n', header=None)[0].tolist()\n",
                "#except Exception as e:\n",
                "    # This exception indicates that the file is missing or is in a bad format\n",
                "#    print('Bad stop_words.txt file: {e}')\n",
                "\n",
                "# Reading the data\n",
                "#train = pd.read_csv('data/train.csv')[['text', 'target']]\n",
                "#test = pd.read_csv('data/test.csv')\n",
                "\n",
                "#from sklearn import model_selection\n",
                "#Train_X, Test_X, Train_Y, Test_Y = model_selection.train_test_split(dfNewsTemp['text'],dfNewsTemp['truthvalue'],test_size=0.1)\n",
                "\n",
                "# Creating the input for the pipeline\n",
                "X_train = dfNewsTemp['text'].tolist()\n",
                "Y_train = dfNewsTemp['truthvalue'].tolist()\n",
                "\n",
                "X_test = dfNewsTemp['text'].tolist()\n",
                "fit_history = []\n",
                "if conf.get('k_fold'):\n",
                "    kfold = KFold(n_splits=4)\n",
                "    acc = []\n",
                "    f1 = []\n",
                "    history = []\n",
                "    for train_index, test_index in kfold.split(X_train):\n",
                "        # Fitting the model and forecasting with a subset of data\n",
                "        k_results = Pipeline(\n",
                "            X_train=np.array(X_train)[train_index],\n",
                "            Y_train=np.array(Y_train)[train_index], \n",
                "            embed_path='glove.6B.300d.txt',\n",
                "            embed_dim=300,\n",
                "            X_test=np.array(X_train)[test_index],\n",
                "            Y_test=np.array(Y_train)[test_index],\n",
                "            max_len=conf.get('max_len'),\n",
                "            epochs=conf.get('epochs'),\n",
                "            batch_size=conf.get('batch_size')\n",
                "        )\n",
                "        # Saving the accuracy\n",
                "        acc += [k_results.acc]\n",
                "        f1 += [k_results.f1]\n",
                "        fit_history.append(k_results.fit_history)\n",
                "        history.append(k_results)\n",
                "        print(f'The accuracy score is: {acc[-1]}') \n",
                "        print(f'The f1 score is: {f1[-1]}') \n",
                "    print(f'Total mean accuracy is: {np.mean(acc)}')\n",
                "    print(f'Total mean f1 score is: {np.mean(f1)}')\n",
                "\n",
                "# Running the pipeline with all the data\n",
                "results = Pipeline(\n",
                "    X_train=X_train,\n",
                "    Y_train=Y_train, \n",
                "    embed_path='glove.6B.300d.txt',\n",
                "    embed_dim=300,\n",
                "    #stop_words=stop_words,\n",
                "    X_test=X_test,\n",
                "    #Y_test=Y_test,\n",
                "    max_len= None ,#   conf.get('max_len'),\n",
                "    epochs=conf.get('epochs'),\n",
                "    batch_size=conf.get('batch_size')\n",
                ")\n",
                "\n",
                "# Some sanity checks\n",
                "#good = [\"Fire in Vilnius! Where is the fire brigade??? #emergency\"]\n",
                "#bad = [\"Sushi or pizza? Life is hard :((\"]\n",
                "\n",
                "TextToTensor_instance = TextToTensor(\n",
                "tokenizer=results.tokenizer,\n",
                "max_len=conf.get('max_len')\n",
                ")\n",
                "\n",
                "# Converting to tensors\n",
                "good_nn = TextToTensor_instance.string_to_tensor(good)\n",
                "bad_nn = TextToTensor_instance.string_to_tensor(bad)\n",
                "\n",
                "# Forecasting\n",
                "p_good = results.model.predict(good_nn)[0][0]\n",
                "p_bad = results.model.predict(bad_nn)[0][0]\n",
                "\n",
                "print(f'Sentence: {good_nn} Score: {p_good}')\n",
                "print(f'Sentence: {bad_nn} Score: {p_bad}')\n",
                "\n",
                "# Saving the predictions\n",
                "test['prob_is_genuine'] = results.yhat\n",
                "test['target'] = [1 if x > 0.5 else 0 for x in results.yhat]\n",
                " \n",
                "# Saving the predictions to a csv file\n",
                "if conf.get('save_results'):\n",
                "    if not os.path.isdir('output'):\n",
                "        os.mkdir('output')    \n",
                "    test[['id', 'target']].to_csv(f'output/submission_{date.today()}.csv', index=False)\n",
                "\n",
                "results.save(\"revisedDeepLearningPretrainedEmbed.model\")\n",
                "upload_file_cos(credentials_news, \"revisedDeepLearningPretrainedEmbed.model\", \"revisedDeepLearningPretrainedEmbed.model\")\n",
                "\n",
                "import pickle\n",
                "filename = \"revisedDeepLearningPretrainedEmbed.history\"\n",
                "with open(filename, 'wb') as f:\n",
                "    pickle.dump(history, f)"
            ],
            "execution_count": 6,
            "outputs": [
                {
                    "output_type": "stream",
                    "name": "stdout",
                    "text": "Epoch 1/7\n4/4 [==============================] - ETA: 0s - loss: 0.6605WARNING:tensorflow:Model was constructed with shape (None, 20) for input Tensor(\"input_1:0\", shape=(None, 20), dtype=float32), but it was called on an input with incompatible shape (None, 1).\n"
                },
                {
                    "output_type": "error",
                    "ename": "UnimplementedError",
                    "evalue": " Cast string to float is not supported\n\t [[node functional_1/Cast (defined at c:\\Users\\AdamDavis\\Documents\\Projects\\Data-Science-Advanced-Capstone\\pipeline.py:57) ]] [Op:__inference_test_function_6748]\n\nFunction call stack:\ntest_function\n",
                    "traceback": [
                        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
                        "\u001b[1;31mUnimplementedError\u001b[0m                        Traceback (most recent call last)",
                        "\u001b[1;32m<ipython-input-6-1ea567796bf0>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     28\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mtrain_index\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_index\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mkfold\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     29\u001b[0m         \u001b[1;31m# Fitting the model and forecasting with a subset of data\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 30\u001b[1;33m         k_results = Pipeline(\n\u001b[0m\u001b[0;32m     31\u001b[0m             \u001b[0mX_train\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtrain_index\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     32\u001b[0m             \u001b[0mY_train\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mY_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtrain_index\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;32mc:\\Users\\AdamDavis\\Documents\\Projects\\Data-Science-Advanced-Capstone\\pipeline.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, X_train, Y_train, embed_path, embed_dim, stop_words, X_test, Y_test, max_len, epochs, batch_size)\u001b[0m\n\u001b[0;32m     55\u001b[0m             \u001b[0mmax_len\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmax_len\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     56\u001b[0m         )\n\u001b[1;32m---> 57\u001b[1;33m         self.fit_history = rnn.model.fit(\n\u001b[0m\u001b[0;32m     58\u001b[0m             \u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     59\u001b[0m             \u001b[0mY_train\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;32mc:\\Users\\AdamDavis\\Documents\\Projects\\Data-Science-Advanced-Capstone\\.venv\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    106\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_method_wrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    107\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_in_multi_worker_mode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 108\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    109\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    110\u001b[0m     \u001b[1;31m# Running inside `run_distribute_coordinator` already.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;32mc:\\Users\\AdamDavis\\Documents\\Projects\\Data-Science-Advanced-Capstone\\.venv\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m   1121\u001b[0m                 \u001b[0mmodel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1122\u001b[0m                 steps_per_execution=self._steps_per_execution)\n\u001b[1;32m-> 1123\u001b[1;33m           val_logs = self.evaluate(\n\u001b[0m\u001b[0;32m   1124\u001b[0m               \u001b[0mx\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mval_x\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1125\u001b[0m               \u001b[0my\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mval_y\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;32mc:\\Users\\AdamDavis\\Documents\\Projects\\Data-Science-Advanced-Capstone\\.venv\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    106\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_method_wrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    107\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_in_multi_worker_mode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 108\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    109\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    110\u001b[0m     \u001b[1;31m# Running inside `run_distribute_coordinator` already.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;32mc:\\Users\\AdamDavis\\Documents\\Projects\\Data-Science-Advanced-Capstone\\.venv\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mevaluate\u001b[1;34m(self, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing, return_dict)\u001b[0m\n\u001b[0;32m   1377\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mtrace\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTrace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'TraceContext'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgraph_type\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstep_num\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1378\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_test_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1379\u001b[1;33m               \u001b[0mtmp_logs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtest_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1380\u001b[0m               \u001b[1;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1381\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;32mc:\\Users\\AdamDavis\\Documents\\Projects\\Data-Science-Advanced-Capstone\\.venv\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    778\u001b[0m       \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    779\u001b[0m         \u001b[0mcompiler\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"nonXla\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 780\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    781\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    782\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;32mc:\\Users\\AdamDavis\\Documents\\Projects\\Data-Science-Advanced-Capstone\\.venv\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    844\u001b[0m               *args, **kwds)\n\u001b[0;32m    845\u001b[0m       \u001b[1;31m# If we did not create any variables the trace we have is good enough.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 846\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_concrete_stateful_fn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcanon_args\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcanon_kwds\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    847\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    848\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfn_with_cond\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minner_args\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0minner_kwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;32mc:\\Users\\AdamDavis\\Documents\\Projects\\Data-Science-Advanced-Capstone\\.venv\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[1;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1841\u001b[0m       \u001b[0;31m`\u001b[0m\u001b[0margs\u001b[0m\u001b[0;31m`\u001b[0m \u001b[1;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1842\u001b[0m     \"\"\"\n\u001b[1;32m-> 1843\u001b[1;33m     return self._call_flat(\n\u001b[0m\u001b[0;32m   1844\u001b[0m         [t for t in nest.flatten((args, kwargs), expand_composites=True)\n\u001b[0;32m   1845\u001b[0m          if isinstance(t, (ops.Tensor,\n",
                        "\u001b[1;32mc:\\Users\\AdamDavis\\Documents\\Projects\\Data-Science-Advanced-Capstone\\.venv\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1921\u001b[0m         and executing_eagerly):\n\u001b[0;32m   1922\u001b[0m       \u001b[1;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1923\u001b[1;33m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0m\u001b[0;32m   1924\u001b[0m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0;32m   1925\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n",
                        "\u001b[1;32mc:\\Users\\AdamDavis\\Documents\\Projects\\Data-Science-Advanced-Capstone\\.venv\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    543\u001b[0m       \u001b[1;32mwith\u001b[0m \u001b[0m_InterpolateFunctionError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    544\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcancellation_manager\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 545\u001b[1;33m           outputs = execute.execute(\n\u001b[0m\u001b[0;32m    546\u001b[0m               \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    547\u001b[0m               \u001b[0mnum_outputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_num_outputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;32mc:\\Users\\AdamDavis\\Documents\\Projects\\Data-Science-Advanced-Capstone\\.venv\\lib\\site-packages\\tensorflow\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     57\u001b[0m   \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 59\u001b[1;33m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[0;32m     60\u001b[0m                                         inputs, attrs, num_outputs)\n\u001b[0;32m     61\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
                        "\u001b[1;31mUnimplementedError\u001b[0m:  Cast string to float is not supported\n\t [[node functional_1/Cast (defined at c:\\Users\\AdamDavis\\Documents\\Projects\\Data-Science-Advanced-Capstone\\pipeline.py:57) ]] [Op:__inference_test_function_6748]\n\nFunction call stack:\ntest_function\n"
                    ]
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": [
                "\n",
                "\n",
                "import text_preprocessing"
            ],
            "execution_count": 8,
            "outputs": [
                {
                    "output_type": "stream",
                    "text": "Using TensorFlow backend.\n/opt/conda/envs/Python36/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n/opt/conda/envs/Python36/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n/opt/conda/envs/Python36/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n/opt/conda/envs/Python36/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n/opt/conda/envs/Python36/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n/opt/conda/envs/Python36/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
                    "name": "stderr"
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": [
                "pd.__version__"
            ],
            "execution_count": null,
            "outputs": []
        },
        {
            "cell_type": "code",
            "execution_count": 18,
            "metadata": {
                "tags": []
            },
            "outputs": [
                {
                    "output_type": "stream",
                    "name": "stdout",
                    "text": "Collecting keras\n  Downloading Keras-2.4.3-py2.py3-none-any.whl (36 kB)\nRequirement already satisfied, skipping upgrade: scipy>=0.14 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from keras) (1.5.2)\nRequirement already satisfied, skipping upgrade: numpy>=1.9.1 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from keras) (1.19.2)\nCollecting pyyaml\n  Downloading PyYAML-5.3.1-cp38-cp38-win_amd64.whl (219 kB)\nCollecting h5py\n  Downloading h5py-2.10.0-cp38-cp38-win_amd64.whl (2.5 MB)\nRequirement already satisfied, skipping upgrade: six in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from h5py->keras) (1.15.0)\nInstalling collected packages: pyyaml, h5py, keras\nSuccessfully installed h5py-2.10.0 keras-2.4.3 pyyaml-5.3.1\n"
                }
            ],
            "source": []
        },
        {
            "cell_type": "code",
            "execution_count": 20,
            "metadata": {
                "tags": []
            },
            "outputs": [
                {
                    "output_type": "stream",
                    "name": "stdout",
                    "text": "Collecting tensorflow\n  Downloading tensorflow-2.3.0-cp38-cp38-win_amd64.whl (342.5 MB)\nCollecting astunparse==1.6.3\n  Downloading astunparse-1.6.3-py2.py3-none-any.whl (12 kB)\nCollecting numpy<1.19.0,>=1.16.0\n  Downloading numpy-1.18.5-cp38-cp38-win_amd64.whl (12.8 MB)\nCollecting absl-py>=0.7.0\n  Downloading absl_py-0.10.0-py3-none-any.whl (127 kB)\nRequirement already satisfied, skipping upgrade: wrapt>=1.11.1 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from tensorflow) (1.12.1)\nCollecting tensorboard<3,>=2.3.0\n  Downloading tensorboard-2.3.0-py3-none-any.whl (6.8 MB)\nCollecting termcolor>=1.1.0\n  Downloading termcolor-1.1.0.tar.gz (3.9 kB)\nCollecting wheel>=0.26\n  Downloading wheel-0.35.1-py2.py3-none-any.whl (33 kB)\nCollecting scipy==1.4.1\n  Using cached scipy-1.4.1-cp38-cp38-win_amd64.whl (31.0 MB)\nCollecting opt-einsum>=2.3.2\n  Downloading opt_einsum-3.3.0-py3-none-any.whl (65 kB)\nRequirement already satisfied, skipping upgrade: h5py<2.11.0,>=2.10.0 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from tensorflow) (2.10.0)\nCollecting google-pasta>=0.1.8\n  Downloading google_pasta-0.2.0-py3-none-any.whl (57 kB)\nCollecting gast==0.3.3\n  Downloading gast-0.3.3-py2.py3-none-any.whl (9.7 kB)\nCollecting keras-preprocessing<1.2,>=1.1.1\n  Downloading Keras_Preprocessing-1.1.2-py2.py3-none-any.whl (42 kB)\nCollecting protobuf>=3.9.2\n  Downloading protobuf-3.13.0-py2.py3-none-any.whl (438 kB)\nCollecting grpcio>=1.8.6\n  Downloading grpcio-1.32.0-cp38-cp38-win_amd64.whl (2.6 MB)\nRequirement already satisfied, skipping upgrade: six>=1.12.0 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from tensorflow) (1.15.0)\nCollecting tensorflow-estimator<2.4.0,>=2.3.0\n  Downloading tensorflow_estimator-2.3.0-py2.py3-none-any.whl (459 kB)\nRequirement already satisfied, skipping upgrade: requests<3,>=2.21.0 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from tensorboard<3,>=2.3.0->tensorflow) (2.24.0)\nCollecting markdown>=2.6.8\n  Downloading Markdown-3.2.2-py3-none-any.whl (88 kB)\nCollecting tensorboard-plugin-wit>=1.6.0\n  Downloading tensorboard_plugin_wit-1.7.0-py3-none-any.whl (779 kB)\nCollecting werkzeug>=0.11.15\n  Using cached Werkzeug-1.0.1-py2.py3-none-any.whl (298 kB)\nRequirement already satisfied, skipping upgrade: setuptools>=41.0.0 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from tensorboard<3,>=2.3.0->tensorflow) (49.2.1)\nCollecting google-auth-oauthlib<0.5,>=0.4.1\n  Downloading google_auth_oauthlib-0.4.1-py2.py3-none-any.whl (18 kB)\nCollecting google-auth<2,>=1.6.3\n  Downloading google_auth-1.21.2-py2.py3-none-any.whl (93 kB)\nRequirement already satisfied, skipping upgrade: idna<3,>=2.5 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow) (2.10)\nRequirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow) (1.25.10)\nRequirement already satisfied, skipping upgrade: certifi>=2017.4.17 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow) (2020.6.20)\nRequirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in c:\\users\\adamdavis\\documents\\projects\\data-science-advanced-capstone\\.venv\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow) (3.0.4)\nCollecting requests-oauthlib>=0.7.0\n  Using cached requests_oauthlib-1.3.0-py2.py3-none-any.whl (23 kB)\nCollecting rsa<5,>=3.1.4; python_version >= \"3.5\"\n  Downloading rsa-4.6-py3-none-any.whl (47 kB)\nCollecting cachetools<5.0,>=2.0.0\n  Downloading cachetools-4.1.1-py3-none-any.whl (10 kB)\nCollecting pyasn1-modules>=0.2.1\n  Downloading pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)\nCollecting oauthlib>=3.0.0\n  Using cached oauthlib-3.1.0-py2.py3-none-any.whl (147 kB)\nCollecting pyasn1>=0.1.3\n  Downloading pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)\nUsing legacy 'setup.py install' for termcolor, since package 'wheel' is not installed.\nInstalling collected packages: wheel, astunparse, numpy, absl-py, protobuf, markdown, grpcio, tensorboard-plugin-wit, werkzeug, oauthlib, requests-oauthlib, pyasn1, rsa, cachetools, pyasn1-modules, google-auth, google-auth-oauthlib, tensorboard, termcolor, scipy, opt-einsum, google-pasta, gast, keras-preprocessing, tensorflow-estimator, tensorflow\n  Attempting uninstall: numpy\n    Found existing installation: numpy 1.19.2\n    Uninstalling numpy-1.19.2:\n      Successfully uninstalled numpy-1.19.2\nERROR: Could not install packages due to an EnvironmentError: [WinError 5] Access is denied: 'C:\\\\Users\\\\AdamDavis\\\\Documents\\\\Projects\\\\Data-Science-Advanced-Capstone\\\\.venv\\\\Lib\\\\site-packages\\\\~umpy\\\\.libs\\\\libopenblas.NOIJJG62EMASZI6NYURL6JBKM4EVBGM7.gfortran-win_amd64.dll'\nConsider using the `--user` option or check the permissions.\n\n"
                }
            ],
            "source": [
                "\n"
            ]
        }
    ],
    "metadata": {
        "kernelspec": {
            "name": "python_defaultSpec_1600696624532",
            "display_name": "Python 3.8.6 64-bit ('.venv': venv)",
            "language": "python"
        },
        "language_info": {
            "name": "python",
            "version": "3.8.6-candidate",
            "mimetype": "text/x-python",
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "pygments_lexer": "ipython3",
            "nbconvert_exporter": "python",
            "file_extension": ".py"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 1
}