Load_Barbieri_Dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sm_ferro54/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import csv\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from nltk.corpus import PlaintextCorpusReader\n",
    "from nltk.tokenize import LineTokenizer\n",
    "folder = './data/Are_emojis_predictable_2017/'\n",
    "tf.enable_eager_execution()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _get_pd_df(folder,tag):\n",
    "    return pd.read_table(folder+tag,header =  None, names = ['input','target'])\n",
    "\n",
    "def get_tfdataset(folder, tag):\n",
    "    d = _get_pd_df(folder,tag)\n",
    "    return _df_to_tfds(d,list(d.columns))\n",
    "\n",
    "def _df_to_tfds(dataframe,names):\n",
    "    print(tf.cast(dataframe[names[0]].values,tf.string).shape)\n",
    "    print(tf.cast(dataframe[names[1]].values,tf.string).shape)\n",
    "    return tf.data.Dataset.from_tensor_slices(\n",
    "        (\n",
    "            tf.cast(dataframe[names[0]].values,tf.string),tf.cast(dataframe[names[1]].values,tf.string)\n",
    "        )\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(293448,)\n",
      "(293448,)\n",
      "(4718,)\n",
      "(4718,)\n",
      "(4809,)\n",
      "(4809,)\n"
     ]
    }
   ],
   "source": [
    "train = get_tfdataset(folder,'5_train')\n",
    "valid = get_tfdataset(folder,'5_validation')\n",
    "test = get_tfdataset(folder,'5_test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(tf.string, tf.string)\n",
      "(TensorShape([]), TensorShape([]))\n",
      "(tf.string, tf.string)\n",
      "(TensorShape([]), TensorShape([]))\n",
      "(tf.string, tf.string)\n",
      "(TensorShape([]), TensorShape([]))\n"
     ]
    }
   ],
   "source": [
    "print(train.output_types)\n",
    "print(train.output_shapes)\n",
    "print(valid.output_types)\n",
    "print(valid.output_shapes)\n",
    "print(test.output_types)\n",
    "print(test.output_shapes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[0] = data[0].str.replace('\\n',' n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: 0, dtype: object)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[0][data[0].str.find('\\n')>0] #This was causing some sentences to split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[0].to_csv('sentences.txt', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "293448 sentences.txt\r\n"
     ]
    }
   ],
   "source": [
    "!wc -l sentences.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/sm_ferro54/emoji_project'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "barb_corpus = PlaintextCorpusReader('./','sentences.txt',sent_tokenizer=LineTokenizer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "293448"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(barb_corpus.sents())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}