emojinet_eda.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/davidhou8791/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "import spacy\n",
    "from tqdm import tqdm\n",
    "import gensim\n",
    "# import warnings; warnings.simplefilter('ignore')\n",
    "import pickle\n",
    "\n",
    "import utils\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2389, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>definition</th>\n",
       "      <th>keywords</th>\n",
       "      <th>name</th>\n",
       "      <th>senses</th>\n",
       "      <th>shortcode</th>\n",
       "      <th>unicode</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Miscellaneous Symbols And Pictographs -&gt; Emoji...</td>\n",
       "      <td>The Woman Gesturing Not OK, Type-6 emoji is a ...</td>\n",
       "      <td>[dark skin tone, hand, forbidden, gesture, wom...</td>\n",
       "      <td>woman gesturing NO: dark skin tone</td>\n",
       "      <td>{'adjectives': [{'bn:00104562a': ['Contrary to...</td>\n",
       "      <td>None</td>\n",
       "      <td>U+1F645 U+1F3FF U+200D U+2640 U+FE0F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Miscellaneous Symbols And Pictographs -&gt; Emoji...</td>\n",
       "      <td>The Female Guard, Type-6 emoji is a sequence o...</td>\n",
       "      <td>[dark skin tone, woman, guard]</td>\n",
       "      <td>woman guard: dark skin tone</td>\n",
       "      <td>{'adjectives': [], 'verbs': [{'bn:00090041v': ...</td>\n",
       "      <td>None</td>\n",
       "      <td>U+1F482 U+1F3FF U+200D U+2640 U+FE0F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>None</td>\n",
       "      <td>The female version of the ?? Runner emoji. The...</td>\n",
       "      <td>[racing, running, woman, marathon]</td>\n",
       "      <td>woman running</td>\n",
       "      <td>{'adjectives': [{'bn:00109994a': ['Of advancin...</td>\n",
       "      <td>None</td>\n",
       "      <td>U+1F3C3 U+200D U+2640 U+FE0F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Miscellaneous Symbols And Pictographs -&gt; Emoji...</td>\n",
       "      <td>The Woman Doing Cartwheel, Type-3 emoji is a s...</td>\n",
       "      <td>[gymnastics, medium-light skin tone, woman, ca...</td>\n",
       "      <td>woman cartwheeling: medium-light skin tone</td>\n",
       "      <td>{'adjectives': [], 'verbs': [{'bn:00084605v': ...</td>\n",
       "      <td>None</td>\n",
       "      <td>U+1F938 U+1F3FC U+200D U+2640 U+FE0F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>None</td>\n",
       "      <td>The female version of the ?? Golfer emoji. The...</td>\n",
       "      <td>[woman, golf]</td>\n",
       "      <td>woman golfing</td>\n",
       "      <td>{'adjectives': [], 'verbs': [{'bn:00088979v': ...</td>\n",
       "      <td>None</td>\n",
       "      <td>U+1F3CC U+FE0F U+200D U+2640 U+FE0F</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            category  \\\n",
       "0  Miscellaneous Symbols And Pictographs -> Emoji...   \n",
       "1  Miscellaneous Symbols And Pictographs -> Emoji...   \n",
       "2                                               None   \n",
       "3  Miscellaneous Symbols And Pictographs -> Emoji...   \n",
       "4                                               None   \n",
       "\n",
       "                                          definition  \\\n",
       "0  The Woman Gesturing Not OK, Type-6 emoji is a ...   \n",
       "1  The Female Guard, Type-6 emoji is a sequence o...   \n",
       "2  The female version of the ?? Runner emoji. The...   \n",
       "3  The Woman Doing Cartwheel, Type-3 emoji is a s...   \n",
       "4  The female version of the ?? Golfer emoji. The...   \n",
       "\n",
       "                                            keywords  \\\n",
       "0  [dark skin tone, hand, forbidden, gesture, wom...   \n",
       "1                     [dark skin tone, woman, guard]   \n",
       "2                 [racing, running, woman, marathon]   \n",
       "3  [gymnastics, medium-light skin tone, woman, ca...   \n",
       "4                                      [woman, golf]   \n",
       "\n",
       "                                         name  \\\n",
       "0          woman gesturing NO: dark skin tone   \n",
       "1                 woman guard: dark skin tone   \n",
       "2                               woman running   \n",
       "3  woman cartwheeling: medium-light skin tone   \n",
       "4                               woman golfing   \n",
       "\n",
       "                                              senses shortcode  \\\n",
       "0  {'adjectives': [{'bn:00104562a': ['Contrary to...      None   \n",
       "1  {'adjectives': [], 'verbs': [{'bn:00090041v': ...      None   \n",
       "2  {'adjectives': [{'bn:00109994a': ['Of advancin...      None   \n",
       "3  {'adjectives': [], 'verbs': [{'bn:00084605v': ...      None   \n",
       "4  {'adjectives': [], 'verbs': [{'bn:00088979v': ...      None   \n",
       "\n",
       "                                unicode  \n",
       "0  U+1F645 U+1F3FF U+200D U+2640 U+FE0F  \n",
       "1  U+1F482 U+1F3FF U+200D U+2640 U+FE0F  \n",
       "2          U+1F3C3 U+200D U+2640 U+FE0F  \n",
       "3  U+1F938 U+1F3FC U+200D U+2640 U+FE0F  \n",
       "4   U+1F3CC U+FE0F U+200D U+2640 U+FE0F  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_json('data/emojis.json')\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "U+1F645\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🙅</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "U+1F3FF\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🏿</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "U+200D\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">‍</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "U+2640\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">♀</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "U+FE0F\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">️</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for u in df.iloc[0].unicode.split(' '):\n",
    "    print(u)\n",
    "    utils.display_emoji(u)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 30000/30000 [00:00<00:00, 76023.62it/s]\n"
     ]
    }
   ],
   "source": [
    "# With Word2vec\n",
    "\n",
    "vocab_size = 30000 # keep this reasonably small for now, we're going to be doing a search over the entire vocab\n",
    "model = gensim.models.KeyedVectors.load_word2vec_format('data/GoogleNews-vectors-negative300.bin', binary=True, limit=vocab_size)\n",
    "\n",
    "# Init blank english spacy nlp object\n",
    "nlp = spacy.blank('en')\n",
    "\n",
    "for i in tqdm(range(vocab_size)):\n",
    "    nlp.vocab.set_vector(model.index2word[i], model.vectors[i])\n",
    "# # Loop through range of all indexes, get words associated with each index.\n",
    "# # The words in the keys list will correspond to the order of the google embed matrix\n",
    "keys = [model.index2word[i] for i in range(vocab_size)]\n",
    "\n",
    "# # Set the vectors for our nlp object to the google news vectors\n",
    "nlp.vocab.vectors = spacy.vocab.Vectors(data=model.vectors, keys=keys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['</s>', 'in', 'for', 'that', 'is', 'on', '##', 'The', 'with', 'said']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(model.vocab.keys())[0:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Interesting: Cosine similarity between a word and the sum of two words is the same as the similarity between the word and the vector average of the two words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2389"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# With Glove.  Let's use word2vec for now.\n",
    "\n",
    "# nlp = spacy.load('en')\n",
    "\n",
    "# with open('data/glove.6B.100d.txt', 'r') as f:\n",
    "#     for line in tqdm(f, total=400000):\n",
    "#         parts = line.split()\n",
    "#         word = parts[0]\n",
    "#         vec = np.array([float(v) for v in parts[1:]], dtype='f')\n",
    "#         nlp.vocab.set_vector(word, vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2389/2389 [00:00<00:00, 5967.30it/s]\n"
     ]
    }
   ],
   "source": [
    "docs = [nlp(x) for x in tqdm(df.keywords.str.join(' '))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "primary_unicodes = [u[0] for u in df.unicode.str.split(' ')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1126"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(primary_unicodes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "woman gesturing NO: dark skin tone\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🙅</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "woman guard: dark skin tone\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">💂</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "woman running\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🏃</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "woman cartwheeling: medium-light skin tone\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🤸</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "woman golfing\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🏌</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    print(df.iloc[i]['name'])\n",
    "    utils.display_emoji(primary_unicodes[i])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "emoji_vectors = {}\n",
    "for i,u in enumerate(primary_unicodes):\n",
    "    emoji = chr(int(u[2:], 16))\n",
    "    emoji_vectors[emoji] = docs[i].vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1126"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(emoji_vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "unicode_vectors = {}\n",
    "for i,u in enumerate(primary_unicodes):\n",
    "    unicode_vectors[primary_unicodes[i][2:]] = docs[i].vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(emoji_vectors, open('data/emoji_vectors.p', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(unicode_vectors, open('data/unicode_vectors.p', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v_vectors = {model.index2word[i]: model.vectors[i] for i in range(len(model.vocab))}\n",
    "pickle.dump(w2v_vectors, open('data/w2v_vectors.p', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = pickle.load(open('data/emoji_vectors.p', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def get_similar_emoji(word_vector, n=5):\n",
    "    '''\n",
    "    returns doc index of most n most similar emoji\n",
    "    args:\n",
    "        word_vector: vector embedding of word\n",
    "    returns:\n",
    "        int index of most similar emoji if n == 1\n",
    "        (list) index of most similar emoji if n > 1\n",
    "    '''\n",
    "    indexes = np.argsort([-utils.cosine_similarity(word_vector, emoji.vector) for emoji in docs])\n",
    "    return indexes[0] if n == 1 else indexes[:n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "one-thirty\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/davidhou8791/anaconda3/lib/python3.6/site-packages/scipy/spatial/distance.py:698: RuntimeWarning: invalid value encountered in float_scalars\n",
      "  dist = 1.0 - uv / np.sqrt(uu * vv)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🕜</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "one o’clock\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🕐</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Japanese “not free of charge” button\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🈶</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "prohibited\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🚫</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nine-thirty\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🕤</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "v = nlp(u'the').vector\n",
    "for i in get_similar_emoji(v):\n",
    "    print(df.iloc[i]['name'])\n",
    "    utils.display_emoji(primary_unicodes[i])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "blowfish\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/davidhou8791/anaconda3/lib/python3.6/site-packages/scipy/spatial/distance.py:698: RuntimeWarning: invalid value encountered in float_scalars\n",
      "  dist = 1.0 - uv / np.sqrt(uu * vv)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🐡</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Pisces\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">♓</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "shark\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🦈</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tropical fish\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🐠</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "fish cake with swirl\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🍥</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "v = nlp(u'fish').vector\n",
    "for i in get_similar_emoji(v):\n",
    "    print(df.iloc[i]['name'])\n",
    "    utils.display_emoji(primary_unicodes[i])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eight-pointed star\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/davidhou8791/anaconda3/lib/python3.6/site-packages/scipy/spatial/distance.py:698: RuntimeWarning: invalid value encountered in float_scalars\n",
      "  dist = 1.0 - uv / np.sqrt(uu * vv)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">✴</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "sparkles\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">✨</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "white medium star\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">⭐</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "night with stars\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🌃</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "dotted six-pointed star\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🔯</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "v = nlp(u'star').vector\n",
    "for i in get_similar_emoji(v):\n",
    "    print(df.iloc[i]['name'])\n",
    "    utils.display_emoji(primary_unicodes[i])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clapper board\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sm_ferro54/anaconda3/lib/python3.6/site-packages/scipy/spatial/distance.py:698: RuntimeWarning: invalid value encountered in float_scalars\n",
      "  dist = 1.0 - uv / np.sqrt(uu * vv)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🎬</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "white medium star\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">⭐</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "sparkles\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">✨</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "eight-pointed star\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">✴</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "dizzy\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">💫</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "v = nlp(u'star').vector + nlp(u'movie').vector\n",
    "for i in get_similar_emoji(v):\n",
    "    print(df.iloc[i]['name'])\n",
    "    utils.display_emoji(primary_unicodes[i])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = list(nlp.vocab.strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_similar_words(emoji_vector, n=5):\n",
    "    '''\n",
    "    returns doc index of most n most similar words\n",
    "    args:\n",
    "        emoji_vector: vector embedding of emoji\n",
    "    returns:\n",
    "        int index of most similar word if n == 1\n",
    "        (list) index of most similar words if n > 1\n",
    "    '''\n",
    "    indexes = np.argsort([-utils.cosine_similarity(emoji_vector, nlp(word).vector) for word in vocab])[:n]\n",
    "    return indexes[0] if n == 1 else indexes[:n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n"
     ]
    }
   ],
   "source": [
    "v1 = nlp(u'star').vector\n",
    "v2 = emoji_vectors['⭐']\n",
    "print(utils.cosine_similarity(v1, v2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.12529227137565613\n"
     ]
    }
   ],
   "source": [
    "v1 = nlp(u'hello').vector\n",
    "v2 = emoji_vectors['⭐']\n",
    "print(utils.cosine_similarity(v1, v2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n"
     ]
    }
   ],
   "source": [
    "print(utils.cosine_similarity(v2, nlp.vocab[u'star'].vector))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sm_ferro54/anaconda3/lib/python3.6/site-packages/scipy/spatial/distance.py:698: RuntimeWarning: invalid value encountered in float_scalars\n",
      "  dist = 1.0 - uv / np.sqrt(uu * vv)\n",
      "100%|██████████| 5/5 [00:00<00:00, 1502.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1795\n",
      "star\n",
      "\n",
      "2910\n",
      "stars\n",
      "\n",
      "10017\n",
      "superstar\n",
      "\n",
      "19634\n",
      "superstars\n",
      "\n",
      "9351\n",
      "standout\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "v = emoji_vectors['⭐']\n",
    "for i in tqdm(get_similar_words(v)):\n",
    "    print(i)\n",
    "    print(vocab[i])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode(text):\n",
    "    '''\n",
    "    encodes text into emoji\n",
    "    args:\n",
    "        (list) text\n",
    "    returns:\n",
    "        (list) unicodes of emoji\n",
    "    '''\n",
    "    vectors = [nlp(word).vector for word in text.split(' ')]\n",
    "    return [primary_unicodes[get_similar_emoji(v, n=1)] for v in tqdm(vectors)] \n",
    "\n",
    "def decode(emoji):\n",
    "    '''\n",
    "    decodes emoji into text\n",
    "    args:\n",
    "        (list) emoji\n",
    "    returns:\n",
    "        (list) words\n",
    "    '''\n",
    "    vectors = [emoji_vectors[e] for e in emoji]\n",
    "    return [vocab[get_similar_words(v, n=1)] for v in tqdm(vectors)] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/2 [00:00<?, ?it/s]/home/sm_ferro54/anaconda3/lib/python3.6/site-packages/scipy/spatial/distance.py:698: RuntimeWarning: invalid value encountered in float_scalars\n",
      "  dist = 1.0 - uv / np.sqrt(uu * vv)\n",
      "100%|██████████| 2/2 [00:00<00:00,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "♀🏌"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "text = 'Woman golfing'\n",
    "encoding = encode(text)\n",
    "emoji = [chr(int(unicode[2:], 16)) for unicode in encoding]\n",
    "for e in emoji:\n",
    "    print(e, end='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/2 [00:00<?, ?it/s]/home/sm_ferro54/anaconda3/lib/python3.6/site-packages/scipy/spatial/distance.py:698: RuntimeWarning: invalid value encountered in float_scalars\n",
      "  dist = 1.0 - uv / np.sqrt(uu * vv)\n",
      "100%|██████████| 2/2 [00:09<00:00,  4.76s/it]\n"
     ]
    }
   ],
   "source": [
    "decoding = decode(emoji)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['woman', 'skin']\n"
     ]
    }
   ],
   "source": [
    "print(decoding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.19798541069030762"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "utils.cosine_similarity(nlp(text).vector, nlp(' '.join(decoding)).vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "woman scientist\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sm_ferro54/anaconda3/lib/python3.6/site-packages/scipy/spatial/distance.py:698: RuntimeWarning: invalid value encountered in float_scalars\n",
      "  dist = 1.0 - uv / np.sqrt(uu * vv)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">👩</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "man scientist\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">👨</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pot of food\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">🍲</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "woman scientist: dark skin tone\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">👩</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "woman scientist: light skin tone\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<font size=\"+2\">👩</font>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for i in get_similar_emoji(nlp('Berkeley').vector):\n",
    "    print(df.iloc[i]['name'])\n",
    "    utils.display_emoji(primary_unicodes[i])\n",
    "    print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}