NLP_task_8_Sentence_Semantic_Similarity.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# python -m spacy download en_core_web_lg\n",
    "# Download and installation successful\n",
    "# You can now load the model via spacy.load('en_core_web_lg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from spacy.vectors import Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bu model 780 mb büyüklüğünde\n",
    "nlp = spacy.load('en_core_web_lg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Bottle_name</th>\n",
       "      <th>Brand</th>\n",
       "      <th>Title</th>\n",
       "      <th>Review_Content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>The Yamazaki Single Malt Whisky - Distiller’s ...</td>\n",
       "      <td>Yamazaki</td>\n",
       "      <td>Overpriced dissapointment</td>\n",
       "      <td>Dull taste. High price. No finish. Over-hyped ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>The Yamazaki Single Malt Whisky - Distiller’s ...</td>\n",
       "      <td>Yamazaki</td>\n",
       "      <td>Delicious</td>\n",
       "      <td>Delicious! sugared red fruits and sweet with a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>The Yamazaki Single Malt Whisky - Distiller’s ...</td>\n",
       "      <td>Yamazaki</td>\n",
       "      <td>Good for beginners. i know cos i am a beginner</td>\n",
       "      <td>I am not a whisky expert but i really love the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>The Yamazaki Single Malt Whisky - Distiller’s ...</td>\n",
       "      <td>Yamazaki</td>\n",
       "      <td>Yamazaki Tutorial</td>\n",
       "      <td>This is a terrible Yamazaki.  Very young, unsh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>The Yamazaki Single Malt Whisky - Distiller’s ...</td>\n",
       "      <td>Yamazaki</td>\n",
       "      <td>Very Nice</td>\n",
       "      <td>First time and I like it - fresh but not thin ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                        Bottle_name     Brand  \\\n",
       "0           1  The Yamazaki Single Malt Whisky - Distiller’s ...  Yamazaki   \n",
       "1           2  The Yamazaki Single Malt Whisky - Distiller’s ...  Yamazaki   \n",
       "2           3  The Yamazaki Single Malt Whisky - Distiller’s ...  Yamazaki   \n",
       "3           4  The Yamazaki Single Malt Whisky - Distiller’s ...  Yamazaki   \n",
       "4           5  The Yamazaki Single Malt Whisky - Distiller’s ...  Yamazaki   \n",
       "\n",
       "                                            Title  \\\n",
       "0                       Overpriced dissapointment   \n",
       "1                                       Delicious   \n",
       "2  Good for beginners. i know cos i am a beginner   \n",
       "3                               Yamazaki Tutorial   \n",
       "4                                       Very Nice   \n",
       "\n",
       "                                      Review_Content  \n",
       "0  Dull taste. High price. No finish. Over-hyped ...  \n",
       "1  Delicious! sugared red fruits and sweet with a...  \n",
       "2  I am not a whisky expert but i really love the...  \n",
       "3  This is a terrible Yamazaki.  Very young, unsh...  \n",
       "4  First time and I like it - fresh but not thin ...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('./japanese_whisky_review.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1130, 5)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Brand</th>\n",
       "      <th>Title</th>\n",
       "      <th>Review_Content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Yamazaki</td>\n",
       "      <td>Overpriced dissapointment</td>\n",
       "      <td>Dull taste. High price. No finish. Over-hyped ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Yamazaki</td>\n",
       "      <td>Delicious</td>\n",
       "      <td>Delicious! sugared red fruits and sweet with a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Yamazaki</td>\n",
       "      <td>Good for beginners. i know cos i am a beginner</td>\n",
       "      <td>I am not a whisky expert but i really love the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Yamazaki</td>\n",
       "      <td>Yamazaki Tutorial</td>\n",
       "      <td>This is a terrible Yamazaki.  Very young, unsh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Yamazaki</td>\n",
       "      <td>Very Nice</td>\n",
       "      <td>First time and I like it - fresh but not thin ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Brand                                           Title  \\\n",
       "0  Yamazaki                       Overpriced dissapointment   \n",
       "1  Yamazaki                                       Delicious   \n",
       "2  Yamazaki  Good for beginners. i know cos i am a beginner   \n",
       "3  Yamazaki                               Yamazaki Tutorial   \n",
       "4  Yamazaki                                       Very Nice   \n",
       "\n",
       "                                      Review_Content  \n",
       "0  Dull taste. High price. No finish. Over-hyped ...  \n",
       "1  Delicious! sugared red fruits and sweet with a...  \n",
       "2  I am not a whisky expert but i really love the...  \n",
       "3  This is a terrible Yamazaki.  Very young, unsh...  \n",
       "4  First time and I like it - fresh but not thin ...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.drop(columns=['Unnamed: 0','Bottle_name'],inplace=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_vector = [nlp(x).vector for x in df[df['Brand']=='Yamazaki']['Review_Content'].values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_vector = np.stack( sentence_vector, axis=0 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(457, 300)"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_vector.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import TruncatedSVD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "svd = TruncatedSVD(n_components=10)\n",
    "svd_sentences = svd.fit_transform(sentence_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2.8889973e+00,  2.2345310e-01,  6.0537498e-02, ...,\n",
       "         6.9616008e-01,  4.2496905e-01, -2.4715742e-01],\n",
       "       [ 2.4281244e+00,  1.0796108e+00, -4.9393526e-01, ...,\n",
       "        -1.0113779e-01, -6.8672933e-02,  1.1561264e-01],\n",
       "       [ 3.4394896e+00, -2.1666817e-01,  9.4507840e-03, ...,\n",
       "        -1.5305886e-02,  3.2230671e-02, -3.0919972e-03],\n",
       "       ...,\n",
       "       [ 3.5944409e+00,  4.6319893e-01,  1.0962107e+00, ...,\n",
       "        -1.3598756e-01, -2.6959601e-01,  9.4364993e-03],\n",
       "       [ 2.9300835e+00,  4.1738659e-01, -5.3451645e-01, ...,\n",
       "        -3.3762324e-01, -1.1544983e-02,  1.3604306e-01],\n",
       "       [ 3.1513660e+00,  1.9204714e+00, -1.1487272e+00, ...,\n",
       "         4.7945631e-01, -9.1283709e-01, -6.3845873e-01]], dtype=float32)"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svd_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "cos_sim=cosine_similarity(svd_sentences,svd_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7      1.000000\n",
       "153    0.999779\n",
       "115    0.998892\n",
       "203    0.998343\n",
       "150    0.998272\n",
       "255    0.998188\n",
       "218    0.998163\n",
       "0      0.997799\n",
       "206    0.997755\n",
       "52     0.997485\n",
       "Name: 7, dtype: float32"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(cos_sim)[7].sort_values(ascending=False)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nice. I like some vanilla and this has the goods.  \n",
      "\n",
      "Solid drop.\n"
     ]
    }
   ],
   "source": [
    "print(df[df['Brand']=='Yamazaki']['Review_Content'][7],'\\n')\n",
    "print(df[df['Brand']=='Yamazaki']['Review_Content'][153])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_sentece_vec = np.stack([nlp(\"It is really good. I have never taste such a taste in my life. I will buy two more.\").vector])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_vector = np.append(sentence_vector,my_sentece_vec,axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(458, 300)"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_vector.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "svd_sentences = svd.fit_transform(sentence_vector)\n",
    "cos_sim=cosine_similarity(svd_sentences,svd_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "457    1.000000\n",
       "452    0.996916\n",
       "2      0.996753\n",
       "148    0.996623\n",
       "430    0.996566\n",
       "170    0.996176\n",
       "135    0.996080\n",
       "40     0.996025\n",
       "132    0.995705\n",
       "441    0.995704\n",
       "Name: 457, dtype: float32"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(cos_sim)[457].sort_values(ascending=False)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "It is really good. I have never taste such a taste in my life. I will buy two more. \n",
      "\n",
      "I was fortunate enough to win the dram lottery. This was a fantastic dram. Sophisticated, flavorful, lasting. I wish I would have won the bottle, but I feel fortunate to have had a chance to try it.\n"
     ]
    }
   ],
   "source": [
    "print(\"It is really good. I have never taste such a taste in my life. I will buy two more.\",'\\n')\n",
    "print(df[df['Brand']=='Yamazaki']['Review_Content'][452])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}