Dimension_Stability.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Importing libraries\n",
    "import pickle\n",
    "import nltk\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.test.utils import common_texts\n",
    "import sklearn.manifold\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "import multiprocessing\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from gensim.models.doc2vec import Doc2Vec, TaggedDocument\n",
    "stop_words = set(stopwords.words('english'))\n",
    "ps = PorterStemmer()\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from collections import Counter\n",
    "import matplotlib.pyplot as plt \n",
    "import seaborn as sns\n",
    "import time\n",
    "import collections\n",
    "from scipy import stats \n",
    "import numpy as np "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open(\"wiki.txt\", \"r\",errors='ignore').read() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus is now 94298560 characters long\n"
     ]
    }
   ],
   "source": [
    "print(\"Corpus is now {0} characters long\".format(len(file)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Split the corpus into sentences\n",
    "tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "raw_sentences = tokenizer.tokenize(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#convert into a list of words\n",
    "#remove unnecessary, split into words, no hyphens\n",
    "#list of words\n",
    "def sentence_to_wordlist(raw):\n",
    "    clean=re.sub('[\\n]',' ',raw.lower())\n",
    "    clean = re.sub(\"[^a-zA-Z]\",\" \", raw.lower())\n",
    "    words = clean.split()\n",
    "    #words = [ps.stem(w) for w in words if not w in stop_words]  \n",
    "    words = [w for w in words if not w in stop_words] \n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sentence where each word is tokenized\n",
    "sentences = []\n",
    "for raw_sentence in raw_sentences:\n",
    "    if len(raw_sentence) > 0:\n",
    "        sentences.append(sentence_to_wordlist(raw_sentence))\n",
    "#print(sentences[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The corpus contains 7,976,868 tokens\n"
     ]
    }
   ],
   "source": [
    "token_count = sum([len(sentence) for sentence in sentences])\n",
    "print(\"The corpus contains {0:,} tokens\".format(token_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getWordList(sentences,minCount):\n",
    "    wordList = []\n",
    "    count = Counter()\n",
    "    for sentence in sentences:\n",
    "        for word in sentence:\n",
    "            count[word] += 1\n",
    "    for word,c in count.items():\n",
    "        if c >= minCount:\n",
    "            wordList.append(word)\n",
    "    return wordList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "WordList=getWordList(sentences,minCount=5) # for min frequency of a word which should be 5\n",
    "#print(WordList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23079"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Vocab Size\n",
    "len(WordList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "62960183"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "import itertools\n",
    "ab=list(itertools.chain.from_iterable(sentences))\n",
    "new_str =' '.join(ab)  \n",
    "open('wiki_clean.txt', 'w').write(new_str) # writing clean text to new file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dimension 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model1 = Word2Vec(sentences,size=100,window=5,min_count=5,seed=42,workers=multiprocessing.cpu_count(),batch_words=1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "model2 = Word2Vec(sentences,size=100,window=5,min_count=5,seed=102,workers=multiprocessing.cpu_count(),batch_words=1) # the two embedding spaces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#word_frequency\n",
    "w2c = dict()\n",
    "for item in model1.wv.vocab:\n",
    "    w2c[item]=model1.wv.vocab[item].count\n",
    "w2cSorted=dict(sorted(w2c.items(), key=lambda x: x[1],reverse=True))\n",
    "# reverse output word and their frequency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Stability(word,similar1,similar2):    \n",
    "    overlap=0\n",
    "    for i in range(len(similar1)):\n",
    "        for j in range(len(similar2)):\n",
    "            if similar1[i][0] == similar2[j][0]:\n",
    "                overlap=overlap+1\n",
    "    return overlap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "163.65625 seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = time.process_time()\n",
    "with open('swiki.csv', 'a',errors='replace') as the_file:\n",
    "    the_file.write('Word,Stability(%),Stability(%) IU,Frequency\\n')\n",
    "    for word in WordList:\n",
    "        similar1=model1.wv.most_similar(word, topn=10)\n",
    "        similar2=model2.wv.most_similar(word, topn=10)\n",
    "        stab=Stability(word,similar1,similar2)\n",
    "        the_file.write(word+','+str(stab*10)+','+str(stab*100/(20-stab))+','+str(w2cSorted[word])+'\\n')\n",
    "print (time.process_time() - start_time, \"seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('swiki.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "738446.6209487107\n"
     ]
    }
   ],
   "source": [
    "df['Stability(%) IU'] = pd.to_numeric(df['Stability(%) IU'])\n",
    "s = df['Stability(%) IU'].sum()\n",
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dimension 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "model3 = Word2Vec(sentences,size=300,window=5,min_count=5,seed=42,workers=multiprocessing.cpu_count(),batch_words=1) # the two embedding spaces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "model4 = Word2Vec(sentences,size=300,window=5,min_count=5,seed=102,workers=multiprocessing.cpu_count(),batch_words=1) # the two embedding spaces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2c = dict()\n",
    "for item in model3.wv.vocab:\n",
    "    w2c[item]=model3.wv.vocab[item].count\n",
    "w2cSorted=dict(sorted(w2c.items(), key=lambda x: x[1],reverse=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "378.46875 seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = time.process_time()\n",
    "with open('swiki300.csv', 'a',errors='replace') as the_file:\n",
    "    the_file.write('Word,Stability(%),Stability(%) IU,Frequency\\n')\n",
    "    for word in WordList:\n",
    "        similar1=model3.wv.most_similar(word, topn=10)\n",
    "        similar2=model4.wv.most_similar(word, topn=10)\n",
    "        stab=Stability(word,similar1,similar2)\n",
    "        the_file.write(word+','+str(stab*10)+','+str(stab*100/(20-stab))+','+str(w2cSorted[word])+'\\n')\n",
    "print (time.process_time() - start_time, \"seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('swiki300.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word</th>\n",
       "      <th>Stability(%)</th>\n",
       "      <th>Stability(%) IU</th>\n",
       "      <th>Frequency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>action</td>\n",
       "      <td>90</td>\n",
       "      <td>81.818182</td>\n",
       "      <td>9901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>taken</td>\n",
       "      <td>100</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>8199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>parliament</td>\n",
       "      <td>80</td>\n",
       "      <td>66.666667</td>\n",
       "      <td>37171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>resolutions</td>\n",
       "      <td>90</td>\n",
       "      <td>81.818182</td>\n",
       "      <td>1076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>see</td>\n",
       "      <td>90</td>\n",
       "      <td>81.818182</td>\n",
       "      <td>11064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>minutes</td>\n",
       "      <td>90</td>\n",
       "      <td>81.818182</td>\n",
       "      <td>3638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>documents</td>\n",
       "      <td>90</td>\n",
       "      <td>81.818182</td>\n",
       "      <td>1715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>received</td>\n",
       "      <td>100</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>2023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>written</td>\n",
       "      <td>60</td>\n",
       "      <td>42.857143</td>\n",
       "      <td>2141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>statements</td>\n",
       "      <td>80</td>\n",
       "      <td>66.666667</td>\n",
       "      <td>2116</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Word  Stability(%)  Stability(%) IU  Frequency\n",
       "0       action            90        81.818182       9901\n",
       "1        taken           100       100.000000       8199\n",
       "2   parliament            80        66.666667      37171\n",
       "3  resolutions            90        81.818182       1076\n",
       "4          see            90        81.818182      11064\n",
       "5      minutes            90        81.818182       3638\n",
       "6    documents            90        81.818182       1715\n",
       "7     received           100       100.000000       2023\n",
       "8      written            60        42.857143       2141\n",
       "9   statements            80        66.666667       2116"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "808373.7499514589\n"
     ]
    }
   ],
   "source": [
    "df['Stability(%) IU'] = pd.to_numeric(df['Stability(%) IU'])\n",
    "s = df['Stability(%) IU'].sum()\n",
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dimension 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "model5 = Word2Vec(sentences,size=500,window=5,min_count=5,seed=42,workers=multiprocessing.cpu_count(),batch_words=1) # the two embedding spaces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "model6 = Word2Vec(sentences,size=500,window=5,min_count=5,seed=102,workers=multiprocessing.cpu_count(),batch_words=1) # the two embedding spaces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2c = dict()\n",
    "for item in model5.wv.vocab:\n",
    "    w2c[item]=model5.wv.vocab[item].count\n",
    "w2cSorted=dict(sorted(w2c.items(), key=lambda x: x[1],reverse=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "602.734375 seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = time.process_time()\n",
    "with open('swiki500.csv', 'a',errors='replace') as the_file:\n",
    "    the_file.write('Word,Stability(%),Stability(%) IU')\n",
    "    for word in WordList:\n",
    "        similar1=model5.wv.most_similar(word, topn=10)\n",
    "        similar2=model6.wv.most_similar(word, topn=10)\n",
    "        stab=Stability(word,similar1,similar2)\n",
    "        the_file.write(word+','+str(stab*10)+','+str(stab*100/(20-stab))+'\\n')\n",
    "print (time.process_time() - start_time, \"seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('swiki500.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55719.81974246512\n"
     ]
    }
   ],
   "source": [
    "#df['Stability(%) IU'] = pd.to_numeric(df['Stability(%) IU'])\n",
    "s = df['Stability(%) IUaction'].sum()\n",
    "print(s)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}