Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "parser = { \"filename\" \"stemmer\": \"lemma\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "import numpy as np\n",
    "import lda\n",
    "import lda.datasets\n",
    "import csv\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from stop_words import get_stop_words\n",
    "from gensim import corpora, models\n",
    "\n",
    "from stemming.porter2 import stem\n",
    "from nltk.stem import *\n",
    "import unicodecsv\n",
    "import re\n",
    "import pyLDAvis.gensim\n",
    "import gensim\n",
    "\n",
    "import argparse\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    list_of_stemmer_choices = [\"none\", \"porter\", \"porter2\", \"lemma\"]\n",
    "    parser = argparse.ArgumentParser(description='run LDA on an input csv file.')\n",
    "    parser.add_argument('-i','--input',dest=\"filename\", help='input CSV file', required=True)\n",
    "    parser.add_argument('-s','--stemmer', help='pick stemmer', default=\"porter2\", choices=list_of_stemmer_choices)\n",
    "    parser.add_argument('-ni','--num_iter', help='number of iterations', default=\"1000\")\n",
    "    parser.add_argument('-twc','--topwords_count', help='number of top_words', default=\"8\")\n",
    "    args = parser.parse_args()\n",
    "_digits = re.compile('\\d')\n",
    "def contains_digits(d):\n",
    "    return bool(_digits.search(d))\n",
    "\n",
    "def process_tokens(tokens,stemmer):\n",
    "  tokens = [i for i in tokens if not i in en_stop and not contains_digits(i)]\n",
    "  if stemmer == 'porter':\n",
    "    stemmer = PorterStemmer()\n",
    "    tokens = [stemmer.stem(i) for i in tokens]\n",
    "  elif stemmer == 'porter2':\n",
    "    tokens = [stem(i) for i in tokens]\n",
    "  elif stemmer == 'lemma':\n",
    "    lemmatiser = WordNetLemmatizer()\n",
    "    tokens = [lemmatiser.lemmatize(i) for i in tokens]\n",
    "  return tokens\n",
    "\n",
    "\n",
    "if args.filename == \"sample\":\n",
    "  X = lda.datasets.load_reuters()\n",
    "  dictionary = lda.datasets.load_reuters_vocab()\n",
    "  titles = lda.datasets.load_reuters_titles()\n",
    "else:\n",
    "  f = open(args.filename)\n",
    "  reader = unicodecsv.reader(f, encoding='utf-8')\n",
    "  # csv_length = sum(1 for row in reader)\n",
    "  # f.seek(0) #reset reader position\n",
    "  identifiers = reader.next()\n",
    "  contents_idx = identifiers.index(\"contents\")\n",
    "  title_idx = identifiers.index(\"title\")\n",
    "\n",
    "  contents = [ row[contents_idx] for row in reader if row[contents_idx] ]\n",
    "\n",
    "  f.seek(0)\n",
    "  reader.next()\n",
    "  titles = [ row[title_idx] for row in reader if row[contents_idx] ]\n",
    "\n",
    "\n",
    "\n",
    "  texts = list()\n",
    "  tokenizer = RegexpTokenizer(r'\\w+')\n",
    "  en_stop = get_stop_words('en')\n",
    "  for idx,i in enumerate(contents):\n",
    "    if not idx % 10:\n",
    "      print \"INFO: Tokenizing articles <{}> \".format(idx)\n",
    "    raw = i.lower()\n",
    "    tokens = tokenizer.tokenize(raw)\n",
    "    texts.append(process_tokens(tokens, args.stemmer))\n",
    "    # print idx\n",
    "    # add tokens to list\n",
    "\n",
    "  print \"[DEBUG] Length of Texts : {}\".format(len(texts))\n",
    "  dictionary = corpora.Dictionary(texts)\n",
    "  corpus = [dictionary.doc2bow(text) for text in texts]\n",
    "\n",
    "  # X = np.zeros((len(contents), len(dictionary)), dtype=np.int)\n",
    "  # for idx,i in enumerate(corpus):\n",
    "  #   for j in i:\n",
    "  #     X[idx][j[0]] = j[1]\n",
    "\n",
    "\n",
    "lda = gensim.models.ldamodel.LdaModel(corpus, num_topics=10)\n",
    "pyLDAvis.gensim.prepare(lda, corpus, dictionary)\n",
    "# model = lda.LDA(n_topics=20, n_iter=int(args.num_iter), random_state=1)\n",
    "# model.fit(X)\n",
    "\n",
    "\n",
    "# n_top_words = int(args.topwords_count)\n",
    "# topic_word = model.topic_word_  # odel.components_ also works\n",
    "# for i, topic_dist in enumerate(topic_word):\n",
    "#   if args.filename == \"sample\":\n",
    "#     topic_words = np.array(dictionary)[np.argsort(topic_dist)][:-(n_top_words+1):-1]\n",
    "#   else:\n",
    "#     topic_words = [ dictionary[x] for x in np.array(dictionary)[np.argsort(topic_dist)][:-(n_top_words+1):-1] ]\n",
    "#   print u'Topic {}: {}'.format(i, ' '.join(topic_words))\n",
    "# doc_topic = model.doc_topic_\n",
    "# for i in range(10):\n",
    "#    print u'\"{} (top topic: {})\"'.format(titles[i], doc_topic[i].argmax())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}