DataETL.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 文字探勘之資料ETL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用requests.get"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n",
      "text/html; charset=utf-8\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "res = requests.get('http://www.appledaily.com.tw/realtimenews/section/new/1/')\n",
    "#print res.text\n",
    "print res.status_code \n",
    "print res.headers['content-type']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用requests.post"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "payload = {\n",
    "'StartStation':'977abb69-413a-4ccf-a109-0272c24fd490',\n",
    "'EndStation':'60831846-f0e4-47f6-9b5b-46323ebdcef7',\n",
    "'SearchDate':'2015/05/27',\n",
    "'SearchTime':'10:30',\n",
    "'SearchWay':'DepartureInMandarin'\n",
    "}\n",
    "res = requests.post('http://www.thsrc.com.tw/tw/TimeTable/SearchResult', data=payload)\n",
    "#print res.text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BeautifulSoup4 範例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Hello World This is link1 This is link2  \n"
     ]
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup \n",
    "html_sample = ' \\\n",
    "<html> \\\n",
    " <body> \\\n",
    " <h1 id=\"title\">Hello World</h1> \\\n",
    " <a href=\"#\" class=\"link\">This is link1</a> \\\n",
    " <a href=\"# link2\" class=\"link\">This is link2</a> \\\n",
    " </body> \\\n",
    " </html>'\n",
    "\n",
    "soup = BeautifulSoup(html_sample)\n",
    "print soup.text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<a class=\"link\" href=\"#\">This is link1</a>, <a class=\"link\" href=\"# link2\">This is link2</a>]\n",
      "[<a class=\"link\" href=\"#\">This is link1</a>, <a class=\"link\" href=\"# link2\">This is link2</a>]\n"
     ]
    }
   ],
   "source": [
    "a = soup.select(\"a\")\n",
    "print a\n",
    "\n",
    "link = soup.select(\".link\")\n",
    "print link"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 使用爬蟲蒐集每日頻果要聞"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "section = requests.get('http://www.appledaily.com.tw/realtimenews/section/new/1')\n",
    "soup = BeautifulSoup(section.text)\n",
    "#for news in soup.select('.rtddt'):\n",
    "#    print news"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 根據不同HTML標籤取得對應內容"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "domain = 'http://www.appledaily.com.tw'\n",
    "for news in soup.select('.rtddt'):    \n",
    "    main = news.select('h1')[0].text\n",
    "    category = news.select('h2')[0].text \n",
    "    time = news.select('time')[0].text \n",
    "    url = domain + news.select('a')[0]['href']    \n",
    "    #print main, category, time, url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "domain = 'http://www.appledaily.com.tw'\n",
    "for news in soup.select('.rtddt'):    \n",
    "    main = news.select('h1')[0].text\n",
    "    m = re.match('(.*)\\((\\d+)\\)',main)\n",
    "    if m:\n",
    "        title = m.group(1)\n",
    "        popularity = int(m.group(2))\n",
    "    #print title, popularity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "section = requests.get('http://www.appledaily.com.tw/realtimenews/section/new/1')\n",
    "soup = BeautifulSoup(section.text)\n",
    "domain = 'http://www.appledaily.com.tw'\n",
    "newsary = []\n",
    "for news in soup.select('.rtddt'):    \n",
    "    main = news.select('h1')[0].text\n",
    "    m = re.match('(.*)\\((\\d+)\\)',main)\n",
    "    if m:\n",
    "        title = m.group(1)\n",
    "        popularity = int(m.group(2))\n",
    "    category = news.select('h2')[0].text \n",
    "    time = news.select('time')[0].text \n",
    "    url = domain + news.select('a')[0]['href']\n",
    "    dic = {'title': title, 'popularity':popularity,'category':category, 'time':time, 'url':url}\n",
    "    newsary.append(dic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 根據頁數蒐集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "domain = 'http://www.appledaily.com.tw'\n",
    "\n",
    "newsary = []\n",
    "url = 'http://www.appledaily.com.tw/realtimenews/section/new/%d'\n",
    "for page in range(1,2):\n",
    "    section = requests.get(url%(page))\n",
    "    soup = BeautifulSoup(section.text)\n",
    "    for news in soup.select('.rtddt'):    \n",
    "        main = news.select('h1')[0].text\n",
    "        m = re.match('(.*)\\((\\d+)\\)',main)\n",
    "        if m:\n",
    "            title = m.group(1)\n",
    "            popularity = int(m.group(2))\n",
    "        category = news.select('h2')[0].text \n",
    "        time = news.select('time')[0].text \n",
    "        url = domain + news.select('a')[0]['href']\n",
    "        dic = {'title': title, 'popularity':popularity,'category':category, 'time':time, 'url':url}\n",
    "        newsary.append(dic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 印出蒐集新聞"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30\n"
     ]
    }
   ],
   "source": [
    "print len(newsary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'category': u'\\u653f\\u6cbb', 'url': u'http://www.appledaily.com.tw/realtimenews/article/politics/20150528/618173/\\u639b\\u770b\\u677f\\u300c\\u5c0b\\u8cf4\\u6e05\\u5fb7\\u300d\\u3000\\u674e\\u5168\\u6559\\uff1a\\u4eca\\u6709\\u91cd\\u5927\\u5ba3\\u5e03', 'popularity': 0, 'time': u'06:44', 'title': u'\\u639b\\u770b\\u677f\\u300c\\u5c0b\\u8cf4\\u6e05\\u5fb7\\u300d\\u3000\\u674e\\u5168\\u6559\\uff1a\\u4eca\\u6709\\u91cd\\u5927\\u5ba3\\u5e03'}, {'category': u'\\u793e\\u6703', 'url': u'http://www.appledaily.com.tw/realtimenews/article/local/20150528/618169/\\u7f8e\\u9b54\\u5973\\u5343\\u676f\\u4e0d\\u5012\\u3000\\u683d\\u57281\\u676f\\u5564\\u9152\\u7591\\u906d\\u8ff7\\u59e6', 'popularity': 28, 'time': u'06:39', 'title': u'\\u7f8e\\u9b54\\u5973\\u5343\\u676f\\u4e0d\\u5012\\u3000\\u683d\\u57281\\u676f\\u5564\\u9152\\u7591\\u906d\\u8ff7\\u59e6'}, {'category': u'\\u793e\\u6703', 'url': u'http://www.appledaily.com.tw/realtimenews/article/local/20150528/618166/\\u9ad8\\u91ab\\u60c5\\u6bba\\u5167\\u5e55\\u3000\\u91ab\\u5b78\\u751f\\u6bba\\u4eba\\u5f8c\\u8ddf\\u5b78\\u59b9\\u8aaa\\u2026', 'popularity': 598, 'time': u'06:32', 'title': u'\\u9ad8\\u91ab\\u60c5\\u6bba\\u5167\\u5e55\\u3000\\u91ab\\u5b78\\u751f\\u6bba\\u4eba\\u5f8c\\u8ddf\\u5b78\\u59b9\\u8aaa\\u2026'}]\n"
     ]
    }
   ],
   "source": [
    "print newsary[0:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "掛看板「尋賴清德」　李全教：今有重大宣布 06:44\n",
      "美魔女千杯不倒　栽在1杯啤酒疑遭迷姦 06:39\n",
      "高醫情殺內幕　醫學生殺人後跟學妹說… 06:32\n"
     ]
    }
   ],
   "source": [
    "for news in newsary[0:3]:\n",
    "    print news['title'], news['time']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 使用pandas 做敘述性統計"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>popularity</th>\n",
       "      <th>time</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td> 政治</td>\n",
       "      <td>   0</td>\n",
       "      <td> 06:44</td>\n",
       "      <td> 掛看板「尋賴清德」　李全教：今有重大宣布</td>\n",
       "      <td> http://www.appledaily.com.tw/realtimenews/arti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td> 社會</td>\n",
       "      <td>  28</td>\n",
       "      <td> 06:39</td>\n",
       "      <td>   美魔女千杯不倒　栽在1杯啤酒疑遭迷姦</td>\n",
       "      <td> http://www.appledaily.com.tw/realtimenews/arti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td> 社會</td>\n",
       "      <td> 598</td>\n",
       "      <td> 06:32</td>\n",
       "      <td>   高醫情殺內幕　醫學生殺人後跟學妹說…</td>\n",
       "      <td> http://www.appledaily.com.tw/realtimenews/arti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td> 國際</td>\n",
       "      <td>  58</td>\n",
       "      <td> 06:30</td>\n",
       "      <td>      五月二十八日世界各報頭條搶先報</td>\n",
       "      <td> http://www.appledaily.com.tw/realtimenews/arti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td> 社會</td>\n",
       "      <td> 269</td>\n",
       "      <td> 06:29</td>\n",
       "      <td> 毛毛der！正妹頭七　男友同車同地同時死</td>\n",
       "      <td> http://www.appledaily.com.tw/realtimenews/arti...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  category  popularity   time                 title  \\\n",
       "0       �踵祥           0  06:44  ����踴�撠�陷皜�噸���������隞����之摰��   \n",
       "1       蝷暹�          28  06:39    蝢��憟喳��臭�����賢�1�臬�����剛蕙憪�  \n",
       "2       蝷暹�         598  06:32    擃����捏�批����摮貊�畾箔犖敺��摮詨此隤芬�   \n",
       "3       ���          58  06:30       鈭��鈭���急�銝������剜��嗅���  \n",
       "4       蝷暹�         269  06:29  瘥��der嚗�迤憒寥�銝���瑕����������甇�  \n",
       "\n",
       "                                                 url  \n",
       "0  http://www.appledaily.com.tw/realtimenews/arti...  \n",
       "1  http://www.appledaily.com.tw/realtimenews/arti...  \n",
       "2  http://www.appledaily.com.tw/realtimenews/arti...  \n",
       "3  http://www.appledaily.com.tw/realtimenews/arti...  \n",
       "4  http://www.appledaily.com.tw/realtimenews/arti...  \n",
       "\n",
       "[5 rows x 5 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "newspd = pd.DataFrame(newsary)\n",
    "newspd.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 根據人氣排序"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>popularity</th>\n",
       "      <th>time</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td> 社會</td>\n",
       "      <td> 6546</td>\n",
       "      <td> 05:54</td>\n",
       "      <td> ​AV女優遊夜店　巧遇警臨檢好掃興</td>\n",
       "      <td> http://www.appledaily.com.tw/realtimenews/arti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td> 國際</td>\n",
       "      <td> 5156</td>\n",
       "      <td> 03:48</td>\n",
       "      <td> 南非銷售最好手機　不是蘋果也非三星</td>\n",
       "      <td> http://www.appledaily.com.tw/realtimenews/arti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td> 國際</td>\n",
       "      <td> 2889</td>\n",
       "      <td> 02:40</td>\n",
       "      <td>  神學士稱策畫突襲　整夜槍戰釀4死</td>\n",
       "      <td> http://www.appledaily.com.tw/realtimenews/arti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td> 娛樂</td>\n",
       "      <td> 1946</td>\n",
       "      <td> 03:16</td>\n",
       "      <td>   宇宙人東京鬧區滑板　慘仆街見血</td>\n",
       "      <td> http://www.appledaily.com.tw/realtimenews/arti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td> 社會</td>\n",
       "      <td> 1801</td>\n",
       "      <td> 04:51</td>\n",
       "      <td>  地下室起火上千住戶逃　驚呼好可怕</td>\n",
       "      <td> http://www.appledaily.com.tw/realtimenews/arti...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   category  popularity   time              title  \\\n",
       "14       蝷暹�        6546  05:54  ��V憟喳����摨��撌折�霅西�瑼Ｗ末���   \n",
       "25       ���        5156  03:48  ����瑕���末�������航���������  \n",
       "28       ���        2889  02:40   蟡�飛憯怎迂蝑��蝒�必���憭���圈�4甇�  \n",
       "27       憡��        1946  03:16    摰��鈭箸�鈭祇洹����踴����銵��銵�  \n",
       "19       蝷暹�        1801  04:51   �唬�摰方絲�思�����園�����澆末�舀�   \n",
       "\n",
       "                                                  url  \n",
       "14  http://www.appledaily.com.tw/realtimenews/arti...  \n",
       "25  http://www.appledaily.com.tw/realtimenews/arti...  \n",
       "28  http://www.appledaily.com.tw/realtimenews/arti...  \n",
       "27  http://www.appledaily.com.tw/realtimenews/arti...  \n",
       "19  http://www.appledaily.com.tw/realtimenews/arti...  \n",
       "\n",
       "[5 rows x 5 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newspd.sort(['popularity'], ascending=False).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 依據各分類排行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "category\n",
      "政治           552\n",
      "社會          6546\n",
      "國際          5156\n",
      "生活           322\n",
      "財經           506\n",
      "論壇           884\n",
      "娛樂          1946\n",
      "體育          1363\n",
      "Name: popularity, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "df1 = newspd.groupby(['category'], sort=False)['popularity'].max()\n",
    "print df1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 抓出各分類最熱門文章"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>popularity</th>\n",
       "      <th>time</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td> 生活</td>\n",
       "      <td>  322</td>\n",
       "      <td> 06:17</td>\n",
       "      <td>    氣溫回升　新竹以南有局部大雨</td>\n",
       "      <td> http://www.appledaily.com.tw/realtimenews/arti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td> 政治</td>\n",
       "      <td>  552</td>\n",
       "      <td> 06:08</td>\n",
       "      <td> 變電箱一整排很醜　柯文哲指示要美化</td>\n",
       "      <td> http://www.appledaily.com.tw/realtimenews/arti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td> 社會</td>\n",
       "      <td> 6546</td>\n",
       "      <td> 05:54</td>\n",
       "      <td> ​AV女優遊夜店　巧遇警臨檢好掃興</td>\n",
       "      <td> http://www.appledaily.com.tw/realtimenews/arti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td> 財經</td>\n",
       "      <td>  506</td>\n",
       "      <td> 05:52</td>\n",
       "      <td>     五月二十八日財經焦點搶先看</td>\n",
       "      <td> http://www.appledaily.com.tw/realtimenews/arti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td> 論壇</td>\n",
       "      <td>  884</td>\n",
       "      <td> 04:00</td>\n",
       "      <td>    史丹利：無辜的宏都拉斯大使 </td>\n",
       "      <td> http://www.appledaily.com.tw/realtimenews/arti...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   category  popularity   time              title  \\\n",
       "10       ��暑         322  06:17     瘞�澈������蝡嫣誑���撅��憭折�   \n",
       "13       �踵祥         552  06:08  霈��蝞曹��湔�敺����������內閬����  \n",
       "14       蝷暹�        6546  05:54  ��V憟喳����摨��撌折�霅西�瑼Ｗ末���   \n",
       "15       鞎∠�         506  05:52      鈭��鈭���急�鞎∠��阡��嗅���  \n",
       "24       隢��         884  04:00     �脖號�抬��∟�����賣��臬之雿�   \n",
       "\n",
       "                                                  url  \n",
       "10  http://www.appledaily.com.tw/realtimenews/arti...  \n",
       "13  http://www.appledaily.com.tw/realtimenews/arti...  \n",
       "14  http://www.appledaily.com.tw/realtimenews/arti...  \n",
       "15  http://www.appledaily.com.tw/realtimenews/arti...  \n",
       "24  http://www.appledaily.com.tw/realtimenews/arti...  \n",
       "\n",
       "[5 rows x 5 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = newspd.groupby(['category'])['popularity'].transform(max) == newspd['popularity']\n",
    "newspd[idx].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 印出每日新聞摘要"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "台南市區昨出現大型「尋人啟事」廣告牆，看板用紅色大字寫「市長賴清德與台南市民意已失聯」、「第142天」、「請190萬台南市民協尋市長賴清德」，引起人車注目。今日出版的《蘋果日報》報導，李全教坦承大字報是他指示張貼，大罵賴清德獨裁，以李涉嫌賄選，從議會一月開定期會就不進議會備詢，至今跟議會失聯一百四十二天，他的賄選案法院仍在審理，希望賴不要以此當藉口。《自由時報》報導，李全教也透露台南市府已經接獲行政院公函，表達依照地制法，台南市市長及官員必須到議會做業務報告及備詢，並同時預告他今天將有「重大宣布」。(葉國吏/綜合報導)\n"
     ]
    }
   ],
   "source": [
    "for newsurl in newsary[0:1]:    \n",
    "    res = requests.get(newsurl['url'])\n",
    "    soup = BeautifulSoup(res.text)\n",
    "    print soup.select('#summary')[0].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for newsurl in newsary:    \n",
    "    res = requests.get(newsurl['url'])\n",
    "    soup = BeautifulSoup(res.text)\n",
    "    newsurl['summary'] = soup.select('#summary')[0].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "台南市區昨出現大型「尋人啟事」廣告牆，看板用紅色大字寫「市長賴清德與台南市民意已失聯」、「第142天」、「請190萬台南市民協尋市長賴清德」，引起人車注目。今日出版的《蘋果日報》報導，李全教坦承大字報\n"
     ]
    }
   ],
   "source": [
    "print newsary[0]['summary'][0:100]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 從Facebook 討論區了解網民評論"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://www.appledaily.com.tw/realtimenews/article/international/20150528/618168/五月二十八日世界各報頭條搶先報\n",
      "883071741753928\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Python27\\lib\\site-packages\\requests\\packages\\urllib3\\util\\ssl_.py:79: InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.\n",
      "  InsecurePlatformWarning\n"
     ]
    }
   ],
   "source": [
    "import urllib\n",
    "import json\n",
    "print newsary[3]['url']\n",
    "app_id = '409090199295451'\n",
    "app_secret = '<secret>'\n",
    "graph_url  = 'https://graph.facebook.com/v2.3/%s?access_token=%s|%s'%(urllib.quote(newsary[0]['url'].encode('utf8')), app_id, app_secret)\n",
    "res = requests.get(graph_url)\n",
    "dt = json.loads(res.text)\n",
    "post_id =  dt['og_object']['id']\n",
    "print post_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Python27\\lib\\site-packages\\requests\\packages\\urllib3\\util\\ssl_.py:79: InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.\n",
      "  InsecurePlatformWarning\n"
     ]
    }
   ],
   "source": [
    "post_id = '938417442844934'\n",
    "import urllib\n",
    "import json\n",
    "import pandas as pd\n",
    "app_id = '409090199295451'\n",
    "app_secret = '<secret>'\n",
    "post_url  = 'https://graph.facebook.com/v2.3/%s/comments?access_token=%s|%s'%(post_id, app_id, app_secret)\n",
    "res = requests.get(post_url)\n",
    "json_data = json.loads(res.text)\n",
    "#print json_data['data']\n",
    "#df = pd.read_json(json_data['data'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{u'paging': {u'cursors': {u'after': u'Mg==', u'before': u'NA=='}}, u'data': [{u'from': {u'id': u'10204704242467035', u'name': u'Joy Chien'}, u'like_count': 0, u'can_remove': False, u'created_time': u'2015-05-25T17:44:09+0000', u'message': u'\\u770b\\u5f71\\u7247\\u767c\\u73fe\\u5c0f\\u8ca8\\u8eca\\u6703\\u6025\\u715e\\u6253\\u6ed1\\n\\u662f\\u56e0\\u70ba\\u524d\\u8eca\\u6025\\u715e\\n\\u5c0f\\u8ca8\\u8eca\\u6253\\u6ed1\\u5f8c\\u524d\\u8eca\\u715e\\u8eca\\u71c8\\u9084\\u4eae\\u8457\\n\\u7136\\u5f8c\\u98db\\u901f\\u5f80\\u524d\\u884c\\u99db\\n\\u6240\\u4ee5\\u96e8\\u5929\\u8eca\\u591a\\u6642\\u9084\\u662f\\u76e1\\u91cf\\u4fdd\\u6301\\u5b89\\u5168\\u8ddd\\u96e2\\u5427', u'id': u'938417442844934_938572532829425', u'user_likes': False}, {u'from': {u'id': u'1006511392693378', u'name': u'Daniel Shih'}, u'like_count': 1, u'can_remove': False, u'created_time': u'2015-05-25T15:48:54+0000', u'message': u'\\u9019\\u500b\\u8def\\u6bb5\\u4e09\\u5929\\u5169\\u982d\\u7684\\u5c31\\u6709\\u4e8b\\u6545\\u767c\\u751f,\\u660e\\u660e\\u662f\\u9053\\u8def\\u8a2d\\u8a08\\u4e0d\\u826f\\u537b\\u4e00\\u76f4\\u64fa\\u8457\\u4e0d\\u53bb\\u6539\\u5584!\\n\\u56e0\\u70ba\\u8eca\\u6d41\\u91cf\\u904e\\u5927\\u5e38\\u5e38\\u5c0e\\u81f4\\u8a72\\u8def\\u6bb5\\u6bcf\\u5929\\u585e\\u8eca,\\n\\u56e0\\u70ba\\u8eca\\u6d41\\u91cf\\u904e\\u5927\\u5e38\\u5e38\\u6709\\u4e00\\u5927\\u5806\\u767d\\u7661\\u4e0d\\u6392\\u968a\\u8981\\u5f9e\\u7b2c\\u4e09\\u7dda\\u9053\\u786c\\u63d2\\u968a,\\n\\u56e0\\u70ba\\u8eca\\u6d41\\u91cf\\u904e\\u5927\\u53c8\\u4e00\\u5927\\u5806\\u767d\\u7661\\u63d2\\u968a,\\u6240\\u4ee5\\u5c0e\\u81f4\\u8d8a\\u63a5\\u8fd1\\u5165\\u53e3\\u531d\\u9053\\u5927\\u5bb6\\u8eca\\u5b50\\u8cbc\\u8d8a\\u8fd1!\\n\\u9019\\u4e9b\\u554f\\u984c\\u9ad8\\u516c\\u5c40\\u7684\\u5b98\\u54e1\\u5230\\u5e95\\u8981\\u4e0d\\u8981\\u8655\\u7406???', u'id': u'938417442844934_938538919499453', u'user_likes': False}, {u'from': {u'id': u'826889840694186', u'name': u'Reginald Ben'}, u'like_count': 0, u'can_remove': False, u'created_time': u'2015-05-25T13:06:45+0000', u'message': u'\\u53c8\\u662f\\u8ca8\\u8eca\\u754c\\u7684\\u795e\\u8eca\\u51fa\\u7cd7\\u4e86\\u3002', u'id': u'938417442844934_938478732838805', u'user_likes': False}]}\n"
     ]
    }
   ],
   "source": [
    "print json_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_time</th>\n",
       "      <th>like_count</th>\n",
       "      <th>message</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td> 2015-05-25T17:44:09+0000</td>\n",
       "      <td> 0</td>\n",
       "      <td> 看影片發現小貨車會急煞打滑\\n是因為前車急煞\\n小貨車打滑後前車煞車燈還亮著\\n然後飛速往前...</td>\n",
       "      <td>    Joy Chien</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td> 2015-05-25T15:48:54+0000</td>\n",
       "      <td> 1</td>\n",
       "      <td> 這個路段三天兩頭的就有事故發生,明明是道路設計不良卻一直擺著不去改善!\\n因為車流量過大常常...</td>\n",
       "      <td>  Daniel Shih</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td> 2015-05-25T13:06:45+0000</td>\n",
       "      <td> 0</td>\n",
       "      <td>                                      又是貨車界的神車出糗了。</td>\n",
       "      <td> Reginald Ben</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               created_time  like_count  \\\n",
       "0  2015-05-25T17:44:09+0000           0   \n",
       "1  2015-05-25T15:48:54+0000           1   \n",
       "2  2015-05-25T13:06:45+0000           0   \n",
       "\n",
       "                                             message          name  \n",
       "0  ��蔣����曉�鞎刻�������皛�n�臬��箏�頠����n撠�疏頠��皛�����������鈭株�\\n�嗅�憌��敺��...     Joy Chien  \n",
       "1  ���頝舀挾銝�予�拚���停������������舫�頝航身閮���臬�銝���箄�銝���孵�!\\n���頠�����憭批虜撣�..   Daniel Shih  \n",
       "2                                       ���鞎刻����蟡���箇�鈭��  Reginald Ben  \n",
       "\n",
       "[3 rows x 4 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtarray = []\n",
    "for comment in json_data['data']:\n",
    "    dtarray.append( {'created_time':comment['created_time'], 'name':comment['from']['name'], \\\n",
    "          'message':comment['message'], 'like_count':comment['like_count']})\n",
    "posts_df = pd.DataFrame(dtarray)\n",
    "posts_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 使用資料庫管理新聞內容"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 建立表格存放新聞"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*- \n",
    "import sqlite3\n",
    "\n",
    "db = sqlite3.connect('news.db') \n",
    "cur = db.cursor()\n",
    "cur.execute('''DROP TABLE IF EXISTS news_entry''')\n",
    "cur.execute('''CREATE TABLE IF NOT EXISTS news_entry(\n",
    "                id INTEGER PRIMARY KEY,\n",
    "                category TEXT,\n",
    "                popularity INTEGER,\n",
    "                time VARCHAR(32),\n",
    "                title VARCHAR(256),\n",
    "                summary TEXT,\n",
    "                url VARCHAR(1024)\n",
    "                )''')\n",
    "cur.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 新增一筆資料進SQLite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "掛看板「尋賴清德」　李全教：今有重大宣布\n"
     ]
    }
   ],
   "source": [
    "col = ', ' .join(newsary[0].keys())\n",
    "val = ','.join('?'* (len(newsary[0].keys())))\n",
    "\n",
    "sql = 'insert into news_entry(%s) values(%s)'%(col, val)\n",
    "db = sqlite3.connect('news.db') \n",
    "cur = db.cursor()\n",
    "cur.execute(sql,newsary[0].values())\n",
    "cur.execute('select * from news_entry')\n",
    "rec = cur.fetchone()\n",
    "print rec[4]            \n",
    "cur.close()\n",
    "db.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 將全部資料放進SQLite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "db = sqlite3.connect('news.db') \n",
    "\n",
    "cur = db.cursor()\n",
    "\n",
    "delete_sql = 'delete from news_entry'\n",
    "cur.execute(delete_sql)\n",
    "\n",
    "for news in newsary:\n",
    "    col = ', ' .join(news.keys())\n",
    "    val = ','.join('?'* (len(news.keys())))\n",
    "    sql = 'insert into news_entry(%s) values(%s)'%(col, val)\n",
    "    cur.execute(sql,news.values())\n",
    "    db.commit()\n",
    "          \n",
    "cur.close()\n",
    "db.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 讀取表格中的所有資料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 政治 0 06:44 掛看板「尋賴清德」　李全教：今有重大宣布\n",
      "2 社會 28 06:39 美魔女千杯不倒　栽在1杯啤酒疑遭迷姦\n",
      "3 社會 598 06:32 高醫情殺內幕　醫學生殺人後跟學妹說…\n",
      "4 國際 58 06:30 五月二十八日世界各報頭條搶先報\n",
      "5 社會 269 06:29 毛毛der！正妹頭七　男友同車同地同時死\n",
      "6 政治 243 06:25 五月二十八日各報頭條搶先報\n",
      "7 社會 283 06:25 警消51天找嘸　山友靠「尋音」2天找到\n",
      "8 社會 736 06:24 私娼寮偽裝神壇　阿公一早就去嫖妓\n",
      "9 社會 901 06:18 劈腿女主播後愛陳敏薰　王燕軍「劈腿被打腿...\n",
      "10 社會 962 06:17 【更新】印刷公司深夜冒火　警消急搶救未釀...\n",
      "11 生活 322 06:17 氣溫回升　新竹以南有局部大雨\n",
      "12 社會 722 06:16 大立光6內鬼竊密跳槽　光紅利就少領200...\n",
      "13 社會 633 06:10 愛子山難母寫百萬字報告　創山難搜救不力國...\n",
      "14 政治 552 06:08 變電箱一整排很醜　柯文哲指示要美化\n",
      "15 社會 6546 05:54 ​AV女優遊夜店　巧遇警臨檢好掃興\n",
      "16 財經 506 05:52 五月二十八日財經焦點搶先看\n",
      "17 國際 483 05:40 陸漢怒吞水果刀　忍痛一周才求醫\n",
      "18 國際 1508 05:29 藍寶堅尼義大利生產　打入豪華休旅車市場\n",
      "19 國際 1044 05:17 FIFA收賄　贓款藏香港等地\n",
      "20 社會 1801 04:51 地下室起火上千住戶逃　驚呼好可怕\n",
      "21 國際 402 04:40 周永康案　分析稱在五中全會前結審\n",
      "22 國際 378 04:28 英前首相布萊爾　辭任中東和平特使\n",
      "23 國際 458 04:21 聯合國提議　成員國再收4萬難民\n",
      "24 論壇 396 04:00 Choyce：上湧別春爛漫 鬱金香繽紛登...\n",
      "25 論壇 884 04:00 史丹利：無辜的宏都拉斯大使 \n",
      "26 國際 5156 03:48 南非銷售最好手機　不是蘋果也非三星\n",
      "27 國際 934 03:34 衝突又起　以色列警告巴勒斯坦將付代價\n",
      "28 娛樂 1946 03:16 宇宙人東京鬧區滑板　慘仆街見血\n",
      "29 國際 2889 02:40 神學士稱策畫突襲　整夜槍戰釀4死\n",
      "30 體育 1363 02:35 法網詹皓晴組合直落二獲勝　台將第二組晉女...\n"
     ]
    }
   ],
   "source": [
    "db = sqlite3.connect('news.db') \n",
    "cur = db.cursor()\n",
    "cur.execute('select * from news_entry')\n",
    "allNews = cur.fetchall()\n",
    "for rec in allNews:\n",
    "    print rec[0], rec[1], rec[2], rec[3], rec[4]\n",
    "cur.close()\n",
    "db.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 產生 2-gram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "那我\n",
      "我們\n",
      "們酸\n",
      "酸民\n",
      "民婉\n",
      "婉君\n",
      "君也\n",
      "也可\n",
      "可以\n",
      "以報\n",
      "報名\n",
      "名嗎\n"
     ]
    }
   ],
   "source": [
    "input_sentence='那我們酸民婉君也可以報名嗎'\n",
    "\n",
    "sentence  = input_sentence.decode('utf-8')\n",
    "\n",
    "for i in range(0, len(sentence) - 2 + 1):\n",
    "    print sentence[i:i+2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 產生 3-gram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "那我們\n",
      "我們酸\n",
      "們酸民\n",
      "酸民婉\n",
      "民婉君\n",
      "婉君也\n",
      "君也可\n",
      "也可以\n",
      "可以報\n",
      "以報名\n",
      "報名嗎\n"
     ]
    }
   ],
   "source": [
    "for i in range(0, len(sentence) -3 + 1):\n",
    "    print sentence[i:i+3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 建立 n-gram 函式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def ngram(input_sentence, n = 2):\n",
    "    word_dic = {}\n",
    "    sentence  = input_sentence.decode('utf-8')\n",
    "    for i in range(0, len(sentence) - n + 1):\n",
    "        if sentence[i:i+n] not in word_dic:\n",
    "            word_dic[sentence[i:i+n]] = 1\n",
    "        else:\n",
    "            word_dic[sentence[i:i+n]] = word_dic[sentence[i:i+n]] + 1\n",
    "    return word_dic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 測試 bigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "婉君 報名 可以 名嗎 君也 也可 民婉 酸民 以報\n"
     ]
    }
   ],
   "source": [
    "for word in ngram('酸民婉君也可以報名嗎'):\n",
    "    print word,"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 測試 trigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "酸民婉\n",
      "君也可\n",
      "名嗎?\n",
      "可以報\n",
      "也可以\n",
      "們酸民\n",
      "以報名\n",
      "民婉君\n",
      "報名嗎\n",
      "那我們\n",
      "婉君也\n",
      "我們酸\n"
     ]
    }
   ],
   "source": [
    "for word in ngram('那我們酸民婉君也可以報名嗎?' ,n=3):\n",
    "    print word"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 移除標點符號"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence = '123?'\n",
    "skip_word = '.?'\n",
    "len([word for word in sentence if word in skip_word]) > 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "skip_dic = '?.,、　「！】？：《」》～【'.decode('utf-8')\n",
    "def ngram(input_sentence, n = 2, skip_word= skip_dic):\n",
    "    word_dic = {}\n",
    "    sentence  = input_sentence.decode('utf-8')\n",
    "    for i in range(0, len(sentence) - n + 1):\n",
    "        invalid = len([word for word in sentence[i:i+n] if word in skip_word])\n",
    "        if sentence[i:i+n] not in word_dic and invalid ==0:\n",
    "            word_dic[sentence[i:i+n]] = 1\n",
    "        elif invalid ==0:\n",
    "            word_dic[sentence[i:i+n]] = word_dic[sentence[i:i+n]] + 1\n",
    "    return word_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "我們\n",
      "婉君\n",
      "報名\n",
      "們酸\n",
      "可以\n",
      "名嗎\n",
      "君也\n",
      "也可\n",
      "那我\n",
      "酸民\n",
      "以報\n"
     ]
    }
   ],
   "source": [
    "for word in ngram('那我們酸民、婉君也可以報名嗎?'):\n",
    "    print word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "db = sqlite3.connect('news.db') \n",
    "cur = db.cursor()\n",
    "cur.execute('select * from news_entry')\n",
    "allNews = cur.fetchall()\n",
    "dic = {}\n",
    "for rec in allNews:\n",
    "    ngram_dic = ngram(rec[4].encode('utf-8'))\n",
    "    for ele in ngram_dic:\n",
    "        if ele not in dic:\n",
    "            dic[ele] = ngram_dic[ele]\n",
    "        else:\n",
    "            dic[ele] = dic[ele] + ngram_dic[ele]\n",
    "cur.close()\n",
    "db.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "搶先 3\n",
      "五月 3\n",
      "八日 3\n",
      "二十 3\n",
      "十八 3\n",
      "月二 3\n",
      "天找 2\n",
      "頭條 2\n",
      "警消 2\n",
      "山難 2\n",
      "先報 2\n",
      "報頭 2\n",
      "劈腿 2\n",
      "條搶 2\n",
      "各報 2\n"
     ]
    }
   ],
   "source": [
    "import operator\n",
    "words_freq = sorted(dic.iteritems(),key=operator.itemgetter(1),reverse=True)\n",
    "    \n",
    "for word in words_freq:\n",
    "    if word[1] >= 2:\n",
    "        print word[0], word[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 長詞優先演算法"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 如何依標點符號切開字詞"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3年前就讀高雄應用大學二年級的陳顯佳\n",
      "為閃避跨越雙黃線來車\n",
      "撞上路邊\n",
      "摔斷頸椎\n",
      "頸部以下全癱\n",
      "嘴被插管\n",
      "父母每天去看他\n",
      "他都意識清醒\n",
      "眨眼流淚\n",
      "17天後他因頸椎受損抑制呼吸\n",
      "醫師用盡強力針與電擊都無效\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "delimiter = \"，|。|、\"  \n",
    "text = '3年前就讀高雄應用大學二年級的陳顯佳，為閃避跨越雙黃線來車，撞上路邊、摔斷頸椎，頸部以下全癱，嘴被插管，父母每天去看他，他都意識清醒、眨眼流淚，17天後他因頸椎受損抑制呼吸，醫師用盡強力針與電擊都無效，'\n",
    "for i in re.split(delimiter, text):\n",
    "    print i"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 測試切除效果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def splitSentense(text, delimiter):\n",
    "    return re.split(delimiter, text)\n",
    "    \n",
    "delimiter = \"，|。|、|（|）|／|《|》|】|【|「|」|；|：\".decode('utf-8')  \n",
    "db = sqlite3.connect('news.db') \n",
    "cur = db.cursor()\n",
    "cur.execute('select * from news_entry')\n",
    "allNews = cur.fetchall()\n",
    "sentenceAry = []\n",
    "for rec in allNews:\n",
    "    text = rec[5]\n",
    "    sentenceAry += splitSentense(text,delimiter)\n",
    "cur.close()\n",
    "db.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "台南市區昨出現大型\n",
      "尋人啟事\n",
      "廣告牆\n",
      "看板用紅色大字寫\n",
      "市長賴清德與台南市民意已失聯\n",
      "\n",
      "\n",
      "第142天\n"
     ]
    }
   ],
   "source": [
    "for sentence in sentenceAry[0:8]:\n",
    "    print sentence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 移除關鍵字"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "台南市區昨出現大型\n",
      "台南市區昨出現大型\n"
     ]
    }
   ],
   "source": [
    "def removeKey(text, keyword):\n",
    "    textAry= text\n",
    "    for key in keyword:\n",
    "        textAry = ''.join(textAry.split(key.decode('utf-8')))\n",
    "    return textAry\n",
    "print sentenceAry[0]\n",
    "print removeKey(sentenceAry[0], ['民進黨', '後天'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'\\u5c07\\u516c\\u5e03'"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "a = '民進黨後天將公布'.decode('utf-8')\n",
    "removeKey(a, ['民進黨', '後天'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 修改ngram 處理函式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ngram(input_sentence, n = 2):\n",
    "    word_dic = {}\n",
    "    sentence  = input_sentence\n",
    "    for i in range(0, len(sentence) - n + 1):        \n",
    "        if sentence[i:i+n] not in word_dic:\n",
    "            word_dic[sentence[i:i+n]] = 1\n",
    "        else:\n",
    "            word_dic[sentence[i:i+n]] = word_dic[sentence[i:i+n]] + 1\n",
    "    return word_dic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 依長詞優先產生關鍵字"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "報導 18\n",
      "表示 17\n",
      "景氣 15\n",
      "美國 14\n",
      "台灣 14\n",
      "一個 13\n",
      "萬元 13\n",
      "警方 13\n",
      "綜合 13\n",
      "藍燈 13\n",
      "國賠 12\n",
      "綜合報導 12\n",
      "時間 12\n",
      "26 11\n",
      "難民 11\n",
      "外電報導 10\n",
      "搜救 10\n",
      "山難搜救 10\n",
      "夜店 10\n",
      "今年 10\n",
      "合外電報 10\n",
      "襲擊 10\n",
      "手機 10\n",
      "男子 10\n",
      "生產 10\n",
      "綜合外電 10\n",
      "今日出版 9\n",
      "逮捕 9\n",
      "日出版的 9\n",
      "以色列 9\n"
     ]
    }
   ],
   "source": [
    "keywords=[]        \n",
    "ret_terms={}\n",
    "words_freq    = []\n",
    "for term_length in range(4,1,-1):\n",
    "    word_dic = {}\n",
    "    for sentence in sentenceAry:\n",
    "        text_list = removeKey(sentence,keywords)        \n",
    "        ngram_words = ngram(text_list,term_length) \n",
    "        for word in ngram_words:\n",
    "            if word not in word_dic:\n",
    "                word_dic[word] = 1\n",
    "            else:\n",
    "                word_dic[word] += ngram_words[word]   \n",
    "    for word in word_dic:\n",
    "        if word_dic[word] >= 5:\n",
    "            keywords.append(word.encode('utf-8'))            \n",
    "            ret_terms.update({word:word_dic[word]})\n",
    "\n",
    "sorted_terms = sorted(ret_terms.iteritems(),key=operator.itemgetter(1),reverse=True) \n",
    "for term in sorted_terms[0:30]:\n",
    "    print term[0], term[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from C:\\Python27\\lib\\site-packages\\jieba\\dict.txt ...\n",
      "DEBUG:jieba:Building prefix dict from C:\\Python27\\lib\\site-packages\\jieba\\dict.txt ...\n",
      "Loading model from cache c:\\users\\david\\appdata\\local\\temp\\jieba.cache\n",
      "DEBUG:jieba:Loading model from cache c:\\users\\david\\appdata\\local\\temp\\jieba.cache\n",
      "Loading model cost 1.05099987984 seconds.\n",
      "DEBUG:jieba:Loading model cost 1.05099987984 seconds.\n",
      "Prefix dict has been built succesfully.\n",
      "DEBUG:jieba:Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "大\n",
      "巨蛋\n",
      "案對\n",
      "市府\n",
      "同仁\n",
      "下\n",
      "封口令\n",
      "？\n",
      "　\n",
      "柯\n",
      "P\n",
      "否認\n"
     ]
    }
   ],
   "source": [
    "import jieba\n",
    "\n",
    "sentence = \"大巨蛋案對市府同仁下封口令？　柯P否認\"\n",
    "words = jieba.cut(sentence, cut_all=False)\n",
    "for word in words:\n",
    "    print word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "大巨蛋\n",
      "案對\n",
      "市府\n",
      "同仁\n",
      "下\n",
      "封口令\n",
      "？\n",
      "　\n",
      "柯P\n",
      "否認\n"
     ]
    }
   ],
   "source": [
    "jieba.add_word('柯P',100)\n",
    "jieba.add_word('大巨蛋',100)\n",
    "sentence = \"大巨蛋案對市府同仁下封口令？　柯P否認\"\n",
    "words = jieba.cut(sentence, cut_all=False)\n",
    "for word in words:\n",
    "    print word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "大巨蛋 x\n",
      "案 ng\n",
      "對 p\n",
      "市府 n\n",
      "同仁 nr\n",
      "下 f\n",
      "封口令 n\n",
      "？ x\n",
      "　x\n",
      "柯P x\n",
      "否認 v\n"
     ]
    }
   ],
   "source": [
    "import jieba.posseg as pseg\n",
    "sentence = \"大巨蛋案對市府同仁下封口令？　柯P否認\"\n",
    "words = pseg.cut(sentence)\n",
    "\n",
    "for word in words:\n",
    "    print word.word, word.flag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "大巨蛋 0 3\n",
      "案對 3 5\n",
      "市府 5 7\n",
      "同仁 7 9\n",
      "下 9 10\n",
      "封口令 10 13\n",
      "？ 13 14\n",
      "　14 15\n",
      "柯P 15 17\n",
      "否認 17 19\n"
     ]
    }
   ],
   "source": [
    "words = jieba.tokenize(unicode(sentence, 'utf-8'))\n",
    "\n",
    "for tw in words:\n",
    "    print tw[0], tw[1], tw[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "封口令\n"
     ]
    }
   ],
   "source": [
    "import jieba.analyse\n",
    "tags = jieba.analyse.extract_tags(sentence, 1)\n",
    "print \",\".join(tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "由身\n",
      "負聯\n",
      "合作\n",
      "戰指揮官\n",
      "之\n",
      "參謀\n",
      "總長\n",
      "主導下\n"
     ]
    }
   ],
   "source": [
    "sentence = \"由身負聯合作戰指揮官之參謀總長主導下\"\n",
    "words = jieba.cut(sentence)\n",
    "for word in words:\n",
    "    print word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "柯p\n",
      "洪智坤\n"
     ]
    }
   ],
   "source": [
    "ary = ['【更新】柯P：洪智坤洩漏公文案還沒看到公文　今處理',\n",
    "'留洪智坤 柯：殘障求職不易',\n",
    "'人事處議處洪智坤　柯P：不清楚議處結果']\n",
    "corpus = []\n",
    "for title in ary:\n",
    "    corpus.append(' '.join(jieba.cut(title)))\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer()\n",
    "X = vectorizer.fit_transform(corpus)         \n",
    "for i in vectorizer.get_feature_names():\n",
    "    print i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.70710678  0.70710678]\n",
      " [ 0.          0.        ]\n",
      " [ 0.70710678  0.70710678]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer()    \n",
    "transformer = TfidfTransformer()\n",
    "tfidf = transformer.fit_transform(vectorizer.fit_transform(corpus))\n",
    "    \n",
    "word = vectorizer.get_feature_names() \n",
    "weight = tfidf.toarray()    \n",
    "print weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "No module named scipy",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-f916ca63b166>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpairwise\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mlinear_kernel\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mcosine_similarities\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlinear_kernel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtfidf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtfidf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflatten\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Python27\\lib\\site-packages\\sklearn\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     55\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     56\u001b[0m     \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0m__check_build\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 57\u001b[1;33m     \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mbase\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mclone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     58\u001b[0m     \u001b[0m__check_build\u001b[0m  \u001b[1;31m# avoid flakes unused variable error\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     59\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Python27\\lib\\site-packages\\sklearn\\base.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 10\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mscipy\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0msparse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     11\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mexternals\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0msix\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mImportError\u001b[0m: No module named scipy"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics.pairwise import linear_kernel\n",
    "cosine_similarities = linear_kernel(tfidf[0:1], tfidf).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import jieba\n",
    "import jieba.analyse\n",
    "section = requests.get('http://www.appledaily.com.tw/realtimenews/section/new/1')\n",
    "soup = BeautifulSoup(section.text)\n",
    "domain = 'http://www.appledaily.com.tw'\n",
    "worddic = {}\n",
    "for news in soup.select('.rtddt'):    \n",
    "    main = news.select('h1')[0].text\n",
    "    m = re.match('(.*)\\((\\d+)\\)',main)\n",
    "    if m:\n",
    "        title = m.group(1)\n",
    "        popularity = int(m.group(2))\n",
    "    for wd in jieba.analyse.extract_tags(title):\n",
    "        if wd not in worddic:\n",
    "            worddic[wd] = 1\n",
    "        else:\n",
    "            worddic[wd] = worddic[wd] + 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <head>\n",
       "    <title>TagCanvas example</title>\n",
       "    <script src=\"tagcanvas.min.js\" type=\"text/javascript\"></script>\n",
       "  </head>\n",
       "  <body>\n",
       "    <h1>TagCanvas example page</h1>\n",
       "    <div id=\"myCanvasContainer\">\n",
       "      <canvas width=\"300\" height=\"300\" id=\"myCanvas\">\n",
       "        <p>Anything in here will be replaced on browsers that support the canvas element</p>\n",
       "      </canvas>\n",
       "    </div>\n",
       "    <div id=\"tags\"><ul id=\"weightTags\"><li><a data-weight=\"30\" style=\"font-size: 5.83ex\">小熊貓</a></li><li><a data-weight=\"14\" style=\"font-size: 2.83ex\">熊貓</a></li></ul></div>\n",
       "  </body>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "$.getScript(\"tagcanvas.min.js\", function () {\n",
       "\n",
       "TagCanvas.Start('myCanvas','weightTags', {\n",
       " textFont: null,\n",
       " textColour: null,\n",
       " weight: true,\n",
       " weightMode: 'both',\n",
       " textFont: 'Impact,\"Arial Black\",sans-serif',\n",
       " /* more options */\n",
       " \n",
       "});\n",
       "$('#tags').css('display','None');\n",
       "});\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import display, HTML \n",
    "\n",
    "page= '''\n",
    "  <head>\n",
    "    <title>TagCanvas example</title>\n",
    "    <script src=\"tagcanvas.min.js\" type=\"text/javascript\"></script>\n",
    "  </head>\n",
    "  <body>\n",
    "    <h1>TagCanvas example page</h1>\n",
    "    <div id=\"myCanvasContainer\">\n",
    "      <canvas width=\"300\" height=\"300\" id=\"myCanvas\">\n",
    "        <p>Anything in here will be replaced on browsers that support the canvas element</p>\n",
    "      </canvas>\n",
    "    </div>\n",
    "    <div id=\"tags\"><ul id=\"weightTags\">%s</ul></div>\n",
    "  </body>\n",
    "'''\n",
    "\n",
    "js = '''\n",
    "TagCanvas.Start('myCanvas','weightTags', {\n",
    " textFont: null,\n",
    " textColour: null,\n",
    " weight: true,\n",
    " weightMode: 'both',\n",
    " textFont: 'Impact,\"Arial Black\",sans-serif',\n",
    " /* more options */\n",
    " \n",
    "});\n",
    "$('#tags').css('display','None');\n",
    "'''\n",
    "js_libs = ['tagcanvas.min.js']\n",
    " \n",
    "import IPython\n",
    "from IPython.core.display import display_html, display_javascript, Javascript\n",
    "display_html(IPython.core.display.HTML(data=page%('<li><a data-weight=\"30\" style=\"font-size: 5.83ex\">小熊貓</a></li>\\\n",
    "<li><a data-weight=\"14\" style=\"font-size: 2.83ex\">熊貓</a></li>'.decode('utf-8'))))\n",
    "display_javascript( Javascript(data=js, lib= js_libs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "Invalid font name. Should be one of Nobile, Old Standard TT, Cantarell, Reenie Beanie, Cuprum, Molengo, Neucha, Philosopher, Yanone Kaffeesatz, Cardo, Neuton, Inconsolata, Crimson Text, Josefin Sans, Droid Sans, Lobster, IM Fell DW Pica, Vollkorn, Tangerine, Coustard, PT Sans Regular",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-9-ffe4ba143998>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[0mtags\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmake_tags\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mswd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmaxsize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m120\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m \u001b[0mcreate_tag_image\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtags\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wc.png'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m600\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m400\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfontname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'tw'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      9\u001b[0m \u001b[0mImage\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'wc.png'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Python27\\lib\\site-packages\\pytagcloud\\__init__.pyc\u001b[0m in \u001b[0;36mcreate_tag_image\u001b[1;34m(tags, output, size, background, layout, fontname, rectangular)\u001b[0m\n\u001b[0;32m    342\u001b[0m                                       \u001b[0msize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    343\u001b[0m                                       \u001b[0mfontname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfontname\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 344\u001b[1;33m                                       rectangular=rectangular)\n\u001b[0m\u001b[0;32m    345\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    346\u001b[0m     \u001b[0mimage_surface\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSurface\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msizeRect\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msizeRect\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mh\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mSRCALPHA\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m32\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Python27\\lib\\site-packages\\pytagcloud\\__init__.pyc\u001b[0m in \u001b[0;36m_draw_cloud\u001b[1;34m(tag_list, layout, size, fontname, rectangular)\u001b[0m\n\u001b[0;32m    273\u001b[0m     \u001b[0marea\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    274\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mtag\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtag_list\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 275\u001b[1;33m         \u001b[0mtag_sprite\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTag\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtag\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfontname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfontname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    276\u001b[0m         \u001b[0marea\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mtag_sprite\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmask\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcount\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    277\u001b[0m         \u001b[0mtag_sprites\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtag_sprite\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Python27\\lib\\site-packages\\pytagcloud\\__init__.pyc\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, tag, initial_position, fontname)\u001b[0m\n\u001b[0;32m     57\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrotation\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     58\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 59\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfont_spec\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mload_font\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfontname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     60\u001b[0m         self.font = font.Font(os.path.join(FONT_DIR,\n\u001b[0;32m     61\u001b[0m                                            self.font_spec['ttf']),\n",
      "\u001b[1;32mC:\\Python27\\lib\\site-packages\\pytagcloud\\__init__.pyc\u001b[0m in \u001b[0;36mload_font\u001b[1;34m(name)\u001b[0m\n\u001b[0;32m    100\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mfont\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    101\u001b[0m     raise AttributeError('Invalid font name. Should be one of %s' % \n\u001b[1;32m--> 102\u001b[1;33m                          \", \".join([f['name'] for f in FONT_CACHE]))\n\u001b[0m\u001b[0;32m    103\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    104\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mdefscale\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcount\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmincount\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmaxcount\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mminsize\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmaxsize\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: Invalid font name. Should be one of Nobile, Old Standard TT, Cantarell, Reenie Beanie, Cuprum, Molengo, Neucha, Philosopher, Yanone Kaffeesatz, Cardo, Neuton, Inconsolata, Crimson Text, Josefin Sans, Droid Sans, Lobster, IM Fell DW Pica, Vollkorn, Tangerine, Coustard, PT Sans Regular"
     ]
    }
   ],
   "source": [
    "from pytagcloud import create_tag_image, make_tags\n",
    "from IPython.display import Image\n",
    "from operator import itemgetter\n",
    "swd = sorted(worddic.iteritems(), key=itemgetter(1), reverse=True)\n",
    "swd = [w for w in swd[1:50] if w[0] not in '【】...'.decode('utf-8')]\n",
    "\n",
    "tags = make_tags(swd, maxsize=120)\n",
    "create_tag_image(tags, 'wc.png', size=(600,400), fontname='tw')\n",
    "Image(filename='wc.png') \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}