研究方法亂亂做.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.request as res \n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import bs4\n",
    "import pandas as pd\n",
    "import csv\n",
    "import jieba\n",
    "import jieba.posseg as pseg\n",
    "import sklearn\n",
    "import jieba.analyse\n",
    "from sklearn import feature_extraction\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "csvfile=\"PythonUrl.csv\"\n",
    "for i in range(1):\n",
    "    url=\"https://www.businesstoday.com.tw/group_search/article?keywords=%E8%81%B7%E6%A5%AD%E8%82%A1%E6%9D%B1\"\n",
    "    req=res.Request(url, headers={\"user-agent\":\"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36\"})\n",
    "    with res.urlopen(req) as response:\n",
    "        data=response.read().decode(\"utf-8\")\n",
    "    root=bs4.BeautifulSoup(data,\"lxml\")#將data丟給beautifulsoup它會用 lxml解析爬下來的資料\n",
    "    titles=root.find_all(\"h4\", class_=\"searchitem__title\")#從網頁資訊尋找標籤 要尋找的標籤為h1 篩選條件為 class=title\n",
    "    #find_all: 只有find的話只會抓取符合條件的第一筆資料\n",
    "    with open(csvfile, 'a',newline='',encoding=\"utf-8\") as fp:\n",
    "        #打開一開始建立的csv檔案 把爬到的網頁放進去 用encoding=\"utf-8\"編碼\n",
    "        #'a':打開一個文件 如果該文件已經存在 新的內容會被寫道已有內容後 如果文件不存在 創建新文件寫入\n",
    "        #newlin3=''函數: 為了讓資料中包含的換行字元能夠被解析\n",
    "        writer=csv.writer(fp)\n",
    "        #建立CSV檔寫入器\n",
    "        for title in titles:#建立迴圈從第一筆網址開始抓取\n",
    "            titleUrl=[]#建立一個空群集將資料放入\n",
    "            titleUrl.append(title.a.get('href'))#append()函數:將資料放入群集\n",
    "                                                #get()函數:取出指定資料\n",
    "            writer.writerow(titleUrl)#writerrow()寫入資料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "新聞標題: 職業股東 - 今周刊\n",
      "關鍵字:\n",
      "25 0.15811388300841897\n",
      "ceo 0.15811388300841897\n",
      "一個 0.15811388300841897\n",
      "不已 0.15811388300841897\n",
      "不要命 0.15811388300841897\n",
      "中油 0.15811388300841897\n",
      "什麼當 0.15811388300841897\n",
      "他們 0.15811388300841897\n",
      "光電所養 0.15811388300841897\n",
      "入骨 0.15811388300841897\n",
      "劉世芳 0.15811388300841897\n",
      "半月刊 0.15811388300841897\n",
      "啟源 0.15811388300841897\n",
      "圓桌 0.15811388300841897\n",
      "基金 0.15811388300841897\n",
      "大老 0.15811388300841897\n",
      "天下 0.15811388300841897\n",
      "家臣 0.15811388300841897\n",
      "小股 0.15811388300841897\n",
      "工研院 0.15811388300841897\n",
      "心路 0.15811388300841897\n",
      "忠瑞 0.15811388300841897\n",
      "拼命 0.15811388300841897\n",
      "故事 0.15811388300841897\n",
      "日本 0.15811388300841897\n",
      "時機 0.15811388300841897\n",
      "會議 0.15811388300841897\n",
      "東愛之 0.15811388300841897\n",
      "楊承清 0.15811388300841897\n",
      "歷程 0.15811388300841897\n",
      "殷允芃 0.15811388300841897\n",
      "每個 0.15811388300841897\n",
      "永泰 0.15811388300841897\n",
      "現在 0.15811388300841897\n",
      "痛恨 0.15811388300841897\n",
      "第三季 0.15811388300841897\n",
      "股行情 0.15811388300841897\n",
      "董事 0.15811388300841897\n",
      "賣命 0.15811388300841897\n",
      "邁向 0.15811388300841897\n",
      "新聞標題: 沒了職業股東  來了工會代表 P.39 - 今周刊\n",
      "關鍵字:\n",
      "不滿 0.19245008972987526\n",
      "主管 0.19245008972987526\n",
      "二十八 0.19245008972987526\n",
      "今年 0.19245008972987526\n",
      "勞工 0.19245008972987526\n",
      "同一天 0.19245008972987526\n",
      "因為 0.19245008972987526\n",
      "基層 0.19245008972987526\n",
      "心懷 0.19245008972987526\n",
      "情形 0.19245008972987526\n",
      "或許 0.19245008972987526\n",
      "把握 0.19245008972987526\n",
      "早就 0.19245008972987526\n",
      "機會 0.19245008972987526\n",
      "機構 0.19245008972987526\n",
      "減少 0.19245008972987526\n",
      "熱鬧 0.19245008972987526\n",
      "職業 0.19245008972987526\n",
      "股東 0.19245008972987526\n",
      "股東會 0.3849001794597505\n",
      "舉辦 0.19245008972987526\n",
      "金融 0.19245008972987526\n",
      "高階 0.19245008972987526\n",
      "鬧場 0.19245008972987526\n",
      "新聞標題: 每年都要紅一次的職業股東 P.54 - 今周刊\n",
      "關鍵字:\n",
      "一位 0.2\n",
      "之一 0.2\n",
      "兩人 0.2\n",
      "參加 0.2\n",
      "同時 0.2\n",
      "媒體 0.2\n",
      "對話 0.2\n",
      "就是 0.2\n",
      "曹興 0.2\n",
      "東會 0.2\n",
      "樂趣 0.2\n",
      "欣賞 0.2\n",
      "每年 0.2\n",
      "現在 0.2\n",
      "精采 0.2\n",
      "聯電 0.2\n",
      "聯電股 0.2\n",
      "股東 0.4\n",
      "董事 0.2\n",
      "記者 0.2\n",
      "資深 0.2\n",
      "阿土 0.2\n",
      "新聞標題: 工運龍頭變成職業股東 P.51 - 今周刊\n",
      "關鍵字:\n",
      "cd 0.12403473458920847\n",
      "一位 0.12403473458920847\n",
      "一定 0.12403473458920847\n",
      "上市公司 0.12403473458920847\n",
      "中環 0.12403473458920847\n",
      "之一 0.12403473458920847\n",
      "了解 0.12403473458920847\n",
      "今年 0.24806946917841693\n",
      "光碟 0.12403473458920847\n",
      "公司 0.3721042037676254\n",
      "出席 0.12403473458920847\n",
      "出現 0.12403473458920847\n",
      "利碟 0.24806946917841693\n",
      "唯讀 0.12403473458920847\n",
      "問題 0.24806946917841693\n",
      "單刀 0.12403473458920847\n",
      "嚴厲 0.12403473458920847\n",
      "姚國建 0.24806946917841693\n",
      "姚老師 0.24806946917841693\n",
      "崛起 0.12403473458920847\n",
      "差一點 0.12403473458920847\n",
      "市場 0.12403473458920847\n",
      "批評 0.12403473458920847\n",
      "招架不住 0.12403473458920847\n",
      "提出 0.12403473458920847\n",
      "明顯 0.24806946917841693\n",
      "本身 0.12403473458920847\n",
      "東會 0.12403473458920847\n",
      "營運 0.12403473458920847\n",
      "環股 0.12403473458920847\n",
      "發起 0.12403473458920847\n",
      "直入 0.12403473458920847\n",
      "競爭 0.12403473458920847\n",
      "美齊 0.12403473458920847\n",
      "股東會 0.24806946917841693\n",
      "董事 0.12403473458920847\n",
      "新聞標題: 職業股東　閉上尊嘴！ P.56 - 今周刊\n",
      "關鍵字:\n",
      "一個 0.1796053020267749\n",
      "上市 0.1796053020267749\n",
      "公司 0.3592106040535498\n",
      "兵來 0.1796053020267749\n",
      "其實 0.1796053020267749\n",
      "千奇百怪 0.1796053020267749\n",
      "反制 0.1796053020267749\n",
      "可以 0.1796053020267749\n",
      "土掩 0.1796053020267749\n",
      "字來 0.1796053020267749\n",
      "將擋 0.1796053020267749\n",
      "形容 0.1796053020267749\n",
      "很多 0.1796053020267749\n",
      "所謂 0.1796053020267749\n",
      "招數 0.1796053020267749\n",
      "方法 0.1796053020267749\n",
      "水來 0.1796053020267749\n",
      "法度 0.1796053020267749\n",
      "真的 0.1796053020267749\n",
      "簡直 0.1796053020267749\n",
      "職業 0.3592106040535498\n",
      "股東 0.1796053020267749\n",
      "股東來 0.1796053020267749\n",
      "面對 0.1796053020267749\n",
      "鬧場 0.1796053020267749\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-4-6c506dda9999>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     15\u001b[0m         \u001b[0mrows\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcsv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcsvfile\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;31m#讀檔\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mrow\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrows\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;31m#建立迴圈 依次抓取網址出來\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m             \u001b[0mreq\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mres\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mRequest\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m{\u001b[0m\u001b[1;34m\"user-agent\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;34m\"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36\"\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     18\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mres\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0murlopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;31m#打開網站\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m                 \u001b[0mdata\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"utf-8\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;31m#抓取資訊\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "import urllib.request as res \n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import bs4\n",
    "import pandas as pd\n",
    "import csv\n",
    "import jieba\n",
    "import jieba.posseg as pseg\n",
    "import sklearn\n",
    "import jieba.analyse\n",
    "from sklearn import feature_extraction\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "with open('PythonUrl2.csv', newline='') as csvfile:#打開PythonUrl.csv\n",
    "        rows = csv.reader(csvfile)#讀檔\n",
    "        for row in rows:#建立迴圈 依次抓取網址出來\n",
    "            req=res.Request(row[0], headers={\"user-agent\":\"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36\"})\n",
    "            with res.urlopen(req) as response:#打開網站\n",
    "                data=response.read().decode(\"utf-8\")#抓取資訊\n",
    "            root=bs4.BeautifulSoup(data,\"lxml\")#將data丟給beautifulsoup 它會用lxml幫我們解析爬下來的網頁資訊\n",
    "            contents=root.find_all(\"div\", class_=\"article__introduction\")#find:這次只需要找一筆資料 就是新聞內容\n",
    "        #從網頁資訊尋找標籤 要尋找的標籤為div 篩選條件為 class=indent\n",
    "            newstitle=root.find(\"title\")#抓取新聞標題\n",
    "            for content in contents:#建立迴圈\n",
    "                seg_list=jieba.cut(content.get_text(), cut_all=False)#jueba.cut()函數 分割句子成有意義的字詞元素\n",
    "                                                                    #cut_all=False:精確模式，將句子最精確地切開，以便文本分析\n",
    "                contentList_1=(\" \".join(seg_list))#join()函數:序列中的元素以指定的字詞連接生成一個新的字詞串 每增加一個字詞元素就加一個空白\n",
    "                contentList=[]#建立一個空群集待會將資料放入\n",
    "                contentList.append(contentList_1)#append()函數:將資料放入群集\n",
    "                vectorizer=CountVectorizer()#該函數會將文本中的詞語轉換為詞頻矩陣，矩陣元素a[i][j] 表示j詞在i類文本下的詞頻\n",
    "                transformer=TfidfTransformer()#該函數會統計每個詞語的tf-idf權值\n",
    "                tfidf=transformer.fit_transform(vectorizer.fit_transform(contentList))\n",
    "                     #第一個fit_transform是計算tf-idf，第二個fit_transform是將contentList轉為詞頻矩陣\n",
    "                word=vectorizer.get_feature_names()#獲取詞袋模型中的所有字詞\n",
    "                weight=tfidf.toarray()#將tf-idf矩陣抽取出來利用.toarray()函數將轉成陣列weight，元素a[i][j]表示j詞在i類文本中的tf-idf權重\n",
    "                for i in range(len(weight)):#打印每類文本的tf-idf詞語權重，第一個for遍歷所有文本，\n",
    "                                            #len()函數:抓出weight陣列長度得知文本數量決定迴圈大小\n",
    "                    print(\"新聞標題:\",newstitle.string)\n",
    "                                            #.string:將新聞標題的文字印出來 不加string的話會有其他標籤\n",
    "                    print(\"關鍵字:\")\n",
    "                    for j in range(len(word)):#第二個for抓出某一類文本下的詞語權重\n",
    "                                            #len()函數:抓出一文本內字詞數量得知決定迴圈大小\n",
    "                       print (word[j],weight[i][j])\n",
    "                  \n",
    "               \n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}