sample-content-based.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "84de56a8",
   "metadata": {},
   "source": [
    "# Sample Code"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "688675f5",
   "metadata": {},
   "source": [
    "## 基礎建設"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b32226db",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/wei/.virtualenvs/py3/lib/python3.6/site-packages/pandas/compat/__init__.py:120: UserWarning: Could not import the lzma module. Your installed Python is incomplete. Attempting to use lzma compression will result in a RuntimeError.\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import gzip, json\n",
    "\n",
    "def parse(path):\n",
    "    g = gzip.open(path, 'rb')\n",
    "    for l in g:\n",
    "        yield json.loads(l)\n",
    "\n",
    "def getDF(path):\n",
    "    i = 0\n",
    "    df = {}\n",
    "    for d in parse(path):\n",
    "        df[i] = d\n",
    "        i += 1\n",
    "    return pd.DataFrame.from_dict(df, orient='index')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84ff0f9d",
   "metadata": {},
   "source": [
    "## 載入資料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9402e9c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget http://deepyeti.ucsd.edu/jianmo/amazon/categoryFilesSmall/All_Beauty.csv\n",
    "!wget http://deepyeti.ucsd.edu/jianmo/amazon/metaFiles2/meta_All_Beauty.json.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "84b66cc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata = getDF('/content/meta_All_Beauty.json.gz')\n",
    "ratings = pd.read_csv('/content/All_Beauty.csv', names=['asin', 'reviewerID', 'overall', 'unixReviewTime'], header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9fd77203",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>tech1</th>\n",
       "      <th>description</th>\n",
       "      <th>fit</th>\n",
       "      <th>title</th>\n",
       "      <th>also_buy</th>\n",
       "      <th>tech2</th>\n",
       "      <th>brand</th>\n",
       "      <th>feature</th>\n",
       "      <th>rank</th>\n",
       "      <th>also_view</th>\n",
       "      <th>details</th>\n",
       "      <th>main_cat</th>\n",
       "      <th>similar_item</th>\n",
       "      <th>date</th>\n",
       "      <th>price</th>\n",
       "      <th>asin</th>\n",
       "      <th>imageURL</th>\n",
       "      <th>imageURLHighRes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td>[Loud 'N Clear Personal Sound Amplifier allows...</td>\n",
       "      <td></td>\n",
       "      <td>Loud 'N Clear&amp;trade; Personal Sound Amplifier</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td>idea village</td>\n",
       "      <td>[]</td>\n",
       "      <td>2,938,573 in Beauty &amp; Personal Care (</td>\n",
       "      <td>[]</td>\n",
       "      <td>{'ASIN: ': '6546546450'}</td>\n",
       "      <td>All Beauty</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>6546546450</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td>[No7 Lift &amp; Luminate Triple Action Serum 50ml ...</td>\n",
       "      <td></td>\n",
       "      <td>No7 Lift &amp;amp; Luminate Triple Action Serum 50...</td>\n",
       "      <td>[B01E7LCSL6, B008X5RVME]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[]</td>\n",
       "      <td>872,854 in Beauty &amp; Personal Care (</td>\n",
       "      <td>[]</td>\n",
       "      <td>{'Shipping Weight:': '0.3 ounces (', 'ASIN: ':...</td>\n",
       "      <td>All Beauty</td>\n",
       "      <td>class=\"a-bordered a-horizontal-stripes  a-spa...</td>\n",
       "      <td></td>\n",
       "      <td>$44.99</td>\n",
       "      <td>7178680776</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td>[No7 Stay Perfect Foundation now stays perfect...</td>\n",
       "      <td></td>\n",
       "      <td>No7 Stay Perfect Foundation Cool Vanilla by No7</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td>No7</td>\n",
       "      <td>[]</td>\n",
       "      <td>956,696 in Beauty &amp; Personal Care (</td>\n",
       "      <td>[B01B8BR0O8, B01B8BR0NO, B014MHXXM8]</td>\n",
       "      <td>{'Shipping Weight:': '3.5 ounces (', 'ASIN: ':...</td>\n",
       "      <td>All Beauty</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>$28.76</td>\n",
       "      <td>7250468162</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td>Wella Koleston Perfect Hair Colour 44/44 Mediu...</td>\n",
       "      <td>[B0041PBXX8]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[]</td>\n",
       "      <td>1,870,258 in Beauty &amp; Personal Care (</td>\n",
       "      <td>[]</td>\n",
       "      <td>{'\n",
       "    Item Weight: \n",
       "    ': '1.76 ounces', 'Sh...</td>\n",
       "      <td>All Beauty</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>7367905066</td>\n",
       "      <td>[https://images-na.ssl-images-amazon.com/image...</td>\n",
       "      <td>[https://images-na.ssl-images-amazon.com/image...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td>[Lacto Calamine Skin Balance Daily Nourishing ...</td>\n",
       "      <td></td>\n",
       "      <td>Lacto Calamine Skin Balance Oil control 120 ml...</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td>Pirmal Healthcare</td>\n",
       "      <td>[]</td>\n",
       "      <td>67,701 in Beauty &amp; Personal Care (</td>\n",
       "      <td>[3254895630, B007VL1D9S, B00EH9A0RI, B0773MBG4...</td>\n",
       "      <td>{'Shipping Weight:': '12 ounces (', 'ASIN: ': ...</td>\n",
       "      <td>All Beauty</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>$12.15</td>\n",
       "      <td>7414204790</td>\n",
       "      <td>[https://images-na.ssl-images-amazon.com/image...</td>\n",
       "      <td>[https://images-na.ssl-images-amazon.com/image...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  category tech1                                        description fit  \\\n",
       "0       []        [Loud 'N Clear Personal Sound Amplifier allows...       \n",
       "1       []        [No7 Lift & Luminate Triple Action Serum 50ml ...       \n",
       "2       []        [No7 Stay Perfect Foundation now stays perfect...       \n",
       "3       []                                                       []       \n",
       "4       []        [Lacto Calamine Skin Balance Daily Nourishing ...       \n",
       "\n",
       "                                               title  \\\n",
       "0      Loud 'N Clear&trade; Personal Sound Amplifier   \n",
       "1  No7 Lift &amp; Luminate Triple Action Serum 50...   \n",
       "2    No7 Stay Perfect Foundation Cool Vanilla by No7   \n",
       "3  Wella Koleston Perfect Hair Colour 44/44 Mediu...   \n",
       "4  Lacto Calamine Skin Balance Oil control 120 ml...   \n",
       "\n",
       "                   also_buy tech2              brand feature  \\\n",
       "0                        []             idea village      []   \n",
       "1  [B01E7LCSL6, B008X5RVME]                               []   \n",
       "2                        []                      No7      []   \n",
       "3              [B0041PBXX8]                               []   \n",
       "4                        []        Pirmal Healthcare      []   \n",
       "\n",
       "                                    rank  \\\n",
       "0  2,938,573 in Beauty & Personal Care (   \n",
       "1    872,854 in Beauty & Personal Care (   \n",
       "2    956,696 in Beauty & Personal Care (   \n",
       "3  1,870,258 in Beauty & Personal Care (   \n",
       "4     67,701 in Beauty & Personal Care (   \n",
       "\n",
       "                                           also_view  \\\n",
       "0                                                 []   \n",
       "1                                                 []   \n",
       "2               [B01B8BR0O8, B01B8BR0NO, B014MHXXM8]   \n",
       "3                                                 []   \n",
       "4  [3254895630, B007VL1D9S, B00EH9A0RI, B0773MBG4...   \n",
       "\n",
       "                                             details    main_cat  \\\n",
       "0                           {'ASIN: ': '6546546450'}  All Beauty   \n",
       "1  {'Shipping Weight:': '0.3 ounces (', 'ASIN: ':...  All Beauty   \n",
       "2  {'Shipping Weight:': '3.5 ounces (', 'ASIN: ':...  All Beauty   \n",
       "3  {'\n",
       "    Item Weight: \n",
       "    ': '1.76 ounces', 'Sh...  All Beauty   \n",
       "4  {'Shipping Weight:': '12 ounces (', 'ASIN: ': ...  All Beauty   \n",
       "\n",
       "                                        similar_item date   price        asin  \\\n",
       "0                                                                  6546546450   \n",
       "1   class=\"a-bordered a-horizontal-stripes  a-spa...       $44.99  7178680776   \n",
       "2                                                          $28.76  7250468162   \n",
       "3                                                                  7367905066   \n",
       "4                                                          $12.15  7414204790   \n",
       "\n",
       "                                            imageURL  \\\n",
       "0                                                 []   \n",
       "1                                                 []   \n",
       "2                                                 []   \n",
       "3  [https://images-na.ssl-images-amazon.com/image...   \n",
       "4  [https://images-na.ssl-images-amazon.com/image...   \n",
       "\n",
       "                                     imageURLHighRes  \n",
       "0                                                 []  \n",
       "1                                                 []  \n",
       "2                                                 []  \n",
       "3  [https://images-na.ssl-images-amazon.com/image...  \n",
       "4  [https://images-na.ssl-images-amazon.com/image...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b402792c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>overall</th>\n",
       "      <th>unixReviewTime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0143026860</td>\n",
       "      <td>A1V6B6TNIC10QE</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1424304000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0143026860</td>\n",
       "      <td>A2F5GHSXFQ0W6J</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1418860800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0143026860</td>\n",
       "      <td>A1572GUYS7DGSR</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1407628800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0143026860</td>\n",
       "      <td>A1PSGLFK1NSVO</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1362960000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0143026860</td>\n",
       "      <td>A6IKXKZMTKGSC</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1324771200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         asin      reviewerID  overall  unixReviewTime\n",
       "0  0143026860  A1V6B6TNIC10QE      1.0      1424304000\n",
       "1  0143026860  A2F5GHSXFQ0W6J      4.0      1418860800\n",
       "2  0143026860  A1572GUYS7DGSR      4.0      1407628800\n",
       "3  0143026860   A1PSGLFK1NSVO      5.0      1362960000\n",
       "4  0143026860   A6IKXKZMTKGSC      5.0      1324771200"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "695362e3",
   "metadata": {},
   "source": [
    "## 資料整理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b552fb23",
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings['DATE'] = pd.to_datetime(ratings['unixReviewTime'], unit='s')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "054eeba4",
   "metadata": {},
   "source": [
    "## 資料切分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "284b14a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_trainings = ratings[\n",
    "    (ratings['DATE'] < '2018-09-01')\n",
    "]\n",
    "ratings_testings = ratings[\n",
    "    (ratings['DATE'] >= '2018-09-01') & \n",
    "    (ratings['DATE'] <= '2018-09-30')\n",
    "]\n",
    "ratings_testings_by_user = ratings_testings.groupby('reviewerID').agg(list).reset_index()[['reviewerID', 'asin']].to_dict('records')\n",
    "ratings_testings_by_user = { rating['reviewerID']: rating['asin'] for rating in ratings_testings_by_user }\n",
    "users = list(ratings_testings_by_user.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6c76061",
   "metadata": {},
   "source": [
    "## 產生推薦"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61e7487b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "# 計算商品用標題所表示的 tfidf 矩陣\n",
    "df = metadata.drop_duplicates('title')\n",
    "tf = TfidfVectorizer(analyzer='word')\n",
    "tfidf_matrix = tf.fit_transform(df['title'])\n",
    "\n",
    "# 計算商品間的相似程度\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "similarity_matrix = cosine_similarity(tfidf_matrix)\n",
    "mapping = pd.Series(df.index,index = df['title'])\n",
    "\n",
    "# 每個商品回傳 k 個最相近的商品\n",
    "def recommend_item(item_input, k=2):\n",
    "    try:\n",
    "        item_index = mapping[item_input]\n",
    "        similarity_score = list(enumerate(similarity_matrix[item_index]))\n",
    "        similarity_score = sorted(similarity_score, key=lambda x: x[1], reverse=True)\n",
    "        similarity_score = similarity_score[:k]\n",
    "        item_indices = [i[0] for i in similarity_score]\n",
    "        return (df['asin'].iloc[item_indices].tolist())\n",
    "    except:\n",
    "        return []\n",
    "\n",
    "# 利用使用者購買過的商品產生推薦\n",
    "def recommend_items(items, k):\n",
    "    res = []\n",
    "    for d in items:\n",
    "        res.extend(recommend_item(d, k))\n",
    "    return res\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9280a043",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def recommender(training_data, users=[], k=10):\n",
    "    '''\n",
    "    * training_data: dataframe 輸入的訓練資料集（2018-09-01 以前資料）\n",
    "    * users: [] 需要被推薦的使用者\n",
    "    * k: int 每個使用者需要推薦的商品數\n",
    "    * recommendations: dict\n",
    "      {\n",
    "          使用者一： [推薦商品一, 推薦商品二, ...],\n",
    "          使用者二： [...], ...\n",
    "      }\n",
    "    '''\n",
    "    recommendations = {}\n",
    "    '''\n",
    "    content-based\n",
    "    '''\n",
    "    ratings_trainings = training_data\n",
    "    recommendations = {user: recommend_items(metadata[metadata['asin'].isin(ratings_trainings[ratings_trainings['reviewerID'] == user]['asin'].tolist())]['title'].tolist(), k) for user in users}\n",
    "    return recommendations\n",
    "\n",
    "ratings_by_user = recommender(ratings_trainings, users)\n",
    "ratings_by_user"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04a4ca9e",
   "metadata": {},
   "source": [
    "## 結果評估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ee763747",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def evaluate(ratings_testings_by_user={}, ratings_by_user={}, method=None):\n",
    "    '''\n",
    "    * ratings_testings_by_user: dict 真實被購買的商品資料（2018-09-01 以後資料）\n",
    "    * ratings_by_user: dict 利用訓練資料學習的推薦商品\n",
    "    * method: str\n",
    "    * score: float\n",
    "    '''\n",
    "    total = 0\n",
    "    for d in ratings_testings_by_user:\n",
    "        if d in ratings_by_user:\n",
    "            total += len(set(ratings_by_user[d]) & set(ratings_testings_by_user[d]))\n",
    "\n",
    "    score = total / len(ratings_testings)\n",
    "    return score\n",
    "\n",
    "evaluate(ratings_testings_by_user, ratings_by_user)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53055ac2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}