chap11/02_bandit_algorithm_compare.ipynb

{
 "cells": [
  {
   "source": [
    "<a href=\"https://colab.research.google.com/github/oreilly-japan/ml-at-work/blob/master/chap11/02_bandit_algorithm_compare.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "![表紙](https://www.oreilly.co.jp/books/images/picture978-4-87311-947-2.gif)\n",
    "\n",
    "このノートブックはオライリー・ジャパンより発行の書籍[『仕事ではじめる機械学習 第2版』](https://www.oreilly.co.jp/books/9784873119472/)のサンプルコードです。コードの解説等は書籍をご参照ください。なお、このコードを動作させた結果について、著者およびオライリー・ジャパンは一切の責任を負いません。"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 11.7 各種バンディットアルゴリズムの比較"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy\n",
    "import scipy.stats\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_strategy(success_counts, fail_counts):\n",
    "    return np.random.choice(len(success_counts))\n",
    "\n",
    "def baysian_ucb_strategy(success_counts, fail_counts, q=0.95):\n",
    "    score = scipy.stats.beta.ppf(q, success_counts + 1, fail_counts + 1)\n",
    "    return np.argmax(score)\n",
    "\n",
    "def ucb1_strategy(success_counts, fail_counts):\n",
    "    mean = (success_counts) / (success_counts + fail_counts)\n",
    "    total_count = np.sum(success_counts + fail_counts)\n",
    "    ucb = (\n",
    "            2.0 * np.log(total_count) / (success_counts + fail_counts)\n",
    "          ) ** 0.5\n",
    "    score = mean + ucb\n",
    "    return np.argmax(score)\n",
    "\n",
    "def softmax_strategy(success_counts, fail_counts, t=0.05):\n",
    "    mean = (success_counts) / (success_counts + fail_counts)\n",
    "    select_rate = np.exp(mean / t)/ np.sum(np.exp(mean / t))\n",
    "    return np.random.choice(len(select_rate), p=select_rate)\n",
    "\n",
    "def softmax_annealing_strategy(success_counts, fail_counts, initial_t=0.1, k=100.0):\n",
    "    mean = (success_counts) / (success_counts + fail_counts)\n",
    "    t = initial_t / np.log(k * np.sum(success_counts + fail_counts) + 2)\n",
    "    select_rate = np.exp(mean / t) / np.sum(np.exp(mean / t))\n",
    "    return np.random.choice(len(select_rate), p=select_rate)\n",
    "\n",
    "def thompson_sampling_strategy(success_counts, fail_counts):\n",
    "    score = scipy.stats.beta.rvs(success_counts + 1, fail_counts + 1)\n",
    "    return np.argmax(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "actual_cvr = [0.12, 0.11, 0.10]\n",
    "bandit_round = 10000\n",
    "#random_seed = 1234567\n",
    "random_seed = 1234\n",
    "\n",
    "strategy_list = [\n",
    "    (\"Random\", random_strategy),\n",
    "    (\"Baysian_UCB\", baysian_ucb_strategy),\n",
    "    (\"UCB1\", ucb1_strategy),\n",
    "    (\"Softmax\", softmax_strategy),\n",
    "    (\"Softmax_annealing\", softmax_annealing_strategy),\n",
    "    (\"Thompson_sampling\", thompson_sampling_strategy)\n",
    "]\n",
    "\n",
    "scores = []\n",
    "arm1_select_rates = []\n",
    "\n",
    "for name, select_arm_method in strategy_list:\n",
    "    # random seedを初期化する\n",
    "    # scipyはnumpyの乱数を使っているので、scipy側もこれで初期化できる\n",
    "    np.random.seed(random_seed)\n",
    "\n",
    "    success_counts = np.array([0.0, 0.0, 0.0])\n",
    "    fail_counts = np.array([0.0, 0.0, 0.0])\n",
    "    \n",
    "    scores.append([])\n",
    "    arm1_select_rates.append([])\n",
    "    \n",
    "    for i in range(bandit_round):\n",
    "        if i < 1000: # 最初の1000ラウンドはランダムに配信する\n",
    "            selected_arm = random_strategy(success_counts, fail_counts)\n",
    "        else:\n",
    "            selected_arm = select_arm_method(success_counts, fail_counts)\n",
    "\n",
    "        # 選んだアームがコンバージョンしていたかどうかを判定\n",
    "        if np.random.rand() < actual_cvr[selected_arm]:\n",
    "            success_counts[selected_arm] += 1\n",
    "        else:\n",
    "            fail_counts[selected_arm] += 1\n",
    "        \n",
    "        score = np.sum(success_counts) / np.sum(success_counts + fail_counts)\n",
    "        scores[-1].append(score)\n",
    "        \n",
    "        arm1_select_rate = (success_counts[0] + fail_counts[0]) / np.sum(success_counts + fail_counts)\n",
    "        arm1_select_rates[-1].append(arm1_select_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(dpi=400)\n",
    "for i in range(len(strategy_list)):\n",
    "    algorithm_name = strategy_list[i][0] \n",
    "    print(algorithm_name, scores[i][-1])\n",
    "    plt.plot(scores[i], label=algorithm_name)\n",
    "    plt.ylim(0.0, 0.2)\n",
    "\n",
    "plt.legend(bbox_to_anchor=(1, 1), loc=\"upper left\")\n",
    "plt.show()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(dpi=400)\n",
    "for i in range(len(strategy_list)):\n",
    "    algorithm_name = strategy_list[i][0] \n",
    "    print(algorithm_name, arm1_select_rates[i][-1])\n",
    "    plt.plot(arm1_select_rates[i], label=algorithm_name)\n",
    "    plt.ylim(0, 1)\n",
    "\n",
    "plt.legend(bbox_to_anchor=(1, 1), loc=\"upper left\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}