notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implementation of a Policy Gradient Model using Keras and a custom Gym environment\n",
    "Data used in this project can be downloaded from https://www.transtats.bts.gov/Fields.asp. The preprocessing of the data to create the custom gym environment is adapted from this paper https://ieeexplore.ieee.org/document/7411275. \n",
    "\n",
    "#### We will use the data from April 2019 in this demonstration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from datetime import datetime, date, time, timedelta\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "df = pd.read_csv('data.csv').drop('Unnamed: 24', axis=1)\n",
    "df = df[(df.CANCELLED == 0) & (df.DIVERTED == 0)].drop(\n",
    "    ['CANCELLED', 'DIVERTED'], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Processing\n",
    "X1 is going to be Taxi Out time, our target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['x1'] = df['TAXI_OUT']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Windowing\n",
    "To be able to achieve the windowing described in the paper we will need to convert DEP_TIME and ARR_TIME into a python datetime object. We will also use the date from FL_DATE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16.67% done.\n",
      "33.33% done.\n",
      "50.0% done.\n",
      "66.67% done.\n",
      "83.33% done.\n",
      "100.0% done.\n"
     ]
    }
   ],
   "source": [
    "def time_converter(in_):\n",
    "    in_date = in_[0]\n",
    "    in_time = in_[1]\n",
    "    in_time = str(in_time)\n",
    "    if '.' in in_time:\n",
    "        in_time = in_time[:-2]\n",
    "    while len(str(in_time)) < 4:\n",
    "        in_time = '0' + in_time\n",
    "    if in_time[:2] == '24':\n",
    "        in_time = '00' + in_time[2:]\n",
    "    time_ = time(int(in_time[:2]), int(in_time[2:4]), 0)\n",
    "    in_date = in_date.split('-')\n",
    "    date_ = date(int(in_date[0]), int(in_date[1]), int(in_date[2]))\n",
    "    return datetime.combine(date_, time_)\n",
    "\n",
    "\n",
    "time_cols = ['CRS_DEP_TIME', 'DEP_TIME', 'WHEELS_OFF',\n",
    "             'WHEELS_ON', 'CRS_ARR_TIME', 'ARR_TIME', ]\n",
    "\n",
    "for i, col in enumerate(time_cols):\n",
    "    df[col] = df[['FL_DATE', col]].apply(time_converter, axis=1)\n",
    "    print(f\"{round(100*(i+1)/len(time_cols),2)}% done.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0% done.\n",
      "4.0% done.\n",
      "6.0% done.\n",
      "8.0% done.\n",
      "10.0% done.\n",
      "12.0% done.\n",
      "14.0% done.\n",
      "16.0% done.\n",
      "18.0% done.\n",
      "20.0% done.\n",
      "22.0% done.\n",
      "24.0% done.\n",
      "26.0% done.\n",
      "28.0% done.\n",
      "30.0% done.\n",
      "32.0% done.\n",
      "34.0% done.\n",
      "36.0% done.\n",
      "38.0% done.\n",
      "40.0% done.\n",
      "42.0% done.\n",
      "44.0% done.\n",
      "46.0% done.\n",
      "48.0% done.\n",
      "50.0% done.\n",
      "52.0% done.\n",
      "54.0% done.\n",
      "56.0% done.\n",
      "58.0% done.\n",
      "60.0% done.\n",
      "62.0% done.\n",
      "64.0% done.\n",
      "66.0% done.\n",
      "68.0% done.\n",
      "70.0% done.\n",
      "72.0% done.\n",
      "74.0% done.\n",
      "76.0% done.\n",
      "78.0% done.\n",
      "80.0% done.\n",
      "82.0% done.\n",
      "84.0% done.\n",
      "86.0% done.\n",
      "88.0% done.\n",
      "90.0% done.\n",
      "92.0% done.\n",
      "94.0% done.\n",
      "96.0% done.\n",
      "98.0% done.\n"
     ]
    }
   ],
   "source": [
    "def get_bounds(in_time):\n",
    "    time_d = timedelta(minutes=20)\n",
    "    return (in_time - time_d),  (in_time + time_d)\n",
    "\n",
    "\n",
    "def window_func(TIME, variable, df):\n",
    "    lower_bound, upper_bound = get_bounds(TIME)\n",
    "    df_ = df[(df[variable] >= lower_bound) & (df[variable] <=\n",
    "                                              upper_bound)].copy()\n",
    "    count_airplanes = len(df_) - 1\n",
    "    return count_airplanes\n",
    "\n",
    "\n",
    "def pct_tracker(to_be_done):\n",
    "    done = 0\n",
    "    count = 0\n",
    "    while True:\n",
    "        yield\n",
    "        done += 1\n",
    "        count += 1\n",
    "        if done > to_be_done:\n",
    "            done -= to_be_done\n",
    "        if (count >= to_be_done * 0.02):\n",
    "            count = 0\n",
    "            pct = round((float(done) / float(to_be_done)) * 100, 2)\n",
    "            print(f\"{pct}% done.\")\n",
    "\n",
    "\n",
    "def track_progress(func=window_func, progress=pct_tracker(len(df))):\n",
    "    def call_func(*args, **kwargs):\n",
    "        progress.send(None)\n",
    "        return func(*args, **kwargs)\n",
    "    return call_func\n",
    "\n",
    "\n",
    "def airplane_counter(func, variable, airport_type, list, add_cols=[]):\n",
    "    listy = []\n",
    "    sub_df = df.sort_values(by=[airport_type, variable])[\n",
    "        [airport_type, variable] + add_cols].copy()\n",
    "    for i, airport in enumerate(list):\n",
    "        x = sub_df[sub_df[airport_type] == airport].copy()\n",
    "        x['target'] = x[variable].apply(\n",
    "            track_progress(func), args=([variable, x]))\n",
    "        listy.append(x)\n",
    "    series = pd.concat(listy).sort_values(by=[airport_type, variable]).target\n",
    "    return series\n",
    "\n",
    "\n",
    "df = df.sort_values(by=['ORIGIN', 'DEP_TIME']).copy()\n",
    "origins = df.ORIGIN.unique().tolist()\n",
    "\n",
    "x2 = airplane_counter(window_func, 'DEP_TIME', 'ORIGIN', origins)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Similarly, we will calculate the number of arrival aircrafts sharing the runway with the aircraft being considered in the +-20 minutes window (x3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.01% done.\n",
      "2.01% done.\n",
      "4.01% done.\n",
      "6.01% done.\n",
      "8.01% done.\n",
      "10.01% done.\n",
      "12.01% done.\n",
      "14.01% done.\n",
      "16.01% done.\n",
      "18.01% done.\n",
      "20.01% done.\n",
      "22.01% done.\n",
      "24.01% done.\n",
      "26.01% done.\n",
      "28.01% done.\n",
      "30.01% done.\n",
      "32.01% done.\n",
      "34.01% done.\n",
      "36.01% done.\n",
      "38.01% done.\n",
      "40.01% done.\n",
      "42.01% done.\n",
      "44.01% done.\n",
      "46.01% done.\n",
      "48.01% done.\n",
      "50.01% done.\n",
      "52.01% done.\n",
      "54.01% done.\n",
      "56.01% done.\n",
      "58.01% done.\n",
      "60.01% done.\n",
      "62.01% done.\n",
      "64.01% done.\n",
      "66.01% done.\n",
      "68.01% done.\n",
      "70.01% done.\n",
      "72.01% done.\n",
      "74.01% done.\n",
      "76.01% done.\n",
      "78.01% done.\n",
      "80.01% done.\n",
      "82.01% done.\n",
      "84.01% done.\n",
      "86.01% done.\n",
      "88.01% done.\n",
      "90.01% done.\n",
      "92.01% done.\n",
      "94.01% done.\n",
      "96.01% done.\n",
      "98.01% done.\n"
     ]
    }
   ],
   "source": [
    "df['x2'] = x2\n",
    "\n",
    "df = df.sort_values(by=['DEST', 'ARR_TIME']).copy()\n",
    "destinations = df.DEST.unique().tolist()\n",
    "\n",
    "sub_df = df.sort_values(by=['DEST', 'ARR_TIME'])[\n",
    "    ['DEST', 'ARR_TIME', 'ARR_DELAY']].copy()\n",
    "dict_ = {dest: sub_df[sub_df['DEST'] == dest].copy() for dest in destinations}\n",
    "\n",
    "\n",
    "def window_func_arr(in_):\n",
    "    ORIGIN, TIME = in_[0], in_[1]\n",
    "    lower_bound, upper_bound = get_bounds(TIME)\n",
    "    df = dict_[ORIGIN]\n",
    "    df_ = df[(df['ARR_TIME'] >= lower_bound) & (df['ARR_TIME'] <=\n",
    "                                                upper_bound)].copy()\n",
    "    count_airplanes = len(df_)\n",
    "    return count_airplanes\n",
    "\n",
    "\n",
    "df['x3'] = df[['ORIGIN', 'DEP_TIME']].apply(\n",
    "    track_progress(window_func_arr), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-we will calculate the average taxi out time for each airoprt (x4)\n",
    "\n",
    "-We will also define a function to check if the flight is or is not during peak time (x5)\n",
    "\n",
    "-we will take (x6) to be air time and we will take (x7) to be taxi in"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_peak(dep_time):\n",
    "    dep_time = dep_time.time()\n",
    "    if (dep_time >= time(4, 0) and (dep_time <= time(16, 0))):\n",
    "        return 1\n",
    "    else:\n",
    "        return 2\n",
    "\n",
    "\n",
    "dict_averages={origin: df[df.ORIGIN == origin]\n",
    "                 ['TAXI_OUT'].mean() for origin in origins}\n",
    "\n",
    "\n",
    "def avg_taxi_out(origin, dict=dict_averages):\n",
    "    return dict[origin]\n",
    "\n",
    "\n",
    "df['x4']=df.ORIGIN.apply(avg_taxi_out)\n",
    "\n",
    "df['x5']=df.DEP_TIME.apply(is_peak)\n",
    "\n",
    "df['x6']=df.AIR_TIME\n",
    "\n",
    "df['x7']=df.TAXI_IN\n",
    "\n",
    "df.to_csv('snapshot.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-we will define x* as the number of late aircrafts sharing the runway with the aircraft being considered in the time window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.01% done.\n",
      "2.01% done.\n",
      "4.01% done.\n",
      "6.01% done.\n",
      "8.01% done.\n",
      "10.01% done.\n",
      "12.01% done.\n",
      "14.01% done.\n",
      "16.01% done.\n",
      "18.01% done.\n",
      "20.01% done.\n",
      "22.01% done.\n",
      "24.01% done.\n",
      "26.01% done.\n",
      "28.01% done.\n",
      "30.01% done.\n",
      "32.01% done.\n",
      "34.01% done.\n",
      "36.01% done.\n",
      "38.01% done.\n",
      "40.01% done.\n",
      "42.01% done.\n",
      "44.01% done.\n",
      "46.01% done.\n",
      "48.01% done.\n",
      "50.01% done.\n",
      "52.01% done.\n",
      "54.01% done.\n",
      "56.01% done.\n",
      "58.01% done.\n",
      "60.01% done.\n",
      "62.01% done.\n",
      "64.01% done.\n",
      "66.01% done.\n",
      "68.01% done.\n",
      "70.01% done.\n",
      "72.01% done.\n",
      "74.01% done.\n",
      "76.01% done.\n",
      "78.01% done.\n",
      "80.01% done.\n",
      "82.01% done.\n",
      "84.01% done.\n",
      "86.01% done.\n",
      "88.01% done.\n",
      "90.01% done.\n",
      "92.01% done.\n",
      "94.01% done.\n",
      "96.02% done.\n",
      "98.02% done.\n"
     ]
    }
   ],
   "source": [
    "def late_to_leave(TIME, variable, df):\n",
    "    lower_bound, upper_bound=get_bounds(TIME)\n",
    "    df_=df[(df[variable] >= lower_bound) & (df[variable] <=\n",
    "                                              upper_bound) & ((df['DEP_DELAY'] < -15) | (df['DEP_DELAY'] > 15))].copy()\n",
    "    count_airplanes=len(df_)\n",
    "    return count_airplanes\n",
    "\n",
    "x8=airplane_counter(late_to_leave, 'DEP_TIME', 'ORIGIN',\n",
    "                    origins, add_cols=['DEP_DELAY'])\n",
    "\n",
    "df['x8']=x8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "x9 is the number of late arriving aircrafts on the runway in the time window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.02% done.\n",
      "2.02% done.\n",
      "4.02% done.\n",
      "6.02% done.\n",
      "8.02% done.\n",
      "10.02% done.\n",
      "12.02% done.\n",
      "14.02% done.\n",
      "16.02% done.\n",
      "18.02% done.\n",
      "20.02% done.\n",
      "22.02% done.\n",
      "24.02% done.\n",
      "26.02% done.\n",
      "28.02% done.\n",
      "30.02% done.\n",
      "32.02% done.\n",
      "34.02% done.\n",
      "36.02% done.\n",
      "38.02% done.\n",
      "40.02% done.\n",
      "42.02% done.\n",
      "44.02% done.\n",
      "46.02% done.\n",
      "48.02% done.\n",
      "50.02% done.\n",
      "52.02% done.\n",
      "54.02% done.\n",
      "56.02% done.\n",
      "58.02% done.\n",
      "60.02% done.\n",
      "62.02% done.\n",
      "64.02% done.\n",
      "66.02% done.\n",
      "68.02% done.\n",
      "70.02% done.\n",
      "72.02% done.\n",
      "74.02% done.\n",
      "76.02% done.\n",
      "78.02% done.\n",
      "80.02% done.\n",
      "82.02% done.\n",
      "84.02% done.\n",
      "86.02% done.\n",
      "88.02% done.\n",
      "90.02% done.\n",
      "92.02% done.\n",
      "94.02% done.\n",
      "96.02% done.\n",
      "98.02% done.\n"
     ]
    }
   ],
   "source": [
    "def late_to_arrive(in_):\n",
    "    ORIGIN, TIME=in_[0], in_[1]\n",
    "    lower_bound, upper_bound=get_bounds(TIME)\n",
    "    df=dict_[ORIGIN]\n",
    "    df_=df[(df['ARR_TIME'] >= lower_bound) & (df['ARR_TIME'] <=\n",
    "                                                upper_bound) & ((df['ARR_DELAY'] < -15) | (df['ARR_DELAY'] > 15))].copy()\n",
    "    count_airplanes=len(df_)\n",
    "    return count_airplanes\n",
    "\n",
    "df['x9']=df[['ORIGIN', 'DEP_TIME']].apply(\n",
    "    track_progress(late_to_arrive), axis=1)\n",
    "\n",
    "df[['ORIGIN', 'DEP_TIME'] +\n",
    "    ['x' + str(i) for i in range(1, 10)]].to_csv('variables.csv', index=False)\n",
    "\n",
    "work_df=df[['ORIGIN', 'DEP_TIME'] +\n",
    "    ['x' + str(i) for i in range(1, 10)]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally we will discretize the target value (taxi out time). our target x1 is going to be descritized into 5 bins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "non discretized environment variables\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ORIGIN</th>\n",
       "      <th>DEP_TIME</th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "      <th>x5</th>\n",
       "      <th>x6</th>\n",
       "      <th>x7</th>\n",
       "      <th>x8</th>\n",
       "      <th>x9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SFB</td>\n",
       "      <td>2019-04-01 06:58:00</td>\n",
       "      <td>15.0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>13.801970</td>\n",
       "      <td>1</td>\n",
       "      <td>127.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CLT</td>\n",
       "      <td>2019-04-01 09:46:00</td>\n",
       "      <td>27.0</td>\n",
       "      <td>33</td>\n",
       "      <td>28</td>\n",
       "      <td>21.445305</td>\n",
       "      <td>1</td>\n",
       "      <td>68.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>9</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DTW</td>\n",
       "      <td>2019-04-01 10:09:00</td>\n",
       "      <td>18.0</td>\n",
       "      <td>39</td>\n",
       "      <td>16</td>\n",
       "      <td>17.244655</td>\n",
       "      <td>1</td>\n",
       "      <td>62.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ATL</td>\n",
       "      <td>2019-04-01 10:39:00</td>\n",
       "      <td>11.0</td>\n",
       "      <td>58</td>\n",
       "      <td>42</td>\n",
       "      <td>15.761485</td>\n",
       "      <td>1</td>\n",
       "      <td>95.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>18</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PHL</td>\n",
       "      <td>2019-04-01 12:47:00</td>\n",
       "      <td>14.0</td>\n",
       "      <td>22</td>\n",
       "      <td>9</td>\n",
       "      <td>21.666012</td>\n",
       "      <td>1</td>\n",
       "      <td>15.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>SFB</td>\n",
       "      <td>2019-04-01 12:13:00</td>\n",
       "      <td>13.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>13.801970</td>\n",
       "      <td>1</td>\n",
       "      <td>120.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ORD</td>\n",
       "      <td>2019-04-01 13:56:00</td>\n",
       "      <td>33.0</td>\n",
       "      <td>65</td>\n",
       "      <td>44</td>\n",
       "      <td>21.773309</td>\n",
       "      <td>1</td>\n",
       "      <td>84.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>DTW</td>\n",
       "      <td>2019-04-01 15:40:00</td>\n",
       "      <td>21.0</td>\n",
       "      <td>42</td>\n",
       "      <td>22</td>\n",
       "      <td>17.244655</td>\n",
       "      <td>1</td>\n",
       "      <td>61.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ATL</td>\n",
       "      <td>2019-04-01 15:49:00</td>\n",
       "      <td>15.0</td>\n",
       "      <td>37</td>\n",
       "      <td>53</td>\n",
       "      <td>15.761485</td>\n",
       "      <td>1</td>\n",
       "      <td>100.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>CLT</td>\n",
       "      <td>2019-04-01 16:24:00</td>\n",
       "      <td>19.0</td>\n",
       "      <td>52</td>\n",
       "      <td>19</td>\n",
       "      <td>21.445305</td>\n",
       "      <td>2</td>\n",
       "      <td>74.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  ORIGIN            DEP_TIME    x1  x2  x3         x4  x5     x6   x7  x8  x9\n",
       "0    SFB 2019-04-01 06:58:00  15.0   7   0  13.801970   1  127.0  4.0   6   0\n",
       "1    CLT 2019-04-01 09:46:00  27.0  33  28  21.445305   1   68.0  5.0   9  21\n",
       "2    DTW 2019-04-01 10:09:00  18.0  39  16  17.244655   1   62.0  4.0  13  15\n",
       "3    ATL 2019-04-01 10:39:00  11.0  58  42  15.761485   1   95.0  3.0  18  22\n",
       "4    PHL 2019-04-01 12:47:00  14.0  22   9  21.666012   1   15.0  4.0   3   4\n",
       "5    SFB 2019-04-01 12:13:00  13.0   1   3  13.801970   1  120.0  4.0   1   1\n",
       "6    ORD 2019-04-01 13:56:00  33.0  65  44  21.773309   1   84.0  5.0  20  20\n",
       "7    DTW 2019-04-01 15:40:00  21.0  42  22  17.244655   1   61.0  3.0   9  13\n",
       "8    ATL 2019-04-01 15:49:00  15.0  37  53  15.761485   1  100.0  4.0  10  12\n",
       "9    CLT 2019-04-01 16:24:00  19.0  52  19  21.445305   2   74.0  6.0   7  10"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "work_df['DEP_TIME']=pd.to_datetime(work_df['DEP_TIME'])\n",
    "work_df=pd.read_csv('./env/data/variables.csv')\n",
    "print('non discretized environment variables')\n",
    "work_df.iloc[:10,:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Making the environment\n",
    "We will make a custom gym environment from this data following the code examples in their github.\n",
    "\n",
    "We will use comments inside the code below to document."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import gym\n",
    "from gym import spaces\n",
    "import numpy as np\n",
    "from datetime import datetime, timedelta\n",
    "\n",
    "class ontime_dataset_env(gym.Env):\n",
    "    metadata = {'render.modes': ['human']}\n",
    "\n",
    "    def __init__(self, df):\n",
    "        super(ontime_dataset_env, self).__init__()\n",
    "        self.df = df\n",
    "        # getting the maximum of each variable because variables have to be bound between 0 and 1\n",
    "        self.max_x2 = max(df['x2'])\n",
    "        self.max_x3 = max(df['x3'])\n",
    "        self.max_x4 = max(df['x4'])\n",
    "        self.max_x5 = max(df['x5'])\n",
    "        self.max_x6 = max(df['x6'])\n",
    "        self.max_x7 = max(df['x7'])\n",
    "        self.max_x8 = max(df['x8'])\n",
    "        self.max_x9 = max(df['x9'])\n",
    "        self.prev_time_step = datetime.strptime(\n",
    "            '2019-03-30 23:45:01', '%Y-%m-%d %H:%M:%S')\n",
    "        n_actions = 5\n",
    "        #defining action space, we are going to discritize our actions into 5 bins\n",
    "        self.action_space = spaces.Discrete(n_actions)\n",
    "        \n",
    "        #defining observation space\n",
    "        self.observation_space = spaces.Box(\n",
    "            low=0, high=1, shape=(8,), dtype=np.float16)\n",
    "\n",
    "    def _next_observation(self):\n",
    "        # construct the state (observation) for every step, the state will include all the airplains in a window of 15 minutes and we will pad the resulting array with zeros to reach a uniform shape\n",
    "        obs = np.array([\n",
    "            self.df.loc[self.current_step: self.current_step\n",
    "                        + timedelta(minutes=15), 'x1'].values,\n",
    "            self.df.loc[self.current_step: self.current_step\n",
    "                        + timedelta(minutes=15), 'x2'].values / self.max_x2,\n",
    "            self.df.loc[self.current_step: self.current_step\n",
    "                        + timedelta(minutes=15), 'x3'].values / self.max_x3,\n",
    "            self.df.loc[self.current_step: self.current_step\n",
    "                        + timedelta(minutes=15), 'x4'].values / self.max_x4,\n",
    "            self.df.loc[self.current_step: self.current_step\n",
    "                        + timedelta(minutes=15), 'x5'].values / self.max_x5,\n",
    "            self.df.loc[self.current_step: self.current_step\n",
    "                        + timedelta(minutes=15), 'x6'].values / self.max_x6,\n",
    "            self.df.loc[self.current_step: self.current_step\n",
    "                        + timedelta(minutes=15), 'x7'].values / self.max_x7,\n",
    "            self.df.loc[self.current_step: self.current_step\n",
    "                        + timedelta(minutes=15), 'x8'].values / self.max_x8,\n",
    "            self.df.loc[self.current_step: self.current_step\n",
    "                        + timedelta(minutes=15), 'x9'].values / self.max_x9\n",
    "        ])\n",
    "        #fetch next row until the window is done then get next window\n",
    "        if self.prev_time_step != self.current_step:\n",
    "            self.row_idx = 0\n",
    "            self.input_length = obs.shape[1]\n",
    "        obs = obs[:, self.row_idx]\n",
    "        self.prev_time_step = self.current_step\n",
    "        return obs\n",
    "    \n",
    "    def _take_action(self, action):\n",
    "        # choose a random action\n",
    "        current_dep_latency = random.randint(0, 4)\n",
    "\n",
    "    def step(self, action):\n",
    "        # Execute one time step within the environment\n",
    "        self._take_action(action)\n",
    "        obs = self._next_observation()        \n",
    "        #if we get to the end of the dataset while taking a step, we return to the begining\n",
    "\n",
    "        \n",
    "        #calculating the reward, it will be one for each correct lable and -1* abs(true_label - predicted_label)/4 for each incorrect label\n",
    "        reward = np.absolute(obs[0] - action)*-1/4\n",
    "        if reward == 0:\n",
    "            reward = 1\n",
    "        self.row_idx += 1\n",
    "        \n",
    "        # we will end each episode when we have trained on each row within a 15 minutes window, also if we randomly chose a starting ppoint in the last 15 minutes of the dataset we will be reset to the begining\n",
    "        done = False if self.row_idx < self.input_length else True\n",
    "        if done:\n",
    "            self.current_step = self.current_step = self.df.iloc[random.randint(0, len(self.df.loc[:, 'x2'].values) - 6)].name \n",
    "            if self.current_step > datetime.strptime('2019-04-30 23:45:01', '%Y-%m-%d %H:%M:%S'):\n",
    "                self.current_step = datetime.strptime(\n",
    "                    '2019-04-01 00:00:00', '%Y-%m-%d %H:%M:%S')\n",
    "        return obs, reward, done, {}\n",
    "\n",
    "    def reset(self):\n",
    "        # Set the current step to a random point within the data frame\n",
    "        self.current_step = self.df.iloc[random.randint(\n",
    "            0, len(self.df.loc[:, 'x2'].values) - 6)].name\n",
    "        return self._next_observation()\n",
    "\n",
    "\n",
    "    def render(self, mode='human', close=False):\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Making the Model\n",
    "Our model is based on stable-baselines PGM model, it's also built on the code available in their github."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "tf.config.experimental.set_memory_growth(gpus[0], True)\n",
    "from tensorflow import keras\n",
    "from keras.layers import Dense, Activation, Input, Flatten\n",
    "from keras.models import Model, load_model\n",
    "from keras.optimizers import Adam\n",
    "import keras.backend as K\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "class Agent(object):\n",
    "    def __init__(self, ALPHA, GAMMA=0.99, n_actions=5, layer1_size=16, layer2_size=16, layer3_size=16, input_dims=9, fname='model.h5'):\n",
    "        self.GAMMA = GAMMA\n",
    "        self.lr = ALPHA\n",
    "        self.G = 0\n",
    "        self.input_dims = input_dims\n",
    "        self.fc1_dims = layer1_size\n",
    "        self.fc2_dims = layer2_size\n",
    "        self.fc3_dims = layer3_size\n",
    "        self.n_actions = n_actions\n",
    "        self.state_memory = []\n",
    "        self.action_memory = []\n",
    "        self.reward_memory = []\n",
    "        self.policy, self.predict = self.build_policy_network()\n",
    "        self.action_space = [i for i in range(n_actions)]\n",
    "        self.model_file = fname\n",
    "    \n",
    "    #this is the function that will define the policy agent and and will make predictions\n",
    "    def build_policy_network(self):\n",
    "        input = Input(shape=(self.input_dims,))\n",
    "        advantages = Input(shape=[1])\n",
    "        dense1 = Dense(self.fc1_dims, activation='relu')(input)\n",
    "        dense2 = Dense(self.fc2_dims, activation='relu')(dense1)\n",
    "        dense3 = Dense(self.fc3_dims, activation='relu')(dense2)\n",
    "        probs = Dense(self.n_actions, activation='softmax')(dense3)\n",
    "\n",
    "        def custom_loss(y_true, y_predict):\n",
    "            out = K.clip(y_predict, 1e-8, 1 - 1e-8)\n",
    "            log_lik = y_true * K.log(out)\n",
    "            return K.sum(-log_lik * advantages)\n",
    "\n",
    "        policy = Model(input=[input, advantages], output=[probs])\n",
    "        policy.compile(optimizer=Adam(lr=self.lr), loss=custom_loss)\n",
    "        predict = Model(input=[input], output=[probs])\n",
    "        return policy, predict\n",
    "    \n",
    "    #we will choose an actiom for each observation within the batch by making a random choice weighted by the probabilities predicted by our model\n",
    "    def choose_action(self, observation):\n",
    "        state = observation[np.newaxis, :]\n",
    "        probabilities = self.predict.predict(state)[0]\n",
    "        action = np.random.choice(self.action_space, p=probabilities)\n",
    "        return action\n",
    "\n",
    "    #this is a helper function that stores the history of the model\n",
    "    def store_transition(self, observation, action, reward):\n",
    "        self.state_memory.append(observation)\n",
    "        self.action_memory.append(action)\n",
    "        self.reward_memory.append(reward)\n",
    "        \n",
    "    # this is the main driver function that we will call to train the agent\n",
    "    def learn(self):\n",
    "        state_memory = np.array(self.state_memory)\n",
    "        action_memory = np.array(self.action_memory)\n",
    "        reward_memory = np.array(self.reward_memory)\n",
    "        actions = np.zeros([len(action_memory), self.n_actions])\n",
    "        actions[np.arange(len(action_memory)), action_memory] = 1\n",
    "        \n",
    "        # calculating the gain\n",
    "        G = np.zeros_like(reward_memory)\n",
    "        for t in range(len(reward_memory)):\n",
    "            G_sum = 0\n",
    "            discount = 1\n",
    "            for k in range(t, len(reward_memory)):\n",
    "                G_sum += reward_memory[k]*discount\n",
    "                discount *= self.GAMMA\n",
    "            G[t] = G_sum\n",
    "        mean = np.mean(G)\n",
    "        std = np.std(G) if np.std(G) > 0 else 1\n",
    "        self.G = (G-mean)/std\n",
    "        \n",
    "        #calculating the cost\n",
    "        cost = self.policy.train_on_batch([state_memory, self.G], actions)\n",
    "        \n",
    "        #resetting memory\n",
    "        self.state_memory = []\n",
    "        self.action_memory = []\n",
    "        self.reward_memory = []\n",
    "\n",
    "    \n",
    "    #helper function to save and load the model\n",
    "    def save_model(self):\n",
    "        self.policy.save(self.model_file)\n",
    "\n",
    "    def load_model(self):\n",
    "        self.policy = load_model(self.model_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing the environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import KBinsDiscretizer\n",
    "\n",
    "df = pd.read_csv('./env/data/variables.csv')\n",
    "df['Date'] = pd.to_datetime(df['Date'])\n",
    "df = df.set_index('Date')\n",
    "discretizer = KBinsDiscretizer(n_bins=5, encode='ordinal')\n",
    "df['x1'] = discretizer.fit_transform(\n",
    "df['x1'].to_numpy().reshape(-1, 1))\n",
    "\n",
    "env = ontime_dataset_env(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.        , 0.03448276, 0.02380952, 0.58825559, 0.5       ,\n",
       "       0.05890805, 0.0221519 , 0.        , 0.01724138])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([4.        , 0.14942529, 0.70238095, 0.61140748, 0.5       ,\n",
       "        0.16954023, 0.00949367, 0.1       , 0.15517241]),\n",
       " 1,\n",
       " False,\n",
       " {})"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.step(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.        , 0.11494253, 0.27380952, 0.68524502, 1.        ,\n",
       "        0.27011494, 0.01265823, 0.03333333, 0.13793103]),\n",
       " -1.0,\n",
       " False,\n",
       " {})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.step(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we are satisfied that our custom environment is working, we will start training the model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training the model\n",
    "Unfortunately no matter what hyper parameters were used, this model was not able to be trained on this dataset effectively."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mostafakm/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:43: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=[<tf.Tenso..., outputs=[<tf.Tenso...)`\n",
      "/home/mostafakm/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:45: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=[<tf.Tenso..., outputs=[<tf.Tenso...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 0 score -45.2 average_score -45.2\n",
      "episode 1 score -80.0 average_score -62.6\n",
      "episode 2 score -71.5 average_score -65.6\n",
      "episode 3 score -32.5 average_score -57.3\n",
      "episode 4 score -51.8 average_score -56.2\n",
      "episode 5 score -74.5 average_score -59.2\n",
      "episode 6 score -65.8 average_score -60.2\n",
      "episode 7 score -41.0 average_score -57.8\n",
      "episode 8 score -85.8 average_score -60.9\n",
      "episode 9 score -86.5 average_score -63.5\n",
      "episode 10 score -123.0 average_score -68.9\n",
      "episode 11 score -65.5 average_score -68.6\n",
      "episode 12 score -71.5 average_score -68.8\n",
      "episode 13 score -62.5 average_score -68.4\n",
      "episode 14 score -88.2 average_score -69.7\n",
      "episode 15 score -41.0 average_score -67.9\n",
      "episode 16 score -90.5 average_score -69.2\n",
      "episode 17 score -46.5 average_score -68.0\n",
      "episode 18 score -65.2 average_score -67.8\n",
      "episode 19 score -63.5 average_score -67.6\n",
      "episode 20 score -60.0 average_score -67.2\n",
      "episode 21 score -35.8 average_score -65.8\n",
      "episode 22 score -59.0 average_score -65.5\n",
      "episode 23 score -80.2 average_score -66.1\n",
      "episode 24 score -92.5 average_score -67.2\n",
      "episode 25 score -60.8 average_score -66.9\n",
      "episode 26 score -66.2 average_score -66.9\n",
      "episode 27 score -62.0 average_score -66.7\n",
      "episode 28 score -49.0 average_score -66.1\n",
      "episode 29 score -62.0 average_score -66.0\n",
      "episode 30 score -65.8 average_score -66.0\n",
      "episode 31 score -73.0 average_score -66.2\n",
      "episode 32 score -80.0 average_score -66.6\n",
      "episode 33 score -57.2 average_score -66.3\n",
      "episode 34 score -50.5 average_score -65.9\n",
      "episode 35 score -59.8 average_score -65.7\n",
      "episode 36 score -79.2 average_score -66.1\n",
      "episode 37 score -4.5 average_score -64.5\n",
      "episode 38 score -61.2 average_score -64.4\n",
      "episode 39 score -95.2 average_score -65.2\n",
      "episode 40 score -62.0 average_score -65.1\n",
      "episode 41 score -45.0 average_score -64.6\n",
      "episode 42 score -61.0 average_score -64.5\n",
      "episode 43 score -51.8 average_score -64.2\n",
      "episode 44 score -34.5 average_score -63.6\n",
      "episode 45 score -69.8 average_score -63.7\n",
      "episode 46 score -18.8 average_score -62.7\n",
      "episode 47 score -80.5 average_score -63.1\n",
      "episode 48 score -40.2 average_score -62.6\n",
      "episode 49 score -53.2 average_score -62.5\n",
      "episode 50 score -70.8 average_score -62.6\n",
      "episode 51 score 1.0 average_score -61.4\n",
      "episode 52 score -16.5 average_score -60.5\n",
      "episode 53 score -79.0 average_score -60.9\n",
      "episode 54 score -83.0 average_score -61.3\n",
      "episode 55 score -59.5 average_score -61.3\n",
      "episode 56 score -81.8 average_score -61.6\n",
      "episode 57 score -81.8 average_score -62.0\n",
      "episode 58 score -92.8 average_score -62.5\n",
      "episode 59 score -54.8 average_score -62.4\n",
      "episode 60 score -41.5 average_score -62.0\n",
      "episode 61 score -54.2 average_score -61.9\n",
      "episode 62 score -54.5 average_score -61.8\n",
      "episode 63 score -38.2 average_score -61.4\n",
      "episode 64 score -59.8 average_score -61.4\n",
      "episode 65 score -56.8 average_score -61.3\n",
      "episode 66 score -2.5 average_score -60.4\n",
      "episode 67 score -68.0 average_score -60.5\n",
      "episode 68 score -83.5 average_score -60.9\n",
      "episode 69 score -51.5 average_score -60.7\n",
      "episode 70 score -66.8 average_score -60.8\n",
      "episode 71 score -96.0 average_score -61.3\n",
      "episode 72 score -54.5 average_score -61.2\n",
      "episode 73 score -54.5 average_score -61.1\n",
      "episode 74 score -42.5 average_score -60.9\n",
      "episode 75 score -67.2 average_score -61.0\n",
      "episode 76 score -83.5 average_score -61.3\n",
      "episode 77 score -81.8 average_score -61.5\n",
      "episode 78 score -55.5 average_score -61.4\n",
      "episode 79 score -56.5 average_score -61.4\n",
      "episode 80 score -71.5 average_score -61.5\n",
      "episode 81 score -74.0 average_score -61.7\n",
      "episode 82 score -57.8 average_score -61.6\n",
      "episode 83 score -54.5 average_score -61.5\n",
      "episode 84 score -49.2 average_score -61.4\n",
      "episode 85 score -62.2 average_score -61.4\n",
      "episode 86 score -56.8 average_score -61.3\n",
      "episode 87 score -44.0 average_score -61.1\n",
      "episode 88 score -79.0 average_score -61.3\n",
      "episode 89 score -54.5 average_score -61.3\n",
      "episode 90 score -49.5 average_score -61.1\n",
      "episode 91 score -49.0 average_score -61.0\n",
      "episode 92 score -59.0 average_score -61.0\n",
      "episode 93 score -36.5 average_score -60.7\n",
      "episode 94 score -93.5 average_score -61.1\n",
      "episode 95 score -38.0 average_score -60.8\n",
      "episode 96 score -63.5 average_score -60.9\n",
      "episode 97 score -30.8 average_score -60.6\n",
      "episode 98 score -74.8 average_score -60.7\n",
      "episode 99 score -65.0 average_score -60.7\n",
      "episode 100 score -59.8 average_score -60.9\n",
      "episode 101 score -79.5 average_score -60.9\n",
      "episode 102 score -73.0 average_score -60.9\n",
      "episode 103 score -66.0 average_score -61.2\n",
      "episode 104 score -35.5 average_score -61.1\n",
      "episode 105 score -61.2 average_score -60.9\n",
      "episode 106 score -54.2 average_score -60.8\n",
      "episode 107 score -94.2 average_score -61.4\n",
      "episode 108 score -115.5 average_score -61.6\n",
      "episode 109 score -87.8 average_score -61.7\n",
      "episode 110 score -142.5 average_score -61.9\n",
      "episode 111 score -111.8 average_score -62.3\n",
      "episode 112 score -79.5 average_score -62.4\n",
      "episode 113 score -43.2 average_score -62.2\n",
      "episode 114 score -85.0 average_score -62.2\n",
      "episode 115 score -57.5 average_score -62.3\n",
      "episode 116 score -48.8 average_score -61.9\n",
      "episode 117 score -95.0 average_score -62.4\n",
      "episode 118 score -49.0 average_score -62.2\n",
      "episode 119 score -60.0 average_score -62.2\n",
      "episode 120 score -101.8 average_score -62.6\n",
      "episode 121 score -54.8 average_score -62.8\n",
      "episode 122 score -91.8 average_score -63.1\n",
      "episode 123 score -43.8 average_score -62.8\n",
      "episode 124 score -56.5 average_score -62.4\n",
      "episode 125 score -62.0 average_score -62.4\n",
      "episode 126 score -74.2 average_score -62.5\n",
      "episode 127 score -73.5 average_score -62.6\n",
      "episode 128 score -63.5 average_score -62.8\n",
      "episode 129 score -54.2 average_score -62.7\n",
      "episode 130 score -62.8 average_score -62.7\n",
      "episode 131 score -74.8 average_score -62.7\n",
      "episode 132 score -57.8 average_score -62.5\n",
      "episode 133 score -71.8 average_score -62.6\n",
      "episode 134 score -74.0 average_score -62.8\n",
      "episode 135 score -61.8 average_score -62.9\n",
      "episode 136 score -58.2 average_score -62.6\n",
      "episode 137 score -46.5 average_score -63.1\n",
      "episode 138 score -66.0 average_score -63.1\n",
      "episode 139 score -67.5 average_score -62.8\n",
      "episode 140 score -47.0 average_score -62.7\n",
      "episode 141 score -72.2 average_score -63.0\n",
      "episode 142 score -84.0 average_score -63.2\n",
      "episode 143 score -0.8 average_score -62.7\n",
      "episode 144 score -59.0 average_score -62.9\n",
      "episode 145 score -62.8 average_score -62.9\n",
      "episode 146 score -33.2 average_score -63.0\n",
      "episode 147 score -2.0 average_score -62.2\n",
      "episode 148 score -74.0 average_score -62.6\n",
      "episode 149 score -54.5 average_score -62.6\n",
      "episode 150 score -55.2 average_score -62.4\n",
      "episode 151 score -50.2 average_score -62.9\n",
      "episode 152 score -59.0 average_score -63.3\n",
      "episode 153 score -76.5 average_score -63.3\n",
      "episode 154 score -53.5 average_score -63.0\n",
      "episode 155 score -53.5 average_score -63.0\n",
      "episode 156 score -68.8 average_score -62.8\n",
      "episode 157 score -65.2 average_score -62.7\n",
      "episode 158 score -74.2 average_score -62.5\n",
      "episode 159 score -72.2 average_score -62.7\n",
      "episode 160 score -73.5 average_score -63.0\n",
      "episode 161 score -46.8 average_score -62.9\n",
      "episode 162 score -78.0 average_score -63.1\n",
      "episode 163 score -51.0 average_score -63.3\n",
      "episode 164 score -60.5 average_score -63.3\n",
      "episode 165 score -60.5 average_score -63.3\n",
      "episode 166 score -57.5 average_score -63.9\n",
      "episode 167 score -46.5 average_score -63.6\n",
      "episode 168 score -6.0 average_score -62.9\n",
      "episode 169 score -34.0 average_score -62.7\n",
      "episode 170 score -103.0 average_score -63.1\n",
      "episode 171 score -65.2 average_score -62.8\n",
      "episode 172 score -40.2 average_score -62.6\n",
      "episode 173 score -76.0 average_score -62.8\n",
      "episode 174 score -64.0 average_score -63.0\n",
      "episode 175 score -80.5 average_score -63.2\n",
      "episode 176 score -45.8 average_score -62.8\n",
      "episode 177 score -50.0 average_score -62.5\n",
      "episode 178 score -41.5 average_score -62.3\n",
      "episode 179 score -3.0 average_score -61.8\n",
      "episode 180 score -71.2 average_score -61.8\n",
      "episode 181 score -70.0 average_score -61.8\n",
      "episode 182 score -61.8 average_score -61.8\n",
      "episode 183 score -55.5 average_score -61.8\n",
      "episode 184 score -51.2 average_score -61.8\n",
      "episode 185 score -15.2 average_score -61.4\n",
      "episode 186 score -48.8 average_score -61.3\n",
      "episode 187 score -59.8 average_score -61.4\n",
      "episode 188 score -68.8 average_score -61.3\n",
      "episode 189 score -56.8 average_score -61.4\n",
      "episode 190 score -55.5 average_score -61.4\n",
      "episode 191 score -89.2 average_score -61.8\n",
      "episode 192 score -58.2 average_score -61.8\n",
      "episode 193 score -55.8 average_score -62.0\n",
      "episode 194 score -66.5 average_score -61.7\n",
      "episode 195 score -65.5 average_score -62.0\n",
      "episode 196 score -59.0 average_score -62.0\n",
      "episode 197 score -47.2 average_score -62.1\n",
      "episode 198 score -69.2 average_score -62.1\n",
      "episode 199 score -63.0 average_score -62.1\n",
      "episode 200 score -39.8 average_score -61.9\n",
      "episode 201 score -57.8 average_score -61.6\n",
      "episode 202 score -28.2 average_score -61.2\n",
      "episode 203 score -69.0 average_score -61.2\n",
      "episode 204 score -46.2 average_score -61.3\n",
      "episode 205 score -60.0 average_score -61.3\n",
      "episode 206 score -31.2 average_score -61.1\n",
      "episode 207 score -68.2 average_score -60.8\n",
      "episode 208 score -68.5 average_score -60.4\n",
      "episode 209 score -54.5 average_score -60.0\n",
      "episode 210 score -41.2 average_score -59.0\n",
      "episode 211 score -19.5 average_score -58.1\n",
      "episode 212 score -65.2 average_score -57.9\n",
      "episode 213 score -61.2 average_score -58.1\n",
      "episode 214 score -62.8 average_score -57.9\n",
      "episode 215 score -69.5 average_score -58.0\n",
      "episode 216 score -80.0 average_score -58.3\n",
      "episode 217 score -48.8 average_score -57.9\n",
      "episode 218 score -71.2 average_score -58.1\n",
      "episode 219 score -52.8 average_score -58.0\n",
      "episode 220 score -76.0 average_score -57.8\n",
      "episode 221 score -47.0 average_score -57.7\n",
      "episode 222 score -59.5 average_score -57.4\n",
      "episode 223 score -78.8 average_score -57.7\n",
      "episode 224 score -29.0 average_score -57.4\n",
      "episode 225 score -63.0 average_score -57.5\n",
      "episode 226 score -8.8 average_score -56.8\n",
      "episode 227 score -83.2 average_score -56.9\n",
      "episode 228 score -60.5 average_score -56.9\n",
      "episode 229 score -60.0 average_score -56.9\n",
      "episode 230 score -80.2 average_score -57.1\n",
      "episode 231 score -56.8 average_score -56.9\n",
      "episode 232 score -64.2 average_score -57.0\n",
      "episode 233 score -28.8 average_score -56.6\n",
      "episode 234 score -79.8 average_score -56.6\n",
      "episode 235 score -65.0 average_score -56.6\n",
      "episode 236 score -66.8 average_score -56.7\n",
      "episode 237 score -51.5 average_score -56.8\n",
      "episode 238 score -81.5 average_score -56.9\n",
      "episode 239 score -67.2 average_score -56.9\n",
      "episode 240 score -52.2 average_score -57.0\n",
      "episode 241 score -71.2 average_score -57.0\n",
      "episode 242 score -48.5 average_score -56.6\n",
      "episode 243 score -58.2 average_score -57.2\n",
      "episode 244 score -6.0 average_score -56.7\n",
      "episode 245 score -45.2 average_score -56.5\n",
      "episode 246 score -38.5 average_score -56.5\n",
      "episode 247 score -52.2 average_score -57.0\n",
      "episode 248 score -50.0 average_score -56.8\n",
      "episode 249 score -54.5 average_score -56.8\n",
      "episode 250 score -72.0 average_score -57.0\n",
      "episode 251 score -49.5 average_score -57.0\n",
      "episode 252 score -48.0 average_score -56.9\n",
      "episode 253 score -35.0 average_score -56.4\n",
      "episode 254 score -36.2 average_score -56.3\n",
      "episode 255 score -74.0 average_score -56.5\n",
      "episode 256 score -31.5 average_score -56.1\n",
      "episode 257 score -35.0 average_score -55.8\n",
      "episode 258 score -61.5 average_score -55.7\n",
      "episode 259 score -11.5 average_score -55.1\n",
      "episode 260 score -110.2 average_score -55.4\n",
      "episode 261 score -106.2 average_score -56.0\n",
      "episode 262 score 2.5 average_score -55.2\n",
      "episode 263 score -59.2 average_score -55.3\n",
      "episode 264 score -65.5 average_score -55.4\n",
      "episode 265 score -56.0 average_score -55.3\n",
      "episode 266 score -55.0 average_score -55.3\n",
      "episode 267 score -47.2 average_score -55.3\n",
      "episode 268 score -31.5 average_score -55.5\n",
      "episode 269 score -73.8 average_score -55.9\n",
      "episode 270 score -87.5 average_score -55.8\n",
      "episode 271 score -77.5 average_score -55.9\n",
      "episode 272 score -20.0 average_score -55.7\n",
      "episode 273 score -39.0 average_score -55.3\n",
      "episode 274 score -64.8 average_score -55.3\n",
      "episode 275 score -36.2 average_score -54.9\n",
      "episode 276 score -61.0 average_score -55.1\n",
      "episode 277 score -63.5 average_score -55.2\n",
      "episode 278 score -37.5 average_score -55.1\n",
      "episode 279 score -45.5 average_score -55.6\n",
      "episode 280 score -42.5 average_score -55.3\n",
      "episode 281 score -64.5 average_score -55.2\n",
      "episode 282 score -60.2 average_score -55.2\n",
      "episode 283 score -64.5 average_score -55.3\n",
      "episode 284 score -83.2 average_score -55.6\n",
      "episode 285 score -43.0 average_score -55.9\n",
      "episode 286 score -59.2 average_score -56.0\n",
      "episode 287 score -33.2 average_score -55.7\n",
      "episode 288 score -40.5 average_score -55.5\n",
      "episode 289 score -70.2 average_score -55.6\n",
      "episode 290 score -65.8 average_score -55.7\n",
      "episode 291 score -57.5 average_score -55.4\n",
      "episode 292 score -60.8 average_score -55.4\n",
      "episode 293 score -77.2 average_score -55.6\n",
      "episode 294 score -70.8 average_score -55.7\n",
      "episode 295 score -88.8 average_score -55.9\n",
      "episode 296 score -109.8 average_score -56.4\n",
      "episode 297 score -50.8 average_score -56.4\n",
      "episode 298 score -41.5 average_score -56.2\n",
      "episode 299 score -55.5 average_score -56.1\n",
      "episode 300 score -52.2 average_score -56.2\n",
      "episode 301 score -66.5 average_score -56.3\n",
      "episode 302 score -65.0 average_score -56.7\n",
      "episode 303 score -68.8 average_score -56.7\n",
      "episode 304 score -52.8 average_score -56.7\n",
      "episode 305 score -24.2 average_score -56.4\n",
      "episode 306 score -15.2 average_score -56.2\n",
      "episode 307 score -41.2 average_score -55.9\n",
      "episode 308 score -73.0 average_score -56.0\n",
      "episode 309 score -35.5 average_score -55.8\n",
      "episode 310 score -65.5 average_score -56.0\n",
      "episode 311 score -56.0 average_score -56.4\n",
      "episode 312 score -44.5 average_score -56.2\n",
      "episode 313 score -59.8 average_score -56.2\n",
      "episode 314 score -82.8 average_score -56.4\n",
      "episode 315 score -62.5 average_score -56.3\n",
      "episode 316 score -52.0 average_score -56.0\n",
      "episode 317 score -46.2 average_score -56.0\n",
      "episode 318 score -83.5 average_score -56.1\n",
      "episode 319 score -50.8 average_score -56.1\n",
      "episode 320 score -51.0 average_score -55.9\n",
      "episode 321 score -17.8 average_score -55.6\n",
      "episode 322 score -43.0 average_score -55.4\n",
      "episode 323 score -79.0 average_score -55.4\n",
      "episode 324 score -59.2 average_score -55.7\n",
      "episode 325 score -12.0 average_score -55.2\n",
      "episode 326 score -48.5 average_score -55.6\n",
      "episode 327 score -67.0 average_score -55.4\n",
      "episode 328 score -13.8 average_score -55.0\n",
      "episode 329 score -8.2 average_score -54.4\n",
      "episode 330 score -53.2 average_score -54.2\n",
      "episode 331 score -43.8 average_score -54.0\n",
      "episode 332 score -53.0 average_score -53.9\n",
      "episode 333 score -54.0 average_score -54.2\n",
      "episode 334 score -60.8 average_score -54.0\n",
      "episode 335 score -54.5 average_score -53.9\n",
      "episode 336 score -64.2 average_score -53.9\n",
      "episode 337 score -49.0 average_score -53.8\n",
      "episode 338 score -59.8 average_score -53.6\n",
      "episode 339 score -68.8 average_score -53.6\n",
      "episode 340 score -41.5 average_score -53.5\n",
      "episode 341 score -39.8 average_score -53.2\n",
      "episode 342 score -68.0 average_score -53.4\n",
      "episode 343 score -57.0 average_score -53.4\n",
      "episode 344 score 1.2 average_score -53.3\n",
      "episode 345 score -55.8 average_score -53.4\n",
      "episode 346 score -90.2 average_score -53.9\n",
      "episode 347 score -52.0 average_score -53.9\n",
      "episode 348 score -69.5 average_score -54.1\n",
      "episode 349 score -48.2 average_score -54.1\n",
      "episode 350 score -79.5 average_score -54.2\n",
      "episode 351 score -68.8 average_score -54.3\n",
      "episode 352 score -67.2 average_score -54.5\n",
      "episode 353 score -38.8 average_score -54.6\n",
      "episode 354 score -60.0 average_score -54.8\n",
      "episode 355 score -57.2 average_score -54.6\n",
      "episode 356 score -53.8 average_score -54.9\n",
      "episode 357 score -27.0 average_score -54.8\n",
      "episode 358 score -53.0 average_score -54.7\n",
      "episode 359 score -59.5 average_score -55.2\n",
      "episode 360 score -73.0 average_score -54.8\n",
      "episode 361 score -59.0 average_score -54.3\n",
      "episode 362 score -57.8 average_score -54.9\n",
      "episode 363 score -45.5 average_score -54.8\n",
      "episode 364 score -35.5 average_score -54.5\n",
      "episode 365 score -46.0 average_score -54.4\n",
      "episode 366 score -78.8 average_score -54.6\n",
      "episode 367 score -90.8 average_score -55.1\n",
      "episode 368 score -48.5 average_score -55.2\n",
      "episode 369 score -63.0 average_score -55.1\n",
      "episode 370 score -39.8 average_score -54.7\n",
      "episode 371 score -42.0 average_score -54.3\n",
      "episode 372 score -29.0 average_score -54.4\n",
      "episode 373 score -26.8 average_score -54.3\n",
      "episode 374 score -58.2 average_score -54.2\n",
      "episode 375 score -82.8 average_score -54.7\n",
      "episode 376 score -71.2 average_score -54.8\n",
      "episode 377 score -62.0 average_score -54.8\n",
      "episode 378 score -51.5 average_score -54.9\n",
      "episode 379 score -52.8 average_score -55.0\n",
      "episode 380 score -71.2 average_score -55.3\n",
      "episode 381 score -69.8 average_score -55.3\n",
      "episode 382 score -68.2 average_score -55.4\n",
      "episode 383 score -58.8 average_score -55.3\n",
      "episode 384 score -63.0 average_score -55.1\n",
      "episode 385 score -44.0 average_score -55.1\n",
      "episode 386 score -5.2 average_score -54.6\n",
      "episode 387 score -27.5 average_score -54.5\n",
      "episode 388 score -59.8 average_score -54.7\n",
      "episode 389 score -43.0 average_score -54.5\n",
      "episode 390 score -59.0 average_score -54.4\n",
      "episode 391 score -67.5 average_score -54.5\n",
      "episode 392 score -68.5 average_score -54.6\n",
      "episode 393 score -48.5 average_score -54.3\n",
      "episode 394 score -48.0 average_score -54.1\n",
      "episode 395 score -66.2 average_score -53.8\n",
      "episode 396 score -61.2 average_score -53.4\n",
      "episode 397 score -55.5 average_score -53.4\n",
      "episode 398 score -40.5 average_score -53.4\n",
      "episode 399 score -64.5 average_score -53.5\n",
      "episode 400 score -56.5 average_score -53.5\n",
      "episode 401 score -29.8 average_score -53.2\n",
      "episode 402 score -50.5 average_score -53.0\n",
      "episode 403 score -12.2 average_score -52.4\n",
      "episode 404 score -47.2 average_score -52.4\n",
      "episode 405 score -47.8 average_score -52.6\n",
      "episode 406 score -82.8 average_score -53.3\n",
      "episode 407 score -13.8 average_score -53.0\n",
      "episode 408 score -36.8 average_score -52.7\n",
      "episode 409 score -48.0 average_score -52.8\n",
      "episode 410 score -41.2 average_score -52.5\n",
      "episode 411 score -34.2 average_score -52.3\n",
      "episode 412 score -67.2 average_score -52.6\n",
      "episode 413 score -50.8 average_score -52.5\n",
      "episode 414 score -54.2 average_score -52.2\n",
      "episode 415 score -57.0 average_score -52.1\n",
      "episode 416 score -70.5 average_score -52.3\n",
      "episode 417 score -52.2 average_score -52.4\n",
      "episode 418 score -77.8 average_score -52.3\n",
      "episode 419 score -39.0 average_score -52.2\n",
      "episode 420 score -41.5 average_score -52.1\n",
      "episode 421 score -27.8 average_score -52.2\n",
      "episode 422 score -38.8 average_score -52.2\n",
      "episode 423 score -48.5 average_score -51.9\n",
      "episode 424 score -17.2 average_score -51.4\n",
      "episode 425 score -64.0 average_score -52.0\n",
      "episode 426 score -58.0 average_score -52.0\n",
      "episode 427 score -44.8 average_score -51.8\n",
      "episode 428 score -3.8 average_score -51.7\n",
      "episode 429 score -49.0 average_score -52.1\n",
      "episode 430 score -40.0 average_score -52.0\n",
      "episode 431 score -36.0 average_score -51.9\n",
      "episode 432 score -21.5 average_score -51.6\n",
      "episode 433 score -50.0 average_score -51.6\n",
      "episode 434 score -47.0 average_score -51.4\n",
      "episode 435 score -52.5 average_score -51.4\n",
      "episode 436 score -67.8 average_score -51.4\n",
      "episode 437 score -39.2 average_score -51.3\n",
      "episode 438 score -68.5 average_score -51.4\n",
      "episode 439 score -59.8 average_score -51.3\n",
      "episode 440 score -31.5 average_score -51.2\n",
      "episode 441 score -56.5 average_score -51.4\n",
      "episode 442 score -66.5 average_score -51.4\n",
      "episode 443 score -40.2 average_score -51.2\n",
      "episode 444 score -67.8 average_score -51.9\n",
      "episode 445 score -55.2 average_score -51.9\n",
      "episode 446 score -73.8 average_score -51.7\n",
      "episode 447 score -41.0 average_score -51.6\n",
      "episode 448 score -63.5 average_score -51.6\n",
      "episode 449 score -47.5 average_score -51.6\n",
      "episode 450 score -22.5 average_score -51.0\n",
      "episode 451 score -53.8 average_score -50.9\n",
      "episode 452 score -67.8 average_score -50.9\n",
      "episode 453 score -63.0 average_score -51.1\n",
      "episode 454 score -62.2 average_score -51.1\n",
      "episode 455 score -67.5 average_score -51.2\n",
      "episode 456 score -48.8 average_score -51.2\n",
      "episode 457 score -62.0 average_score -51.5\n",
      "episode 458 score -57.5 average_score -51.6\n",
      "episode 459 score -15.8 average_score -51.1\n",
      "episode 460 score -26.8 average_score -50.7\n",
      "episode 461 score -37.0 average_score -50.4\n",
      "episode 462 score -17.5 average_score -50.0\n",
      "episode 463 score -52.8 average_score -50.1\n",
      "episode 464 score -58.8 average_score -50.4\n",
      "episode 465 score -49.0 average_score -50.4\n",
      "episode 466 score -41.0 average_score -50.0\n",
      "episode 467 score -42.8 average_score -49.5\n",
      "episode 468 score -86.2 average_score -49.9\n",
      "episode 469 score -71.0 average_score -50.0\n",
      "episode 470 score -44.2 average_score -50.0\n",
      "episode 471 score -52.0 average_score -50.1\n",
      "episode 472 score -54.2 average_score -50.4\n",
      "episode 473 score -65.5 average_score -50.8\n",
      "episode 474 score -60.5 average_score -50.8\n",
      "episode 475 score -47.2 average_score -50.4\n",
      "episode 476 score -57.0 average_score -50.3\n",
      "episode 477 score -54.5 average_score -50.2\n",
      "episode 478 score -23.0 average_score -49.9\n",
      "episode 479 score -49.2 average_score -49.9\n",
      "episode 480 score -28.8 average_score -49.5\n",
      "episode 481 score -45.8 average_score -49.2\n",
      "episode 482 score -29.2 average_score -48.8\n",
      "episode 483 score -49.0 average_score -48.7\n",
      "episode 484 score -35.8 average_score -48.5\n",
      "episode 485 score -41.5 average_score -48.4\n",
      "episode 486 score -61.8 average_score -49.0\n",
      "episode 487 score -54.8 average_score -49.3\n",
      "episode 488 score -36.0 average_score -49.0\n",
      "episode 489 score -51.5 average_score -49.1\n",
      "episode 490 score -63.5 average_score -49.2\n",
      "episode 491 score -70.0 average_score -49.2\n",
      "episode 492 score -46.2 average_score -49.0\n",
      "episode 493 score -66.5 average_score -49.2\n",
      "episode 494 score -53.2 average_score -49.2\n",
      "episode 495 score -15.5 average_score -48.7\n",
      "episode 496 score -35.2 average_score -48.4\n",
      "episode 497 score -78.5 average_score -48.7\n",
      "episode 498 score -32.5 average_score -48.6\n",
      "episode 499 score -43.5 average_score -48.4\n",
      "episode 500 score -36.0 average_score -48.2\n",
      "episode 501 score -56.5 average_score -48.4\n",
      "episode 502 score -42.8 average_score -48.4\n",
      "episode 503 score -56.8 average_score -48.8\n",
      "episode 504 score -28.0 average_score -48.6\n",
      "episode 505 score -44.8 average_score -48.6\n",
      "episode 506 score -49.2 average_score -48.3\n",
      "episode 507 score -52.2 average_score -48.6\n",
      "episode 508 score -62.5 average_score -48.9\n",
      "episode 509 score -53.0 average_score -48.9\n",
      "episode 510 score -36.0 average_score -48.9\n",
      "episode 511 score -47.8 average_score -49.0\n",
      "episode 512 score -55.0 average_score -48.9\n",
      "episode 513 score -57.5 average_score -49.0\n",
      "episode 514 score -48.8 average_score -48.9\n",
      "episode 515 score -69.0 average_score -49.0\n",
      "episode 516 score -1.2 average_score -48.3\n",
      "episode 517 score -60.8 average_score -48.4\n",
      "episode 518 score -76.5 average_score -48.4\n",
      "episode 519 score -27.8 average_score -48.3\n",
      "episode 520 score -42.5 average_score -48.3\n",
      "episode 521 score -69.2 average_score -48.7\n",
      "episode 522 score -52.8 average_score -48.9\n",
      "episode 523 score -56.5 average_score -49.0\n",
      "episode 524 score -43.0 average_score -49.2\n",
      "episode 525 score -67.8 average_score -49.2\n",
      "episode 526 score -63.0 average_score -49.3\n",
      "episode 527 score -55.2 average_score -49.4\n",
      "episode 528 score -64.2 average_score -50.0\n",
      "episode 529 score -23.0 average_score -49.7\n",
      "episode 530 score -9.0 average_score -49.4\n",
      "episode 531 score -75.8 average_score -49.8\n",
      "episode 532 score -35.8 average_score -50.0\n",
      "episode 533 score -37.0 average_score -49.8\n",
      "episode 534 score -38.5 average_score -49.8\n",
      "episode 535 score -55.5 average_score -49.8\n",
      "episode 536 score -44.5 average_score -49.6\n",
      "episode 537 score -26.0 average_score -49.4\n",
      "episode 538 score -43.0 average_score -49.2\n",
      "episode 539 score -9.5 average_score -48.7\n",
      "episode 540 score -19.8 average_score -48.6\n",
      "episode 541 score -65.0 average_score -48.6\n",
      "episode 542 score -56.0 average_score -48.5\n",
      "episode 543 score -47.0 average_score -48.6\n",
      "episode 544 score -40.0 average_score -48.3\n",
      "episode 545 score -51.0 average_score -48.3\n",
      "episode 546 score -21.8 average_score -47.8\n",
      "episode 547 score -31.0 average_score -47.7\n",
      "episode 548 score -38.2 average_score -47.4\n",
      "episode 549 score -5.5 average_score -47.0\n",
      "episode 550 score -40.0 average_score -47.2\n",
      "episode 551 score -40.0 average_score -47.0\n",
      "episode 552 score -59.2 average_score -46.9\n",
      "episode 553 score -45.8 average_score -46.8\n",
      "episode 554 score -45.5 average_score -46.6\n",
      "episode 555 score -27.5 average_score -46.2\n",
      "episode 556 score -35.8 average_score -46.1\n",
      "episode 557 score -25.2 average_score -45.7\n",
      "episode 558 score -33.0 average_score -45.5\n",
      "episode 559 score -37.8 average_score -45.7\n",
      "episode 560 score -43.0 average_score -45.8\n",
      "episode 561 score -56.2 average_score -46.0\n",
      "episode 562 score -69.2 average_score -46.5\n",
      "episode 563 score -43.0 average_score -46.5\n",
      "episode 564 score -71.0 average_score -46.6\n",
      "episode 565 score -43.8 average_score -46.5\n",
      "episode 566 score -50.5 average_score -46.6\n",
      "episode 567 score -42.2 average_score -46.6\n",
      "episode 568 score -28.8 average_score -46.0\n",
      "episode 569 score -72.5 average_score -46.1\n",
      "episode 570 score -38.0 average_score -46.0\n",
      "episode 571 score -58.2 average_score -46.1\n",
      "episode 572 score -21.2 average_score -45.7\n",
      "episode 573 score -45.0 average_score -45.5\n",
      "episode 574 score -33.0 average_score -45.2\n",
      "episode 575 score -40.2 average_score -45.2\n",
      "episode 576 score -48.5 average_score -45.1\n",
      "episode 577 score -34.0 average_score -44.9\n",
      "episode 578 score -56.5 average_score -45.2\n",
      "episode 579 score -28.0 average_score -45.0\n",
      "episode 580 score -41.2 average_score -45.1\n",
      "episode 581 score -75.0 average_score -45.4\n",
      "episode 582 score -35.5 average_score -45.5\n",
      "episode 583 score -39.2 average_score -45.4\n",
      "episode 584 score -35.0 average_score -45.4\n",
      "episode 585 score -43.5 average_score -45.4\n",
      "episode 586 score -66.5 average_score -45.4\n",
      "episode 587 score -65.2 average_score -45.6\n",
      "episode 588 score -34.8 average_score -45.5\n",
      "episode 589 score -57.5 average_score -45.6\n",
      "episode 590 score -48.5 average_score -45.5\n",
      "episode 591 score -34.0 average_score -45.1\n",
      "episode 592 score -49.0 average_score -45.1\n",
      "episode 593 score -34.5 average_score -44.8\n",
      "episode 594 score -45.0 average_score -44.7\n",
      "episode 595 score -35.5 average_score -44.9\n",
      "episode 596 score -69.0 average_score -45.3\n",
      "episode 597 score -42.2 average_score -44.9\n",
      "episode 598 score -28.8 average_score -44.9\n",
      "episode 599 score -40.2 average_score -44.8\n",
      "episode 600 score -75.5 average_score -45.2\n",
      "episode 601 score -49.5 average_score -45.1\n",
      "episode 602 score -30.8 average_score -45.0\n",
      "episode 603 score -21.8 average_score -44.7\n",
      "episode 604 score -47.2 average_score -44.9\n",
      "episode 605 score -10.5 average_score -44.5\n",
      "episode 606 score -55.0 average_score -44.6\n",
      "episode 607 score -48.2 average_score -44.5\n",
      "episode 608 score -42.5 average_score -44.3\n",
      "episode 609 score -45.0 average_score -44.3\n",
      "episode 610 score -68.8 average_score -44.6\n",
      "episode 611 score -40.8 average_score -44.5\n",
      "episode 612 score -24.2 average_score -44.2\n",
      "episode 613 score -34.8 average_score -44.0\n",
      "episode 614 score -60.5 average_score -44.1\n",
      "episode 615 score -59.0 average_score -44.0\n",
      "episode 616 score -60.2 average_score -44.6\n",
      "episode 617 score -75.5 average_score -44.7\n",
      "episode 618 score -29.8 average_score -44.3\n",
      "episode 619 score -33.5 average_score -44.3\n",
      "episode 620 score -40.2 average_score -44.3\n",
      "episode 621 score -50.2 average_score -44.1\n",
      "episode 622 score -49.8 average_score -44.1\n",
      "episode 623 score -54.2 average_score -44.1\n",
      "episode 624 score -31.0 average_score -43.9\n",
      "episode 625 score -59.2 average_score -43.9\n",
      "episode 626 score -49.0 average_score -43.7\n",
      "episode 627 score -32.2 average_score -43.5\n",
      "episode 628 score -69.0 average_score -43.5\n",
      "episode 629 score -28.8 average_score -43.6\n",
      "episode 630 score -57.5 average_score -44.1\n",
      "episode 631 score -51.8 average_score -43.8\n",
      "episode 632 score -67.8 average_score -44.2\n",
      "episode 633 score -2.2 average_score -43.8\n",
      "episode 634 score -36.8 average_score -43.8\n",
      "episode 635 score -50.2 average_score -43.7\n",
      "episode 636 score -46.0 average_score -43.8\n",
      "episode 637 score -40.5 average_score -43.9\n",
      "episode 638 score -52.2 average_score -44.0\n",
      "episode 639 score -9.5 average_score -44.0\n",
      "episode 640 score -54.0 average_score -44.3\n",
      "episode 641 score -55.2 average_score -44.2\n",
      "episode 642 score -44.0 average_score -44.1\n",
      "episode 643 score -28.0 average_score -43.9\n",
      "episode 644 score -40.8 average_score -43.9\n",
      "episode 645 score -20.0 average_score -43.6\n",
      "episode 646 score -34.8 average_score -43.8\n",
      "episode 647 score -89.0 average_score -44.3\n",
      "episode 648 score -71.8 average_score -44.7\n",
      "episode 649 score -32.5 average_score -44.9\n",
      "episode 650 score -52.8 average_score -45.1\n",
      "episode 651 score -77.0 average_score -45.4\n",
      "episode 652 score -36.2 average_score -45.2\n",
      "episode 653 score -57.0 average_score -45.3\n",
      "episode 654 score -49.0 average_score -45.4\n",
      "episode 655 score -24.8 average_score -45.3\n",
      "episode 656 score -45.2 average_score -45.4\n",
      "episode 657 score -47.5 average_score -45.6\n",
      "episode 658 score -66.2 average_score -46.0\n",
      "episode 659 score -23.0 average_score -45.8\n",
      "episode 660 score -27.2 average_score -45.7\n",
      "episode 661 score -57.2 average_score -45.7\n",
      "episode 662 score -51.8 average_score -45.5\n",
      "episode 663 score -56.8 average_score -45.6\n",
      "episode 664 score -33.8 average_score -45.3\n",
      "episode 665 score -49.0 average_score -45.3\n",
      "episode 666 score -9.8 average_score -44.9\n",
      "episode 667 score -30.0 average_score -44.8\n",
      "episode 668 score -37.2 average_score -44.9\n",
      "episode 669 score -43.2 average_score -44.6\n",
      "episode 670 score -52.2 average_score -44.7\n",
      "episode 671 score -39.0 average_score -44.5\n",
      "episode 672 score -54.2 average_score -44.9\n",
      "episode 673 score -58.2 average_score -45.0\n",
      "episode 674 score -48.0 average_score -45.2\n",
      "episode 675 score -39.5 average_score -45.1\n",
      "episode 676 score -54.5 average_score -45.2\n",
      "episode 677 score -45.0 average_score -45.3\n",
      "episode 678 score -63.2 average_score -45.4\n",
      "episode 679 score -44.5 average_score -45.5\n",
      "episode 680 score -43.0 average_score -45.6\n",
      "episode 681 score -54.2 average_score -45.4\n",
      "episode 682 score -33.8 average_score -45.3\n",
      "episode 683 score -36.2 average_score -45.3\n",
      "episode 684 score -13.0 average_score -45.1\n",
      "episode 685 score -70.0 average_score -45.4\n",
      "episode 686 score -31.2 average_score -45.0\n",
      "episode 687 score -54.0 average_score -44.9\n",
      "episode 688 score -44.2 average_score -45.0\n",
      "episode 689 score -49.2 average_score -44.9\n",
      "episode 690 score -59.2 average_score -45.0\n",
      "episode 691 score -78.2 average_score -45.5\n",
      "episode 692 score -46.2 average_score -45.4\n",
      "episode 693 score -56.2 average_score -45.6\n",
      "episode 694 score -48.5 average_score -45.7\n",
      "episode 695 score -39.0 average_score -45.7\n",
      "episode 696 score -39.2 average_score -45.4\n",
      "episode 697 score -65.2 average_score -45.6\n",
      "episode 698 score -42.8 average_score -45.8\n",
      "episode 699 score -53.0 average_score -45.9\n",
      "episode 700 score -68.5 average_score -45.8\n",
      "episode 701 score -46.2 average_score -45.8\n",
      "episode 702 score -49.5 average_score -46.0\n",
      "episode 703 score -1.2 average_score -45.8\n",
      "episode 704 score -36.8 average_score -45.7\n",
      "episode 705 score -31.2 average_score -45.9\n",
      "episode 706 score -71.2 average_score -46.1\n",
      "episode 707 score -45.0 average_score -46.0\n",
      "episode 708 score -22.5 average_score -45.8\n",
      "episode 709 score -39.2 average_score -45.8\n",
      "episode 710 score -53.8 average_score -45.6\n",
      "episode 711 score -21.0 average_score -45.4\n",
      "episode 712 score -20.0 average_score -45.4\n",
      "episode 713 score -49.2 average_score -45.5\n",
      "episode 714 score -33.8 average_score -45.3\n",
      "episode 715 score -48.5 average_score -45.1\n",
      "episode 716 score -22.0 average_score -44.8\n",
      "episode 717 score -7.5 average_score -44.1\n",
      "episode 718 score -39.8 average_score -44.2\n",
      "episode 719 score -9.0 average_score -43.9\n",
      "episode 720 score -28.8 average_score -43.8\n",
      "episode 721 score -78.2 average_score -44.1\n",
      "episode 722 score -10.8 average_score -43.7\n",
      "episode 723 score -51.8 average_score -43.7\n",
      "episode 724 score -68.0 average_score -44.1\n",
      "episode 725 score -52.0 average_score -44.0\n",
      "episode 726 score -56.5 average_score -44.1\n",
      "episode 727 score -30.2 average_score -44.0\n",
      "episode 728 score -65.0 average_score -44.0\n",
      "episode 729 score -29.2 average_score -44.0\n",
      "episode 730 score -29.5 average_score -43.7\n",
      "episode 731 score -43.0 average_score -43.6\n",
      "episode 732 score -40.0 average_score -43.4\n",
      "episode 733 score -48.2 average_score -43.8\n",
      "episode 734 score -45.5 average_score -43.9\n",
      "episode 735 score -28.5 average_score -43.7\n",
      "episode 736 score -32.8 average_score -43.6\n",
      "episode 737 score -49.2 average_score -43.6\n",
      "episode 738 score -39.0 average_score -43.5\n",
      "episode 739 score -17.2 average_score -43.6\n",
      "episode 740 score -47.5 average_score -43.5\n",
      "episode 741 score -5.8 average_score -43.0\n",
      "episode 742 score -18.5 average_score -42.8\n",
      "episode 743 score -56.8 average_score -43.1\n",
      "episode 744 score -54.0 average_score -43.2\n",
      "episode 745 score -21.5 average_score -43.2\n",
      "episode 746 score -58.5 average_score -43.5\n",
      "episode 747 score -50.8 average_score -43.1\n",
      "episode 748 score -21.5 average_score -42.6\n",
      "episode 749 score -70.0 average_score -42.9\n",
      "episode 750 score -61.2 average_score -43.0\n",
      "episode 751 score -14.0 average_score -42.4\n",
      "episode 752 score -32.0 average_score -42.4\n",
      "episode 753 score -35.8 average_score -42.1\n",
      "episode 754 score -49.8 average_score -42.1\n",
      "episode 755 score -67.2 average_score -42.6\n",
      "episode 756 score -40.5 average_score -42.5\n",
      "episode 757 score -49.5 average_score -42.5\n",
      "episode 758 score -40.5 average_score -42.3\n",
      "episode 759 score -41.8 average_score -42.5\n",
      "episode 760 score -22.5 average_score -42.4\n",
      "episode 761 score -27.2 average_score -42.1\n",
      "episode 762 score -20.2 average_score -41.8\n",
      "episode 763 score -14.5 average_score -41.4\n",
      "episode 764 score -65.8 average_score -41.7\n",
      "episode 765 score -50.5 average_score -41.7\n",
      "episode 766 score -48.5 average_score -42.1\n",
      "episode 767 score -43.5 average_score -42.2\n",
      "episode 768 score -71.0 average_score -42.6\n",
      "episode 769 score -40.8 average_score -42.6\n",
      "episode 770 score -35.5 average_score -42.4\n",
      "episode 771 score -72.5 average_score -42.7\n",
      "episode 772 score -48.8 average_score -42.7\n",
      "episode 773 score -18.5 average_score -42.3\n",
      "episode 774 score -46.0 average_score -42.3\n",
      "episode 775 score 5.5 average_score -41.8\n",
      "episode 776 score -20.0 average_score -41.5\n",
      "episode 777 score -26.8 average_score -41.3\n",
      "episode 778 score -58.0 average_score -41.2\n",
      "episode 779 score -20.2 average_score -41.0\n",
      "episode 780 score -49.2 average_score -41.0\n",
      "episode 781 score -53.8 average_score -41.0\n",
      "episode 782 score -59.0 average_score -41.3\n",
      "episode 783 score -34.8 average_score -41.3\n",
      "episode 784 score -47.0 average_score -41.6\n",
      "episode 785 score -64.5 average_score -41.6\n",
      "episode 786 score -30.8 average_score -41.6\n",
      "episode 787 score -24.8 average_score -41.3\n",
      "episode 788 score -25.2 average_score -41.1\n",
      "episode 789 score -18.2 average_score -40.8\n",
      "episode 790 score -37.2 average_score -40.5\n",
      "episode 791 score -13.8 average_score -39.9\n",
      "episode 792 score -39.5 average_score -39.8\n",
      "episode 793 score -36.2 average_score -39.6\n",
      "episode 794 score -37.2 average_score -39.5\n",
      "episode 795 score -57.0 average_score -39.7\n",
      "episode 796 score -19.8 average_score -39.5\n",
      "episode 797 score -22.8 average_score -39.1\n",
      "episode 798 score -46.2 average_score -39.1\n",
      "episode 799 score -3.8 average_score -38.6\n",
      "episode 800 score -34.8 average_score -38.3\n",
      "episode 801 score -42.5 average_score -38.2\n",
      "episode 802 score -2.8 average_score -37.8\n",
      "episode 803 score -39.2 average_score -38.2\n",
      "episode 804 score -18.0 average_score -38.0\n",
      "episode 805 score -42.2 average_score -38.1\n",
      "episode 806 score -62.8 average_score -38.0\n",
      "episode 807 score -50.8 average_score -38.1\n",
      "episode 808 score -42.8 average_score -38.3\n",
      "episode 809 score -39.2 average_score -38.3\n",
      "episode 810 score -43.2 average_score -38.2\n",
      "episode 811 score -23.8 average_score -38.2\n",
      "episode 812 score -34.8 average_score -38.3\n",
      "episode 813 score -29.0 average_score -38.1\n",
      "episode 814 score -54.5 average_score -38.3\n",
      "episode 815 score -58.2 average_score -38.4\n",
      "episode 816 score -24.0 average_score -38.5\n",
      "episode 817 score -28.0 average_score -38.7\n",
      "episode 818 score -41.0 average_score -38.7\n",
      "episode 819 score -32.2 average_score -38.9\n",
      "episode 820 score -39.8 average_score -39.0\n",
      "episode 821 score -41.5 average_score -38.6\n",
      "episode 822 score -13.5 average_score -38.7\n",
      "episode 823 score -47.2 average_score -38.6\n",
      "episode 824 score -43.0 average_score -38.4\n",
      "episode 825 score -29.2 average_score -38.1\n",
      "episode 826 score -35.2 average_score -37.9\n",
      "episode 827 score -44.2 average_score -38.1\n",
      "episode 828 score -47.0 average_score -37.9\n",
      "episode 829 score -13.0 average_score -37.7\n",
      "episode 830 score -40.0 average_score -37.8\n",
      "episode 831 score -55.0 average_score -38.0\n",
      "episode 832 score -18.5 average_score -37.7\n",
      "episode 833 score -22.8 average_score -37.5\n",
      "episode 834 score -28.0 average_score -37.3\n",
      "episode 835 score -33.8 average_score -37.4\n",
      "episode 836 score -50.8 average_score -37.5\n",
      "episode 837 score -23.2 average_score -37.3\n",
      "episode 838 score 0.5 average_score -36.9\n",
      "episode 839 score -43.0 average_score -37.1\n",
      "episode 840 score -25.8 average_score -36.9\n",
      "episode 841 score -38.5 average_score -37.3\n",
      "episode 842 score -49.0 average_score -37.6\n",
      "episode 843 score -13.2 average_score -37.1\n",
      "episode 844 score -34.2 average_score -36.9\n",
      "episode 845 score -49.8 average_score -37.2\n",
      "episode 846 score -43.8 average_score -37.1\n",
      "episode 847 score -46.0 average_score -37.0\n",
      "episode 848 score -47.8 average_score -37.3\n",
      "episode 849 score -42.0 average_score -37.0\n",
      "episode 850 score -42.5 average_score -36.8\n",
      "episode 851 score -31.2 average_score -37.0\n",
      "episode 852 score -37.8 average_score -37.0\n",
      "episode 853 score -42.8 average_score -37.1\n",
      "episode 854 score -23.0 average_score -36.8\n",
      "episode 855 score -7.5 average_score -36.2\n",
      "episode 856 score -31.2 average_score -36.2\n",
      "episode 857 score -27.8 average_score -35.9\n",
      "episode 858 score -12.5 average_score -35.7\n",
      "episode 859 score -31.8 average_score -35.6\n",
      "episode 860 score -53.5 average_score -35.9\n",
      "episode 861 score -56.0 average_score -36.2\n",
      "episode 862 score -54.2 average_score -36.5\n",
      "episode 863 score -42.2 average_score -36.8\n",
      "episode 864 score -29.0 average_score -36.4\n",
      "episode 865 score -63.5 average_score -36.5\n",
      "episode 866 score -37.8 average_score -36.4\n",
      "episode 867 score -44.2 average_score -36.4\n",
      "episode 868 score -34.8 average_score -36.1\n",
      "episode 869 score -19.0 average_score -35.9\n",
      "episode 870 score -21.8 average_score -35.7\n",
      "episode 871 score -30.8 average_score -35.3\n",
      "episode 872 score -49.0 average_score -35.3\n",
      "episode 873 score -22.2 average_score -35.3\n",
      "episode 874 score -1.2 average_score -34.9\n",
      "episode 875 score -47.5 average_score -35.4\n",
      "episode 876 score -52.8 average_score -35.8\n",
      "episode 877 score -21.8 average_score -35.7\n",
      "episode 878 score -45.0 average_score -35.6\n",
      "episode 879 score -66.0 average_score -36.0\n",
      "episode 880 score 2.0 average_score -35.5\n",
      "episode 881 score 12.2 average_score -34.9\n",
      "episode 882 score -36.5 average_score -34.6\n",
      "episode 883 score -10.2 average_score -34.4\n",
      "episode 884 score -41.8 average_score -34.3\n",
      "episode 885 score -21.2 average_score -33.9\n",
      "episode 886 score -22.8 average_score -33.8\n",
      "episode 887 score -21.8 average_score -33.8\n",
      "episode 888 score -41.8 average_score -34.0\n",
      "episode 889 score -57.2 average_score -34.3\n",
      "episode 890 score -47.8 average_score -34.5\n",
      "episode 891 score 10.0 average_score -34.2\n",
      "episode 892 score -31.8 average_score -34.1\n",
      "episode 893 score -33.5 average_score -34.1\n",
      "episode 894 score -3.5 average_score -33.8\n",
      "episode 895 score -41.8 average_score -33.6\n",
      "episode 896 score -33.5 average_score -33.8\n",
      "episode 897 score -18.5 average_score -33.7\n",
      "episode 898 score -32.8 average_score -33.6\n",
      "episode 899 score -24.8 average_score -33.8\n",
      "episode 900 score -26.5 average_score -33.7\n",
      "episode 901 score -18.2 average_score -33.5\n",
      "episode 902 score -55.2 average_score -34.0\n",
      "episode 903 score -41.0 average_score -34.0\n",
      "episode 904 score -29.5 average_score -34.1\n",
      "episode 905 score -24.8 average_score -33.9\n",
      "episode 906 score -38.2 average_score -33.7\n",
      "episode 907 score -31.0 average_score -33.5\n",
      "episode 908 score -11.0 average_score -33.2\n",
      "episode 909 score -54.5 average_score -33.3\n",
      "episode 910 score -49.0 average_score -33.4\n",
      "episode 911 score 4.5 average_score -33.1\n",
      "episode 912 score -24.0 average_score -33.0\n",
      "episode 913 score -51.5 average_score -33.2\n",
      "episode 914 score -59.0 average_score -33.3\n",
      "episode 915 score -55.5 average_score -33.2\n",
      "episode 916 score -35.0 average_score -33.4\n",
      "episode 917 score -32.5 average_score -33.4\n",
      "episode 918 score -46.8 average_score -33.5\n",
      "episode 919 score -0.8 average_score -33.1\n",
      "episode 920 score -44.5 average_score -33.2\n",
      "episode 921 score -23.8 average_score -33.0\n",
      "episode 922 score 1.0 average_score -32.9\n",
      "episode 923 score -15.5 average_score -32.6\n",
      "episode 924 score -40.5 average_score -32.5\n",
      "episode 925 score -40.2 average_score -32.6\n",
      "episode 926 score -32.8 average_score -32.6\n",
      "episode 927 score -44.2 average_score -32.6\n",
      "episode 928 score -37.5 average_score -32.5\n",
      "episode 929 score -39.5 average_score -32.8\n",
      "episode 930 score -54.2 average_score -32.9\n",
      "episode 931 score -6.8 average_score -32.4\n",
      "episode 932 score -14.8 average_score -32.4\n",
      "episode 933 score -52.5 average_score -32.7\n",
      "episode 934 score -31.8 average_score -32.7\n",
      "episode 935 score -15.2 average_score -32.6\n",
      "episode 936 score -53.8 average_score -32.6\n",
      "episode 937 score -20.2 average_score -32.6\n",
      "episode 938 score -40.2 average_score -33.0\n",
      "episode 939 score -40.2 average_score -32.9\n",
      "episode 940 score -23.8 average_score -32.9\n",
      "episode 941 score -30.8 average_score -32.8\n",
      "episode 942 score -34.8 average_score -32.7\n",
      "episode 943 score -25.0 average_score -32.8\n",
      "episode 944 score -35.8 average_score -32.8\n",
      "episode 945 score -36.5 average_score -32.7\n",
      "episode 946 score -20.5 average_score -32.5\n",
      "episode 947 score -43.8 average_score -32.4\n",
      "episode 948 score -36.2 average_score -32.3\n",
      "episode 949 score -32.5 average_score -32.2\n",
      "episode 950 score 6.2 average_score -31.7\n",
      "episode 951 score -41.0 average_score -31.8\n",
      "episode 952 score -46.2 average_score -31.9\n",
      "episode 953 score -26.0 average_score -31.8\n",
      "episode 954 score -30.8 average_score -31.8\n",
      "episode 955 score -44.8 average_score -32.2\n",
      "episode 956 score -59.5 average_score -32.5\n",
      "episode 957 score -66.8 average_score -32.9\n",
      "episode 958 score -17.2 average_score -32.9\n",
      "episode 959 score -25.8 average_score -32.9\n",
      "episode 960 score -26.0 average_score -32.6\n",
      "episode 961 score -34.5 average_score -32.4\n",
      "episode 962 score -17.8 average_score -32.0\n",
      "episode 963 score -40.2 average_score -32.0\n",
      "episode 964 score -57.0 average_score -32.3\n",
      "episode 965 score -5.2 average_score -31.7\n",
      "episode 966 score -18.2 average_score -31.5\n",
      "episode 967 score -12.2 average_score -31.2\n",
      "episode 968 score -18.5 average_score -31.0\n",
      "episode 969 score -2.2 average_score -30.8\n",
      "episode 970 score -16.0 average_score -30.8\n",
      "episode 971 score -36.8 average_score -30.9\n",
      "episode 972 score -11.8 average_score -30.5\n",
      "episode 973 score -48.5 average_score -30.7\n",
      "episode 974 score -12.2 average_score -30.9\n",
      "episode 975 score -36.5 average_score -30.7\n",
      "episode 976 score -38.5 average_score -30.6\n",
      "episode 977 score -40.5 average_score -30.8\n",
      "episode 978 score -52.2 average_score -30.9\n",
      "episode 979 score -39.8 average_score -30.6\n",
      "episode 980 score -43.0 average_score -31.0\n",
      "episode 981 score -26.5 average_score -31.4\n",
      "episode 982 score -24.8 average_score -31.3\n",
      "episode 983 score -37.0 average_score -31.6\n",
      "episode 984 score -36.0 average_score -31.5\n",
      "episode 985 score -23.0 average_score -31.5\n",
      "episode 986 score -81.5 average_score -32.1\n",
      "episode 987 score -32.2 average_score -32.2\n",
      "episode 988 score -27.0 average_score -32.1\n",
      "episode 989 score -12.5 average_score -31.6\n",
      "episode 990 score -45.2 average_score -31.6\n",
      "episode 991 score -22.0 average_score -31.9\n",
      "episode 992 score -35.0 average_score -32.0\n",
      "episode 993 score -19.8 average_score -31.8\n",
      "episode 994 score -45.0 average_score -32.2\n",
      "episode 995 score -53.8 average_score -32.4\n",
      "episode 996 score -46.8 average_score -32.5\n",
      "episode 997 score -53.2 average_score -32.8\n",
      "episode 998 score -34.5 average_score -32.9\n",
      "episode 999 score -28.0 average_score -32.9\n",
      "episode 1000 score -36.8 average_score -33.0\n",
      "episode 1001 score -36.2 average_score -33.2\n",
      "episode 1002 score -27.2 average_score -32.9\n",
      "episode 1003 score -21.0 average_score -32.7\n",
      "episode 1004 score -31.5 average_score -32.7\n",
      "episode 1005 score -20.5 average_score -32.7\n",
      "episode 1006 score -37.2 average_score -32.7\n",
      "episode 1007 score -33.8 average_score -32.7\n",
      "episode 1008 score 4.0 average_score -32.5\n",
      "episode 1009 score -48.8 average_score -32.5\n",
      "episode 1010 score -37.5 average_score -32.4\n",
      "episode 1011 score -45.8 average_score -32.9\n",
      "episode 1012 score -51.2 average_score -33.1\n",
      "episode 1013 score -41.8 average_score -33.0\n",
      "episode 1014 score -1.5 average_score -32.5\n",
      "episode 1015 score 4.8 average_score -31.9\n",
      "episode 1016 score -65.0 average_score -32.2\n",
      "episode 1017 score -34.5 average_score -32.2\n",
      "episode 1018 score -6.8 average_score -31.8\n",
      "episode 1019 score -63.8 average_score -32.4\n",
      "episode 1020 score -32.5 average_score -32.3\n",
      "episode 1021 score -25.0 average_score -32.3\n",
      "episode 1022 score -16.8 average_score -32.5\n",
      "episode 1023 score -32.0 average_score -32.7\n",
      "episode 1024 score -16.8 average_score -32.4\n",
      "episode 1025 score -36.2 average_score -32.4\n",
      "episode 1026 score -31.8 average_score -32.4\n",
      "episode 1027 score -40.0 average_score -32.3\n",
      "episode 1028 score -6.0 average_score -32.0\n",
      "episode 1029 score -21.0 average_score -31.8\n",
      "episode 1030 score -45.2 average_score -31.7\n",
      "episode 1031 score -30.8 average_score -32.0\n",
      "episode 1032 score -18.2 average_score -32.0\n",
      "episode 1033 score -25.0 average_score -31.7\n",
      "episode 1034 score -30.8 average_score -31.7\n",
      "episode 1035 score -29.2 average_score -31.9\n",
      "episode 1036 score -36.2 average_score -31.7\n",
      "episode 1037 score -17.8 average_score -31.7\n",
      "episode 1038 score -58.0 average_score -31.8\n",
      "episode 1039 score -14.0 average_score -31.6\n",
      "episode 1040 score -30.2 average_score -31.6\n",
      "episode 1041 score -60.8 average_score -31.9\n",
      "episode 1042 score -53.0 average_score -32.1\n",
      "episode 1043 score -17.0 average_score -32.0\n",
      "episode 1044 score -30.8 average_score -32.0\n",
      "episode 1045 score -6.0 average_score -31.7\n",
      "episode 1046 score -20.8 average_score -31.7\n",
      "episode 1047 score -23.0 average_score -31.5\n",
      "episode 1048 score -37.2 average_score -31.5\n",
      "episode 1049 score -36.2 average_score -31.5\n",
      "episode 1050 score -23.0 average_score -31.8\n",
      "episode 1051 score -51.5 average_score -31.9\n",
      "episode 1052 score -22.8 average_score -31.7\n",
      "episode 1053 score -42.5 average_score -31.9\n",
      "episode 1054 score -15.5 average_score -31.7\n",
      "episode 1055 score -40.0 average_score -31.7\n",
      "episode 1056 score -7.0 average_score -31.1\n",
      "episode 1057 score -29.0 average_score -30.8\n",
      "episode 1058 score -62.2 average_score -31.2\n",
      "episode 1059 score -4.0 average_score -31.0\n",
      "episode 1060 score -29.8 average_score -31.0\n",
      "episode 1061 score -15.5 average_score -30.8\n",
      "episode 1062 score -36.2 average_score -31.0\n",
      "episode 1063 score -21.2 average_score -30.8\n",
      "episode 1064 score -45.8 average_score -30.7\n",
      "episode 1065 score -44.2 average_score -31.1\n",
      "episode 1066 score -30.5 average_score -31.2\n",
      "episode 1067 score -15.5 average_score -31.3\n",
      "episode 1068 score -24.5 average_score -31.3\n",
      "episode 1069 score 31.0 average_score -31.0\n",
      "episode 1070 score -26.5 average_score -31.1\n",
      "episode 1071 score -31.0 average_score -31.0\n",
      "episode 1072 score -46.2 average_score -31.4\n",
      "episode 1073 score -21.0 average_score -31.1\n",
      "episode 1074 score -41.8 average_score -31.4\n",
      "episode 1075 score -30.5 average_score -31.3\n",
      "episode 1076 score -56.0 average_score -31.5\n",
      "episode 1077 score -43.8 average_score -31.6\n",
      "episode 1078 score -50.0 average_score -31.5\n",
      "episode 1079 score -21.0 average_score -31.3\n",
      "episode 1080 score -23.5 average_score -31.1\n",
      "episode 1081 score -43.2 average_score -31.3\n",
      "episode 1082 score -36.8 average_score -31.4\n",
      "episode 1083 score -4.0 average_score -31.1\n",
      "episode 1084 score -20.5 average_score -31.0\n",
      "episode 1085 score -32.5 average_score -31.0\n",
      "episode 1086 score -27.5 average_score -30.5\n",
      "episode 1087 score -41.2 average_score -30.6\n",
      "episode 1088 score -58.8 average_score -30.9\n",
      "episode 1089 score -54.2 average_score -31.3\n",
      "episode 1090 score -17.5 average_score -31.1\n",
      "episode 1091 score -49.8 average_score -31.3\n",
      "episode 1092 score -35.5 average_score -31.3\n",
      "episode 1093 score -30.2 average_score -31.4\n",
      "episode 1094 score -17.0 average_score -31.2\n",
      "episode 1095 score -28.2 average_score -30.9\n",
      "episode 1096 score -8.2 average_score -30.5\n",
      "episode 1097 score -31.0 average_score -30.3\n",
      "episode 1098 score -37.5 average_score -30.3\n",
      "episode 1099 score -15.0 average_score -30.2\n",
      "episode 1100 score -40.2 average_score -30.2\n",
      "episode 1101 score -29.2 average_score -30.2\n",
      "episode 1102 score -43.8 average_score -30.3\n",
      "episode 1103 score -30.2 average_score -30.4\n",
      "episode 1104 score -48.5 average_score -30.6\n",
      "episode 1105 score -65.5 average_score -31.0\n",
      "episode 1106 score -34.8 average_score -31.0\n",
      "episode 1107 score -82.2 average_score -31.5\n",
      "episode 1108 score -44.2 average_score -32.0\n",
      "episode 1109 score -33.8 average_score -31.8\n",
      "episode 1110 score -30.2 average_score -31.8\n",
      "episode 1111 score -19.2 average_score -31.5\n",
      "episode 1112 score -47.5 average_score -31.5\n",
      "episode 1113 score -31.5 average_score -31.4\n",
      "episode 1114 score -69.2 average_score -32.0\n",
      "episode 1115 score -18.5 average_score -32.3\n",
      "episode 1116 score -13.0 average_score -31.7\n",
      "episode 1117 score -31.2 average_score -31.7\n",
      "episode 1118 score -48.8 average_score -32.1\n",
      "episode 1119 score -13.8 average_score -31.6\n",
      "episode 1120 score -32.2 average_score -31.6\n",
      "episode 1121 score -26.2 average_score -31.6\n",
      "episode 1122 score -34.8 average_score -31.8\n",
      "episode 1123 score -34.0 average_score -31.8\n",
      "episode 1124 score -13.5 average_score -31.8\n",
      "episode 1125 score -13.5 average_score -31.6\n",
      "episode 1126 score -32.8 average_score -31.6\n",
      "episode 1127 score -37.5 average_score -31.6\n",
      "episode 1128 score -30.0 average_score -31.8\n",
      "episode 1129 score -31.5 average_score -31.9\n",
      "episode 1130 score -3.2 average_score -31.5\n",
      "episode 1131 score -40.5 average_score -31.6\n",
      "episode 1132 score -35.5 average_score -31.8\n",
      "episode 1133 score -27.8 average_score -31.8\n",
      "episode 1134 score -43.0 average_score -31.9\n",
      "episode 1135 score -34.2 average_score -32.0\n",
      "episode 1136 score -25.8 average_score -31.9\n",
      "episode 1137 score -38.8 average_score -32.1\n",
      "episode 1138 score -41.2 average_score -31.9\n",
      "episode 1139 score -39.5 average_score -32.2\n",
      "episode 1140 score -32.5 average_score -32.2\n",
      "episode 1141 score -44.0 average_score -32.0\n",
      "episode 1142 score 2.5 average_score -31.5\n",
      "episode 1143 score -26.0 average_score -31.5\n",
      "episode 1144 score -11.2 average_score -31.4\n",
      "episode 1145 score -7.8 average_score -31.4\n",
      "episode 1146 score -55.2 average_score -31.7\n",
      "episode 1147 score -41.8 average_score -31.9\n",
      "episode 1148 score -26.0 average_score -31.8\n",
      "episode 1149 score -74.0 average_score -32.2\n",
      "episode 1150 score -9.0 average_score -32.0\n",
      "episode 1151 score -26.2 average_score -31.8\n",
      "episode 1152 score -31.0 average_score -31.9\n",
      "episode 1153 score -39.5 average_score -31.8\n",
      "episode 1154 score -28.5 average_score -32.0\n",
      "episode 1155 score -38.2 average_score -31.9\n",
      "episode 1156 score -42.0 average_score -32.3\n",
      "episode 1157 score -34.0 average_score -32.3\n",
      "episode 1158 score -20.5 average_score -31.9\n",
      "episode 1159 score -3.2 average_score -31.9\n",
      "episode 1160 score -28.8 average_score -31.9\n",
      "episode 1161 score -47.8 average_score -32.2\n",
      "episode 1162 score -54.5 average_score -32.4\n",
      "episode 1163 score -29.0 average_score -32.5\n",
      "episode 1164 score -24.8 average_score -32.3\n",
      "episode 1165 score -18.0 average_score -32.0\n",
      "episode 1166 score -14.2 average_score -31.9\n",
      "episode 1167 score -26.5 average_score -32.0\n",
      "episode 1168 score -59.0 average_score -32.3\n",
      "episode 1169 score -46.2 average_score -33.1\n",
      "episode 1170 score -8.5 average_score -32.9\n",
      "episode 1171 score -20.8 average_score -32.8\n",
      "episode 1172 score -28.2 average_score -32.6\n",
      "episode 1173 score -16.2 average_score -32.6\n",
      "episode 1174 score -35.2 average_score -32.5\n",
      "episode 1175 score -14.5 average_score -32.3\n",
      "episode 1176 score -18.2 average_score -32.0\n",
      "episode 1177 score -38.8 average_score -31.9\n",
      "episode 1178 score -37.8 average_score -31.8\n",
      "episode 1179 score -27.0 average_score -31.9\n",
      "episode 1180 score -37.8 average_score -32.0\n",
      "episode 1181 score -23.8 average_score -31.8\n",
      "episode 1182 score -44.5 average_score -31.9\n",
      "episode 1183 score -19.8 average_score -32.0\n",
      "episode 1184 score -46.8 average_score -32.3\n",
      "episode 1185 score -40.5 average_score -32.4\n",
      "episode 1186 score -2.2 average_score -32.1\n",
      "episode 1187 score -31.0 average_score -32.0\n",
      "episode 1188 score -23.0 average_score -31.7\n",
      "episode 1189 score -56.0 average_score -31.7\n",
      "episode 1190 score -33.2 average_score -31.8\n",
      "episode 1191 score -26.2 average_score -31.6\n",
      "episode 1192 score -44.5 average_score -31.7\n",
      "episode 1193 score -10.5 average_score -31.5\n",
      "episode 1194 score -30.2 average_score -31.6\n",
      "episode 1195 score -6.8 average_score -31.4\n",
      "episode 1196 score -39.8 average_score -31.7\n",
      "episode 1197 score -10.2 average_score -31.5\n",
      "episode 1198 score -21.8 average_score -31.4\n",
      "episode 1199 score -9.2 average_score -31.3\n",
      "episode 1200 score -20.5 average_score -31.1\n",
      "episode 1201 score -28.0 average_score -31.1\n",
      "episode 1202 score -32.8 average_score -31.0\n",
      "episode 1203 score -27.8 average_score -31.0\n",
      "episode 1204 score 1.2 average_score -30.5\n",
      "episode 1205 score 2.2 average_score -29.8\n",
      "episode 1206 score -13.5 average_score -29.6\n",
      "episode 1207 score -8.8 average_score -28.8\n",
      "episode 1208 score -68.8 average_score -29.1\n",
      "episode 1209 score -30.8 average_score -29.1\n",
      "episode 1210 score -25.5 average_score -29.0\n",
      "episode 1211 score -28.8 average_score -29.1\n",
      "episode 1212 score -23.5 average_score -28.9\n",
      "episode 1213 score -41.2 average_score -29.0\n",
      "episode 1214 score -13.8 average_score -28.4\n",
      "episode 1215 score -17.5 average_score -28.4\n",
      "episode 1216 score -50.5 average_score -28.8\n",
      "episode 1217 score -25.0 average_score -28.7\n",
      "episode 1218 score -38.8 average_score -28.6\n",
      "episode 1219 score -27.5 average_score -28.7\n",
      "episode 1220 score -4.2 average_score -28.5\n",
      "episode 1221 score -16.2 average_score -28.4\n",
      "episode 1222 score -41.2 average_score -28.4\n",
      "episode 1223 score -40.8 average_score -28.5\n",
      "episode 1224 score -9.5 average_score -28.5\n",
      "episode 1225 score -4.0 average_score -28.4\n",
      "episode 1226 score -36.2 average_score -28.4\n",
      "episode 1227 score -26.5 average_score -28.3\n",
      "episode 1228 score -34.0 average_score -28.3\n",
      "episode 1229 score -40.2 average_score -28.4\n",
      "episode 1230 score -28.2 average_score -28.7\n",
      "episode 1231 score -44.0 average_score -28.7\n",
      "episode 1232 score 1.2 average_score -28.3\n",
      "episode 1233 score -36.2 average_score -28.4\n",
      "episode 1234 score -1.2 average_score -28.0\n",
      "episode 1235 score -17.8 average_score -27.8\n",
      "episode 1236 score -12.8 average_score -27.7\n",
      "episode 1237 score -17.8 average_score -27.5\n",
      "episode 1238 score -22.0 average_score -27.3\n",
      "episode 1239 score -35.8 average_score -27.3\n",
      "episode 1240 score -25.8 average_score -27.2\n",
      "episode 1241 score -41.8 average_score -27.2\n",
      "episode 1242 score -44.0 average_score -27.6\n",
      "episode 1243 score -31.0 average_score -27.7\n",
      "episode 1244 score -42.8 average_score -28.0\n",
      "episode 1245 score 9.2 average_score -27.8\n",
      "episode 1246 score -36.8 average_score -27.7\n",
      "episode 1247 score -41.5 average_score -27.6\n",
      "episode 1248 score -20.8 average_score -27.6\n",
      "episode 1249 score -27.0 average_score -27.1\n",
      "episode 1250 score -27.0 average_score -27.3\n",
      "episode 1251 score -24.8 average_score -27.3\n",
      "episode 1252 score -23.8 average_score -27.2\n",
      "episode 1253 score -27.5 average_score -27.1\n",
      "episode 1254 score -15.8 average_score -27.0\n",
      "episode 1255 score -27.0 average_score -26.9\n",
      "episode 1256 score -28.5 average_score -26.7\n",
      "episode 1257 score -15.5 average_score -26.5\n",
      "episode 1258 score -15.0 average_score -26.5\n",
      "episode 1259 score -13.0 average_score -26.6\n",
      "episode 1260 score -21.8 average_score -26.5\n",
      "episode 1261 score -45.2 average_score -26.5\n",
      "episode 1262 score -23.5 average_score -26.2\n",
      "episode 1263 score -52.2 average_score -26.4\n",
      "episode 1264 score -48.8 average_score -26.6\n",
      "episode 1265 score -54.8 average_score -27.0\n",
      "episode 1266 score -39.2 average_score -27.3\n",
      "episode 1267 score -24.8 average_score -27.2\n",
      "episode 1268 score -13.2 average_score -26.8\n",
      "episode 1269 score -32.5 average_score -26.7\n",
      "episode 1270 score -37.5 average_score -26.9\n",
      "episode 1271 score -35.8 average_score -27.1\n",
      "episode 1272 score -10.2 average_score -26.9\n",
      "episode 1273 score -49.8 average_score -27.2\n",
      "episode 1274 score -31.5 average_score -27.2\n",
      "episode 1275 score -35.0 average_score -27.4\n",
      "episode 1276 score -11.5 average_score -27.4\n",
      "episode 1277 score -13.5 average_score -27.1\n",
      "episode 1278 score -30.0 average_score -27.0\n",
      "episode 1279 score -26.0 average_score -27.0\n",
      "episode 1280 score -15.5 average_score -26.8\n",
      "episode 1281 score -20.8 average_score -26.8\n",
      "episode 1282 score -39.2 average_score -26.7\n",
      "episode 1283 score -36.0 average_score -26.9\n",
      "episode 1284 score -62.2 average_score -27.0\n",
      "episode 1285 score -51.0 average_score -27.1\n",
      "episode 1286 score -70.2 average_score -27.8\n",
      "episode 1287 score -17.0 average_score -27.7\n",
      "episode 1288 score -15.8 average_score -27.6\n",
      "episode 1289 score -41.5 average_score -27.4\n",
      "episode 1290 score -26.2 average_score -27.4\n",
      "episode 1291 score -34.5 average_score -27.5\n",
      "episode 1292 score -42.2 average_score -27.4\n",
      "episode 1293 score -46.5 average_score -27.8\n",
      "episode 1294 score -16.8 average_score -27.7\n",
      "episode 1295 score -4.5 average_score -27.6\n",
      "episode 1296 score -15.8 average_score -27.4\n",
      "episode 1297 score -43.2 average_score -27.7\n",
      "episode 1298 score -92.0 average_score -28.4\n",
      "episode 1299 score -36.5 average_score -28.7\n",
      "episode 1300 score -42.2 average_score -28.9\n",
      "episode 1301 score -19.8 average_score -28.8\n",
      "episode 1302 score -44.5 average_score -29.0\n",
      "episode 1303 score -21.2 average_score -28.9\n",
      "episode 1304 score -25.8 average_score -29.2\n",
      "episode 1305 score -33.5 average_score -29.5\n",
      "episode 1306 score -40.2 average_score -29.8\n",
      "episode 1307 score -23.0 average_score -29.9\n",
      "episode 1308 score -56.0 average_score -29.8\n",
      "episode 1309 score -27.8 average_score -29.8\n",
      "episode 1310 score -22.2 average_score -29.7\n",
      "episode 1311 score -4.2 average_score -29.5\n",
      "episode 1312 score -29.5 average_score -29.6\n",
      "episode 1313 score -25.8 average_score -29.4\n",
      "episode 1314 score -28.0 average_score -29.5\n",
      "episode 1315 score -9.8 average_score -29.5\n",
      "episode 1316 score -31.2 average_score -29.3\n",
      "episode 1317 score -67.0 average_score -29.7\n",
      "episode 1318 score -14.8 average_score -29.5\n",
      "episode 1319 score -17.8 average_score -29.4\n",
      "episode 1320 score -9.0 average_score -29.4\n",
      "episode 1321 score -23.2 average_score -29.5\n",
      "episode 1322 score -23.5 average_score -29.3\n",
      "episode 1323 score -21.2 average_score -29.1\n",
      "episode 1324 score -28.8 average_score -29.3\n",
      "episode 1325 score -1.8 average_score -29.3\n",
      "episode 1326 score -31.5 average_score -29.2\n",
      "episode 1327 score -36.2 average_score -29.3\n",
      "episode 1328 score -31.5 average_score -29.3\n",
      "episode 1329 score -17.2 average_score -29.1\n",
      "episode 1330 score -22.8 average_score -29.0\n",
      "episode 1331 score -25.5 average_score -28.8\n",
      "episode 1332 score -40.0 average_score -29.2\n",
      "episode 1333 score -27.8 average_score -29.2\n",
      "episode 1334 score -62.0 average_score -29.8\n",
      "episode 1335 score -11.5 average_score -29.7\n",
      "episode 1336 score -40.5 average_score -30.0\n",
      "episode 1337 score -54.5 average_score -30.3\n",
      "episode 1338 score -13.5 average_score -30.3\n",
      "episode 1339 score -11.0 average_score -30.0\n",
      "episode 1340 score -9.0 average_score -29.8\n",
      "episode 1341 score -35.8 average_score -29.8\n",
      "episode 1342 score -58.0 average_score -29.9\n",
      "episode 1343 score -41.5 average_score -30.0\n",
      "episode 1344 score -41.5 average_score -30.0\n",
      "episode 1345 score -51.5 average_score -30.6\n",
      "episode 1346 score -24.8 average_score -30.5\n",
      "episode 1347 score -34.2 average_score -30.4\n",
      "episode 1348 score -21.0 average_score -30.4\n",
      "episode 1349 score -46.2 average_score -30.6\n",
      "episode 1350 score -27.5 average_score -30.6\n",
      "episode 1351 score -29.2 average_score -30.7\n",
      "episode 1352 score -31.2 average_score -30.8\n",
      "episode 1353 score -3.0 average_score -30.5\n",
      "episode 1354 score -9.5 average_score -30.4\n",
      "episode 1355 score -42.0 average_score -30.6\n",
      "episode 1356 score -75.5 average_score -31.1\n",
      "episode 1357 score -48.5 average_score -31.4\n",
      "episode 1358 score -30.5 average_score -31.6\n",
      "episode 1359 score -43.8 average_score -31.9\n",
      "episode 1360 score -33.8 average_score -32.0\n",
      "episode 1361 score 1.8 average_score -31.5\n",
      "episode 1362 score -9.8 average_score -31.4\n",
      "episode 1363 score 1.8 average_score -30.8\n",
      "episode 1364 score -24.8 average_score -30.6\n",
      "episode 1365 score -34.8 average_score -30.4\n",
      "episode 1366 score -27.5 average_score -30.3\n",
      "episode 1367 score -70.5 average_score -30.7\n",
      "episode 1368 score -30.5 average_score -30.9\n",
      "episode 1369 score -45.5 average_score -31.0\n",
      "episode 1370 score -5.5 average_score -30.7\n",
      "episode 1371 score -27.8 average_score -30.6\n",
      "episode 1372 score -28.2 average_score -30.8\n",
      "episode 1373 score -42.5 average_score -30.7\n",
      "episode 1374 score -37.0 average_score -30.8\n",
      "episode 1375 score -32.2 average_score -30.8\n",
      "episode 1376 score -26.0 average_score -30.9\n",
      "episode 1377 score -11.5 average_score -30.9\n",
      "episode 1378 score -13.2 average_score -30.7\n",
      "episode 1379 score -36.0 average_score -30.8\n",
      "episode 1380 score -41.2 average_score -31.1\n",
      "episode 1381 score -44.8 average_score -31.3\n",
      "episode 1382 score -30.5 average_score -31.2\n",
      "episode 1383 score -37.5 average_score -31.2\n",
      "episode 1384 score -35.8 average_score -31.0\n",
      "episode 1385 score -22.8 average_score -30.7\n",
      "episode 1386 score -52.8 average_score -30.5\n",
      "episode 1387 score -10.2 average_score -30.5\n",
      "episode 1388 score -30.8 average_score -30.6\n",
      "episode 1389 score -10.8 average_score -30.3\n",
      "episode 1390 score -43.5 average_score -30.5\n",
      "episode 1391 score -57.5 average_score -30.7\n",
      "episode 1392 score -53.8 average_score -30.8\n",
      "episode 1393 score -36.8 average_score -30.7\n",
      "episode 1394 score -19.2 average_score -30.7\n",
      "episode 1395 score -28.8 average_score -31.0\n",
      "episode 1396 score -44.0 average_score -31.3\n",
      "episode 1397 score -43.2 average_score -31.3\n",
      "episode 1398 score -22.2 average_score -30.6\n",
      "episode 1399 score -26.2 average_score -30.5\n",
      "episode 1400 score -7.5 average_score -30.1\n",
      "episode 1401 score -21.0 average_score -30.1\n",
      "episode 1402 score -24.2 average_score -29.9\n",
      "episode 1403 score 10.0 average_score -29.6\n",
      "episode 1404 score -28.8 average_score -29.7\n",
      "episode 1405 score -29.5 average_score -29.6\n",
      "episode 1406 score -23.8 average_score -29.4\n",
      "episode 1407 score -22.5 average_score -29.4\n",
      "episode 1408 score -39.8 average_score -29.3\n",
      "episode 1409 score -11.0 average_score -29.1\n",
      "episode 1410 score -10.2 average_score -29.0\n",
      "episode 1411 score -39.5 average_score -29.3\n",
      "episode 1412 score -36.5 average_score -29.4\n",
      "episode 1413 score -0.2 average_score -29.2\n",
      "episode 1414 score -36.2 average_score -29.2\n",
      "episode 1415 score -33.2 average_score -29.5\n",
      "episode 1416 score -37.2 average_score -29.5\n",
      "episode 1417 score -48.8 average_score -29.4\n",
      "episode 1418 score -55.5 average_score -29.8\n",
      "episode 1419 score -41.0 average_score -30.0\n",
      "episode 1420 score -29.8 average_score -30.2\n",
      "episode 1421 score -44.0 average_score -30.4\n",
      "episode 1422 score -27.0 average_score -30.4\n",
      "episode 1423 score -45.2 average_score -30.7\n",
      "episode 1424 score -27.8 average_score -30.7\n",
      "episode 1425 score -16.2 average_score -30.8\n",
      "episode 1426 score 15.8 average_score -30.3\n",
      "episode 1427 score -41.8 average_score -30.4\n",
      "episode 1428 score -33.2 average_score -30.4\n",
      "episode 1429 score -35.0 average_score -30.6\n",
      "episode 1430 score -40.2 average_score -30.8\n",
      "episode 1431 score -23.0 average_score -30.7\n",
      "episode 1432 score -48.8 average_score -30.8\n",
      "episode 1433 score -19.5 average_score -30.8\n",
      "episode 1434 score -60.8 average_score -30.7\n",
      "episode 1435 score -56.8 average_score -31.2\n",
      "episode 1436 score -43.8 average_score -31.2\n",
      "episode 1437 score -14.5 average_score -30.8\n",
      "episode 1438 score -33.5 average_score -31.0\n",
      "episode 1439 score -52.0 average_score -31.4\n",
      "episode 1440 score -8.0 average_score -31.4\n",
      "episode 1441 score -28.5 average_score -31.4\n",
      "episode 1442 score -36.2 average_score -31.1\n",
      "episode 1443 score -33.8 average_score -31.1\n",
      "episode 1444 score -47.8 average_score -31.1\n",
      "episode 1445 score -20.2 average_score -30.8\n",
      "episode 1446 score -11.8 average_score -30.7\n",
      "episode 1447 score -45.5 average_score -30.8\n",
      "episode 1448 score -25.0 average_score -30.8\n",
      "episode 1449 score -41.0 average_score -30.8\n",
      "episode 1450 score -3.0 average_score -30.5\n",
      "episode 1451 score -10.2 average_score -30.3\n",
      "episode 1452 score -26.5 average_score -30.3\n",
      "episode 1453 score 6.0 average_score -30.2\n",
      "episode 1454 score -51.8 average_score -30.6\n",
      "episode 1455 score -48.0 average_score -30.7\n",
      "episode 1456 score -26.2 average_score -30.2\n",
      "episode 1457 score -40.8 average_score -30.1\n",
      "episode 1458 score -31.0 average_score -30.1\n",
      "episode 1459 score -67.5 average_score -30.4\n",
      "episode 1460 score -38.0 average_score -30.4\n",
      "episode 1461 score -43.8 average_score -30.9\n",
      "episode 1462 score 22.2 average_score -30.5\n",
      "episode 1463 score -24.0 average_score -30.8\n",
      "episode 1464 score -16.8 average_score -30.7\n",
      "episode 1465 score -10.0 average_score -30.5\n",
      "episode 1466 score -77.0 average_score -31.0\n",
      "episode 1467 score -34.5 average_score -30.6\n",
      "episode 1468 score -16.5 average_score -30.5\n",
      "episode 1469 score -28.0 average_score -30.3\n",
      "episode 1470 score -33.2 average_score -30.6\n",
      "episode 1471 score -48.8 average_score -30.8\n",
      "episode 1472 score -26.8 average_score -30.8\n",
      "episode 1473 score 4.5 average_score -30.3\n",
      "episode 1474 score -41.2 average_score -30.3\n",
      "episode 1475 score -16.2 average_score -30.2\n",
      "episode 1476 score -28.8 average_score -30.2\n",
      "episode 1477 score -12.0 average_score -30.2\n",
      "episode 1478 score -69.2 average_score -30.8\n",
      "episode 1479 score -35.5 average_score -30.8\n",
      "episode 1480 score -14.2 average_score -30.5\n",
      "episode 1481 score -27.5 average_score -30.3\n",
      "episode 1482 score -25.2 average_score -30.3\n",
      "episode 1483 score -5.8 average_score -29.9\n",
      "episode 1484 score -42.0 average_score -30.0\n",
      "episode 1485 score -3.5 average_score -29.8\n",
      "episode 1486 score -14.5 average_score -29.4\n",
      "episode 1487 score -19.2 average_score -29.5\n",
      "episode 1488 score -37.5 average_score -29.6\n",
      "episode 1489 score -25.5 average_score -29.7\n",
      "episode 1490 score -22.0 average_score -29.5\n",
      "episode 1491 score -19.2 average_score -29.1\n",
      "episode 1492 score -33.8 average_score -28.9\n",
      "episode 1493 score -53.0 average_score -29.1\n",
      "episode 1494 score -47.2 average_score -29.4\n",
      "episode 1495 score -38.5 average_score -29.5\n",
      "episode 1496 score -39.0 average_score -29.4\n",
      "episode 1497 score -12.8 average_score -29.1\n",
      "episode 1498 score -23.5 average_score -29.1\n",
      "episode 1499 score -28.8 average_score -29.2\n",
      "episode 1500 score -19.2 average_score -29.3\n",
      "episode 1501 score -36.0 average_score -29.4\n",
      "episode 1502 score -34.8 average_score -29.5\n",
      "episode 1503 score -51.8 average_score -30.2\n",
      "episode 1504 score -25.5 average_score -30.1\n",
      "episode 1505 score 0.2 average_score -29.8\n",
      "episode 1506 score -41.5 average_score -30.0\n",
      "episode 1507 score -44.5 average_score -30.2\n",
      "episode 1508 score -24.2 average_score -30.1\n",
      "episode 1509 score -45.0 average_score -30.4\n",
      "episode 1510 score -62.8 average_score -30.9\n",
      "episode 1511 score 9.0 average_score -30.4\n",
      "episode 1512 score -60.5 average_score -30.7\n",
      "episode 1513 score -50.8 average_score -31.2\n",
      "episode 1514 score -26.5 average_score -31.1\n",
      "episode 1515 score -32.8 average_score -31.1\n",
      "episode 1516 score -14.5 average_score -30.9\n",
      "episode 1517 score 1.2 average_score -30.4\n",
      "episode 1518 score -14.2 average_score -29.9\n",
      "episode 1519 score -22.8 average_score -29.8\n",
      "episode 1520 score -12.0 average_score -29.6\n",
      "episode 1521 score -59.2 average_score -29.7\n",
      "episode 1522 score -21.8 average_score -29.7\n",
      "episode 1523 score -33.0 average_score -29.6\n",
      "episode 1524 score -21.5 average_score -29.5\n",
      "episode 1525 score -23.2 average_score -29.6\n",
      "episode 1526 score -38.2 average_score -30.1\n",
      "episode 1527 score -58.2 average_score -30.3\n",
      "episode 1528 score -37.8 average_score -30.3\n",
      "episode 1529 score -28.8 average_score -30.3\n",
      "episode 1530 score -50.2 average_score -30.4\n",
      "episode 1531 score -47.2 average_score -30.6\n",
      "episode 1532 score -50.2 average_score -30.6\n",
      "episode 1533 score -29.0 average_score -30.7\n",
      "episode 1534 score -54.0 average_score -30.6\n",
      "episode 1535 score -43.8 average_score -30.5\n",
      "episode 1536 score -26.0 average_score -30.3\n",
      "episode 1537 score -16.8 average_score -30.4\n",
      "episode 1538 score 13.5 average_score -29.9\n",
      "episode 1539 score -29.2 average_score -29.7\n",
      "episode 1540 score -29.8 average_score -29.9\n",
      "episode 1541 score -38.2 average_score -30.0\n",
      "episode 1542 score -25.5 average_score -29.9\n",
      "episode 1543 score -40.8 average_score -29.9\n",
      "episode 1544 score -4.8 average_score -29.5\n",
      "episode 1545 score -39.5 average_score -29.7\n",
      "episode 1546 score -37.5 average_score -30.0\n",
      "episode 1547 score -47.8 average_score -30.0\n",
      "episode 1548 score -43.8 average_score -30.2\n",
      "episode 1549 score -24.2 average_score -30.0\n",
      "episode 1550 score -20.0 average_score -30.2\n",
      "episode 1551 score -40.8 average_score -30.5\n",
      "episode 1552 score -41.2 average_score -30.6\n",
      "episode 1553 score -40.0 average_score -31.1\n",
      "episode 1554 score -65.5 average_score -31.2\n",
      "episode 1555 score -23.5 average_score -31.0\n",
      "episode 1556 score -26.0 average_score -31.0\n",
      "episode 1557 score -5.0 average_score -30.6\n",
      "episode 1558 score -28.2 average_score -30.6\n",
      "episode 1559 score -16.8 average_score -30.1\n",
      "episode 1560 score -48.8 average_score -30.2\n",
      "episode 1561 score -27.8 average_score -30.0\n",
      "episode 1562 score -30.2 average_score -30.6\n",
      "episode 1563 score -34.0 average_score -30.7\n",
      "episode 1564 score -29.0 average_score -30.8\n",
      "episode 1565 score 4.5 average_score -30.6\n",
      "episode 1566 score -49.0 average_score -30.4\n",
      "episode 1567 score -61.2 average_score -30.6\n",
      "episode 1568 score 1.5 average_score -30.4\n",
      "episode 1569 score -3.8 average_score -30.2\n",
      "episode 1570 score -33.8 average_score -30.2\n",
      "episode 1571 score -9.8 average_score -29.8\n",
      "episode 1572 score -6.5 average_score -29.6\n",
      "episode 1573 score -34.5 average_score -30.0\n",
      "episode 1574 score -28.0 average_score -29.9\n",
      "episode 1575 score -55.0 average_score -30.3\n",
      "episode 1576 score -2.2 average_score -30.0\n",
      "episode 1577 score -28.0 average_score -30.2\n",
      "episode 1578 score -38.5 average_score -29.8\n",
      "episode 1579 score -48.8 average_score -30.0\n",
      "episode 1580 score -17.5 average_score -30.0\n",
      "episode 1581 score -38.5 average_score -30.1\n",
      "episode 1582 score -19.5 average_score -30.1\n",
      "episode 1583 score -32.8 average_score -30.3\n",
      "episode 1584 score -40.2 average_score -30.3\n",
      "episode 1585 score -32.5 average_score -30.6\n",
      "episode 1586 score -36.5 average_score -30.8\n",
      "episode 1587 score -22.5 average_score -30.9\n",
      "episode 1588 score -31.5 average_score -30.8\n",
      "episode 1589 score 12.2 average_score -30.4\n",
      "episode 1590 score -25.0 average_score -30.4\n",
      "episode 1591 score -18.8 average_score -30.4\n",
      "episode 1592 score -36.8 average_score -30.5\n",
      "episode 1593 score 10.8 average_score -29.8\n",
      "episode 1594 score -33.5 average_score -29.7\n",
      "episode 1595 score -13.5 average_score -29.4\n",
      "episode 1596 score 2.8 average_score -29.0\n",
      "episode 1597 score -22.0 average_score -29.1\n",
      "episode 1598 score -17.5 average_score -29.1\n",
      "episode 1599 score -21.5 average_score -29.0\n",
      "episode 1600 score -3.0 average_score -28.8\n",
      "episode 1601 score -29.2 average_score -28.8\n",
      "episode 1602 score -32.8 average_score -28.7\n",
      "episode 1603 score -35.2 average_score -28.6\n",
      "episode 1604 score -46.0 average_score -28.8\n",
      "episode 1605 score -7.5 average_score -28.9\n",
      "episode 1606 score -41.0 average_score -28.9\n",
      "episode 1607 score -21.2 average_score -28.6\n",
      "episode 1608 score -31.5 average_score -28.7\n",
      "episode 1609 score -11.0 average_score -28.4\n",
      "episode 1610 score -23.5 average_score -28.0\n",
      "episode 1611 score -15.5 average_score -28.2\n",
      "episode 1612 score -44.0 average_score -28.0\n",
      "episode 1613 score -37.5 average_score -27.9\n",
      "episode 1614 score -26.5 average_score -27.9\n",
      "episode 1615 score -14.8 average_score -27.7\n",
      "episode 1616 score -33.0 average_score -27.9\n",
      "episode 1617 score -30.8 average_score -28.2\n",
      "episode 1618 score -36.8 average_score -28.5\n",
      "episode 1619 score -39.0 average_score -28.6\n",
      "episode 1620 score -27.5 average_score -28.8\n",
      "episode 1621 score -21.0 average_score -28.4\n",
      "episode 1622 score -24.8 average_score -28.4\n",
      "episode 1623 score -38.5 average_score -28.5\n",
      "episode 1624 score -11.0 average_score -28.4\n",
      "episode 1625 score -18.0 average_score -28.3\n",
      "episode 1626 score -19.2 average_score -28.1\n",
      "episode 1627 score -39.8 average_score -27.9\n",
      "episode 1628 score -17.8 average_score -27.7\n",
      "episode 1629 score -33.0 average_score -27.8\n",
      "episode 1630 score -34.8 average_score -27.6\n",
      "episode 1631 score -28.2 average_score -27.4\n",
      "episode 1632 score -51.5 average_score -27.5\n",
      "episode 1633 score -27.2 average_score -27.4\n",
      "episode 1634 score -35.8 average_score -27.3\n",
      "episode 1635 score -28.8 average_score -27.1\n",
      "episode 1636 score -36.0 average_score -27.2\n",
      "episode 1637 score 6.5 average_score -27.0\n",
      "episode 1638 score -11.8 average_score -27.2\n",
      "episode 1639 score -25.5 average_score -27.2\n",
      "episode 1640 score -21.8 average_score -27.1\n",
      "episode 1641 score 6.5 average_score -26.7\n",
      "episode 1642 score -16.5 average_score -26.6\n",
      "episode 1643 score -25.5 average_score -26.4\n",
      "episode 1644 score -21.2 average_score -26.6\n",
      "episode 1645 score -52.8 average_score -26.7\n",
      "episode 1646 score -43.2 average_score -26.8\n",
      "episode 1647 score 4.2 average_score -26.3\n",
      "episode 1648 score -49.0 average_score -26.3\n",
      "episode 1649 score -26.0 average_score -26.3\n",
      "episode 1650 score -37.5 average_score -26.5\n",
      "episode 1651 score -21.0 average_score -26.3\n",
      "episode 1652 score -22.5 average_score -26.1\n",
      "episode 1653 score -23.5 average_score -25.9\n",
      "episode 1654 score -37.5 average_score -25.7\n",
      "episode 1655 score -52.2 average_score -26.0\n",
      "episode 1656 score -28.0 average_score -26.0\n",
      "episode 1657 score -48.5 average_score -26.4\n",
      "episode 1658 score -41.0 average_score -26.5\n",
      "episode 1659 score -27.2 average_score -26.6\n",
      "episode 1660 score -36.2 average_score -26.5\n",
      "episode 1661 score -18.5 average_score -26.4\n",
      "episode 1662 score -31.0 average_score -26.4\n",
      "episode 1663 score -29.8 average_score -26.4\n",
      "episode 1664 score -32.0 average_score -26.4\n",
      "episode 1665 score -28.2 average_score -26.8\n",
      "episode 1666 score -52.8 average_score -26.8\n",
      "episode 1667 score -30.2 average_score -26.5\n",
      "episode 1668 score -53.0 average_score -27.0\n",
      "episode 1669 score -56.8 average_score -27.6\n",
      "episode 1670 score -38.8 average_score -27.6\n",
      "episode 1671 score -22.8 average_score -27.7\n",
      "episode 1672 score -60.8 average_score -28.3\n",
      "episode 1673 score -29.2 average_score -28.2\n",
      "episode 1674 score -51.2 average_score -28.5\n",
      "episode 1675 score -33.2 average_score -28.2\n",
      "episode 1676 score -23.0 average_score -28.4\n",
      "episode 1677 score 6.2 average_score -28.1\n",
      "episode 1678 score -20.5 average_score -27.9\n",
      "episode 1679 score -5.8 average_score -27.5\n",
      "episode 1680 score -10.0 average_score -27.4\n",
      "episode 1681 score -43.0 average_score -27.5\n",
      "episode 1682 score -56.0 average_score -27.8\n",
      "episode 1683 score -13.0 average_score -27.6\n",
      "episode 1684 score -33.5 average_score -27.6\n",
      "episode 1685 score -24.2 average_score -27.5\n",
      "episode 1686 score -34.8 average_score -27.5\n",
      "episode 1687 score -45.0 average_score -27.7\n",
      "episode 1688 score -78.2 average_score -28.2\n",
      "episode 1689 score -38.5 average_score -28.7\n",
      "episode 1690 score -62.5 average_score -29.0\n",
      "episode 1691 score -15.8 average_score -29.0\n",
      "episode 1692 score -13.5 average_score -28.8\n",
      "episode 1693 score -43.8 average_score -29.3\n",
      "episode 1694 score -4.8 average_score -29.0\n",
      "episode 1695 score -17.5 average_score -29.1\n",
      "episode 1696 score -36.0 average_score -29.5\n",
      "episode 1697 score -3.0 average_score -29.3\n",
      "episode 1698 score -6.0 average_score -29.2\n",
      "episode 1699 score -65.5 average_score -29.6\n",
      "episode 1700 score -43.2 average_score -30.0\n",
      "episode 1701 score -28.2 average_score -30.0\n",
      "episode 1702 score -25.5 average_score -29.9\n",
      "episode 1703 score -21.0 average_score -29.8\n",
      "episode 1704 score -40.2 average_score -29.7\n",
      "episode 1705 score -23.0 average_score -29.9\n",
      "episode 1706 score -34.2 average_score -29.8\n",
      "episode 1707 score -49.5 average_score -30.1\n",
      "episode 1708 score -13.2 average_score -29.9\n",
      "episode 1709 score -20.8 average_score -30.0\n",
      "episode 1710 score -44.5 average_score -30.2\n",
      "episode 1711 score -30.5 average_score -30.4\n",
      "episode 1712 score -42.0 average_score -30.3\n",
      "episode 1713 score -43.2 average_score -30.4\n",
      "episode 1714 score -3.8 average_score -30.2\n",
      "episode 1715 score -67.8 average_score -30.7\n",
      "episode 1716 score -38.0 average_score -30.8\n",
      "episode 1717 score -40.2 average_score -30.8\n",
      "episode 1718 score -61.8 average_score -31.1\n",
      "episode 1719 score -16.8 average_score -30.9\n",
      "episode 1720 score -40.8 average_score -31.0\n",
      "episode 1721 score -20.8 average_score -31.0\n",
      "episode 1722 score -1.5 average_score -30.8\n",
      "episode 1723 score -12.2 average_score -30.5\n",
      "episode 1724 score -23.0 average_score -30.6\n",
      "episode 1725 score -59.8 average_score -31.0\n",
      "episode 1726 score -74.5 average_score -31.6\n",
      "episode 1727 score -13.5 average_score -31.3\n",
      "episode 1728 score -51.5 average_score -31.7\n",
      "episode 1729 score -0.2 average_score -31.3\n",
      "episode 1730 score -16.2 average_score -31.2\n",
      "episode 1731 score -35.8 average_score -31.2\n",
      "episode 1732 score -53.5 average_score -31.3\n",
      "episode 1733 score -14.0 average_score -31.1\n",
      "episode 1734 score -3.8 average_score -30.8\n",
      "episode 1735 score -20.8 average_score -30.7\n",
      "episode 1736 score -43.5 average_score -30.8\n",
      "episode 1737 score -47.2 average_score -31.3\n",
      "episode 1738 score -31.2 average_score -31.5\n",
      "episode 1739 score -28.2 average_score -31.6\n",
      "episode 1740 score -26.5 average_score -31.6\n",
      "episode 1741 score -24.0 average_score -31.9\n",
      "episode 1742 score -36.5 average_score -32.1\n",
      "episode 1743 score -36.2 average_score -32.2\n",
      "episode 1744 score -50.5 average_score -32.5\n",
      "episode 1745 score -28.8 average_score -32.3\n",
      "episode 1746 score -22.5 average_score -32.1\n",
      "episode 1747 score -22.5 average_score -32.3\n",
      "episode 1748 score -52.5 average_score -32.4\n",
      "episode 1749 score -33.0 average_score -32.4\n",
      "episode 1750 score -24.0 average_score -32.3\n",
      "episode 1751 score -27.8 average_score -32.4\n",
      "episode 1752 score -35.0 average_score -32.5\n",
      "episode 1753 score -30.2 average_score -32.6\n",
      "episode 1754 score -60.2 average_score -32.8\n",
      "episode 1755 score -34.8 average_score -32.6\n",
      "episode 1756 score -44.2 average_score -32.8\n",
      "episode 1757 score -54.5 average_score -32.8\n",
      "episode 1758 score -34.5 average_score -32.8\n",
      "episode 1759 score -45.2 average_score -33.0\n",
      "episode 1760 score -29.5 average_score -32.9\n",
      "episode 1761 score -19.8 average_score -32.9\n",
      "episode 1762 score -28.0 average_score -32.9\n",
      "episode 1763 score -24.8 average_score -32.8\n",
      "episode 1764 score -18.8 average_score -32.7\n",
      "episode 1765 score -66.0 average_score -33.1\n",
      "episode 1766 score -15.5 average_score -32.7\n",
      "episode 1767 score -47.5 average_score -32.9\n",
      "episode 1768 score -55.2 average_score -32.9\n",
      "episode 1769 score -20.0 average_score -32.5\n",
      "episode 1770 score -36.0 average_score -32.5\n",
      "episode 1771 score -17.5 average_score -32.4\n",
      "episode 1772 score -44.5 average_score -32.3\n",
      "episode 1773 score -27.8 average_score -32.3\n",
      "episode 1774 score -5.0 average_score -31.8\n",
      "episode 1775 score -27.2 average_score -31.7\n",
      "episode 1776 score -28.0 average_score -31.8\n",
      "episode 1777 score -40.5 average_score -32.3\n",
      "episode 1778 score -12.8 average_score -32.2\n",
      "episode 1779 score -25.0 average_score -32.4\n",
      "episode 1780 score -5.5 average_score -32.3\n",
      "episode 1781 score -24.2 average_score -32.1\n",
      "episode 1782 score -9.2 average_score -31.7\n",
      "episode 1783 score -27.0 average_score -31.8\n",
      "episode 1784 score -40.2 average_score -31.9\n",
      "episode 1785 score -31.0 average_score -31.9\n",
      "episode 1786 score -37.8 average_score -32.0\n",
      "episode 1787 score -13.5 average_score -31.7\n",
      "episode 1788 score -33.0 average_score -31.2\n",
      "episode 1789 score -33.8 average_score -31.2\n",
      "episode 1790 score -29.2 average_score -30.8\n",
      "episode 1791 score -60.5 average_score -31.3\n",
      "episode 1792 score -22.8 average_score -31.4\n",
      "episode 1793 score -23.5 average_score -31.2\n",
      "episode 1794 score -21.0 average_score -31.3\n",
      "episode 1795 score -47.2 average_score -31.6\n",
      "episode 1796 score -18.2 average_score -31.4\n",
      "episode 1797 score -22.5 average_score -31.6\n",
      "episode 1798 score -32.8 average_score -31.9\n",
      "episode 1799 score -26.5 average_score -31.5\n",
      "episode 1800 score -35.2 average_score -31.4\n",
      "episode 1801 score -51.5 average_score -31.7\n",
      "episode 1802 score -25.2 average_score -31.7\n",
      "episode 1803 score -59.8 average_score -32.1\n",
      "episode 1804 score -26.0 average_score -31.9\n",
      "episode 1805 score -37.5 average_score -32.1\n",
      "episode 1806 score -21.0 average_score -31.9\n",
      "episode 1807 score 19.5 average_score -31.2\n",
      "episode 1808 score -34.5 average_score -31.4\n",
      "episode 1809 score -32.0 average_score -31.6\n",
      "episode 1810 score -38.8 average_score -31.5\n",
      "episode 1811 score -46.2 average_score -31.7\n",
      "episode 1812 score -21.0 average_score -31.4\n",
      "episode 1813 score -23.5 average_score -31.3\n",
      "episode 1814 score -13.5 average_score -31.4\n",
      "episode 1815 score -23.5 average_score -30.9\n",
      "episode 1816 score -12.5 average_score -30.7\n",
      "episode 1817 score -57.0 average_score -30.8\n",
      "episode 1818 score -41.0 average_score -30.6\n",
      "episode 1819 score -28.2 average_score -30.7\n",
      "episode 1820 score -18.5 average_score -30.5\n",
      "episode 1821 score -22.5 average_score -30.5\n",
      "episode 1822 score -25.8 average_score -30.8\n",
      "episode 1823 score -27.8 average_score -30.9\n",
      "episode 1824 score -32.0 average_score -31.0\n",
      "episode 1825 score -30.2 average_score -30.7\n",
      "episode 1826 score -34.8 average_score -30.3\n",
      "episode 1827 score -33.2 average_score -30.5\n",
      "episode 1828 score -15.8 average_score -30.2\n",
      "episode 1829 score -31.8 average_score -30.5\n",
      "episode 1830 score -31.5 average_score -30.6\n",
      "episode 1831 score 19.8 average_score -30.1\n",
      "episode 1832 score -58.2 average_score -30.1\n",
      "episode 1833 score -15.2 average_score -30.1\n",
      "episode 1834 score -34.5 average_score -30.4\n",
      "episode 1835 score -45.0 average_score -30.7\n",
      "episode 1836 score -33.5 average_score -30.6\n",
      "episode 1837 score -21.2 average_score -30.3\n",
      "episode 1838 score -53.0 average_score -30.5\n",
      "episode 1839 score -18.5 average_score -30.4\n",
      "episode 1840 score -0.2 average_score -30.2\n",
      "episode 1841 score -27.0 average_score -30.2\n",
      "episode 1842 score -30.0 average_score -30.1\n",
      "episode 1843 score 11.0 average_score -29.7\n",
      "episode 1844 score -32.5 average_score -29.5\n",
      "episode 1845 score -19.0 average_score -29.4\n",
      "episode 1846 score -60.2 average_score -29.8\n",
      "episode 1847 score -25.2 average_score -29.8\n",
      "episode 1848 score -29.5 average_score -29.6\n",
      "episode 1849 score -48.2 average_score -29.7\n",
      "episode 1850 score -12.5 average_score -29.6\n",
      "episode 1851 score -19.8 average_score -29.5\n",
      "episode 1852 score -58.5 average_score -29.8\n",
      "episode 1853 score -32.0 average_score -29.8\n",
      "episode 1854 score -17.5 average_score -29.4\n",
      "episode 1855 score -2.8 average_score -29.0\n",
      "episode 1856 score -24.2 average_score -28.8\n",
      "episode 1857 score -11.5 average_score -28.4\n",
      "episode 1858 score -35.2 average_score -28.4\n",
      "episode 1859 score -19.5 average_score -28.1\n",
      "episode 1860 score -24.8 average_score -28.1\n",
      "episode 1861 score -14.8 average_score -28.1\n",
      "episode 1862 score -28.2 average_score -28.1\n",
      "episode 1863 score -7.0 average_score -27.9\n",
      "episode 1864 score -11.2 average_score -27.8\n",
      "episode 1865 score -47.2 average_score -27.6\n",
      "episode 1866 score -53.5 average_score -28.0\n",
      "episode 1867 score -12.8 average_score -27.6\n",
      "episode 1868 score -71.0 average_score -27.8\n",
      "episode 1869 score -45.8 average_score -28.1\n",
      "episode 1870 score -23.0 average_score -27.9\n",
      "episode 1871 score -70.2 average_score -28.5\n",
      "episode 1872 score -37.5 average_score -28.4\n",
      "episode 1873 score 1.5 average_score -28.1\n",
      "episode 1874 score -37.2 average_score -28.4\n",
      "episode 1875 score -50.2 average_score -28.6\n",
      "episode 1876 score -45.8 average_score -28.8\n",
      "episode 1877 score -42.8 average_score -28.9\n",
      "episode 1878 score -25.8 average_score -29.0\n",
      "episode 1879 score -19.0 average_score -28.9\n",
      "episode 1880 score -38.5 average_score -29.2\n",
      "episode 1881 score -10.2 average_score -29.1\n",
      "episode 1882 score -12.8 average_score -29.1\n",
      "episode 1883 score -36.2 average_score -29.2\n",
      "episode 1884 score -9.5 average_score -28.9\n",
      "episode 1885 score -25.2 average_score -28.9\n",
      "episode 1886 score -59.5 average_score -29.1\n",
      "episode 1887 score -29.2 average_score -29.2\n",
      "episode 1888 score -4.5 average_score -29.0\n",
      "episode 1889 score 0.2 average_score -28.6\n",
      "episode 1890 score -4.8 average_score -28.4\n",
      "episode 1891 score -56.0 average_score -28.3\n",
      "episode 1892 score -37.5 average_score -28.5\n",
      "episode 1893 score -45.5 average_score -28.7\n",
      "episode 1894 score -13.0 average_score -28.6\n",
      "episode 1895 score -16.2 average_score -28.3\n",
      "episode 1896 score -28.0 average_score -28.4\n",
      "episode 1897 score -57.0 average_score -28.8\n",
      "episode 1898 score -30.0 average_score -28.7\n",
      "episode 1899 score -28.8 average_score -28.7\n",
      "episode 1900 score -28.8 average_score -28.7\n",
      "episode 1901 score -43.0 average_score -28.6\n",
      "episode 1902 score -54.0 average_score -28.9\n",
      "episode 1903 score -20.2 average_score -28.5\n",
      "episode 1904 score -40.2 average_score -28.6\n",
      "episode 1905 score -8.8 average_score -28.3\n",
      "episode 1906 score -11.5 average_score -28.2\n",
      "episode 1907 score -42.8 average_score -28.9\n",
      "episode 1908 score -20.8 average_score -28.7\n",
      "episode 1909 score -16.0 average_score -28.6\n",
      "episode 1910 score -16.5 average_score -28.4\n",
      "episode 1911 score -48.2 average_score -28.4\n",
      "episode 1912 score -18.5 average_score -28.3\n",
      "episode 1913 score -11.0 average_score -28.2\n",
      "episode 1914 score -41.8 average_score -28.5\n",
      "episode 1915 score 2.2 average_score -28.2\n",
      "episode 1916 score -56.2 average_score -28.7\n",
      "episode 1917 score -39.0 average_score -28.5\n",
      "episode 1918 score -53.5 average_score -28.6\n",
      "episode 1919 score -22.2 average_score -28.6\n",
      "episode 1920 score -37.5 average_score -28.8\n",
      "episode 1921 score -62.8 average_score -29.2\n",
      "episode 1922 score -23.0 average_score -29.1\n",
      "episode 1923 score -38.2 average_score -29.2\n",
      "episode 1924 score -30.2 average_score -29.2\n",
      "episode 1925 score -6.0 average_score -29.0\n",
      "episode 1926 score -4.8 average_score -28.7\n",
      "episode 1927 score -60.0 average_score -28.9\n",
      "episode 1928 score -40.0 average_score -29.2\n",
      "episode 1929 score -41.8 average_score -29.3\n",
      "episode 1930 score -0.8 average_score -29.0\n",
      "episode 1931 score -38.8 average_score -29.6\n",
      "episode 1932 score -21.2 average_score -29.2\n",
      "episode 1933 score -49.0 average_score -29.5\n",
      "episode 1934 score -29.0 average_score -29.5\n",
      "episode 1935 score -36.8 average_score -29.4\n",
      "episode 1936 score -23.8 average_score -29.3\n",
      "episode 1937 score -31.2 average_score -29.4\n",
      "episode 1938 score -38.8 average_score -29.3\n",
      "episode 1939 score -15.5 average_score -29.2\n",
      "episode 1940 score -27.2 average_score -29.5\n",
      "episode 1941 score -57.5 average_score -29.8\n",
      "episode 1942 score -26.2 average_score -29.8\n",
      "episode 1943 score -40.5 average_score -30.3\n",
      "episode 1944 score -37.5 average_score -30.3\n",
      "episode 1945 score -46.5 average_score -30.6\n",
      "episode 1946 score -31.5 average_score -30.3\n",
      "episode 1947 score -38.0 average_score -30.4\n",
      "episode 1948 score -76.0 average_score -30.9\n",
      "episode 1949 score -39.5 average_score -30.8\n",
      "episode 1950 score -32.8 average_score -31.0\n",
      "episode 1951 score -17.2 average_score -31.0\n",
      "episode 1952 score -30.0 average_score -30.7\n",
      "episode 1953 score -40.5 average_score -30.8\n",
      "episode 1954 score -45.2 average_score -31.1\n",
      "episode 1955 score -15.0 average_score -31.2\n",
      "episode 1956 score -44.2 average_score -31.4\n",
      "episode 1957 score -39.2 average_score -31.7\n",
      "episode 1958 score -34.2 average_score -31.7\n",
      "episode 1959 score -32.2 average_score -31.8\n",
      "episode 1960 score -33.5 average_score -31.9\n",
      "episode 1961 score -7.8 average_score -31.8\n",
      "episode 1962 score -53.0 average_score -32.1\n",
      "episode 1963 score -34.8 average_score -32.3\n",
      "episode 1964 score -23.0 average_score -32.5\n",
      "episode 1965 score -18.8 average_score -32.2\n",
      "episode 1966 score -20.5 average_score -31.8\n",
      "episode 1967 score -22.0 average_score -31.9\n",
      "episode 1968 score -27.2 average_score -31.5\n",
      "episode 1969 score 4.8 average_score -31.0\n",
      "episode 1970 score -24.8 average_score -31.0\n",
      "episode 1971 score -17.0 average_score -30.5\n",
      "episode 1972 score -8.5 average_score -30.2\n",
      "episode 1973 score -15.5 average_score -30.4\n",
      "episode 1974 score 16.2 average_score -29.8\n",
      "episode 1975 score -5.8 average_score -29.4\n",
      "episode 1976 score -47.2 average_score -29.4\n",
      "episode 1977 score -29.8 average_score -29.3\n",
      "episode 1978 score -18.5 average_score -29.2\n",
      "episode 1979 score -38.5 average_score -29.4\n",
      "episode 1980 score -45.8 average_score -29.5\n",
      "episode 1981 score -18.8 average_score -29.5\n",
      "episode 1982 score -58.8 average_score -30.0\n",
      "episode 1983 score -11.8 average_score -29.8\n",
      "episode 1984 score -34.0 average_score -30.0\n",
      "episode 1985 score -52.0 average_score -30.3\n",
      "episode 1986 score -21.8 average_score -29.9\n",
      "episode 1987 score -60.5 average_score -30.2\n",
      "episode 1988 score -30.5 average_score -30.5\n",
      "episode 1989 score -27.5 average_score -30.7\n",
      "episode 1990 score -8.2 average_score -30.8\n",
      "episode 1991 score 1.5 average_score -30.2\n",
      "episode 1992 score -16.2 average_score -30.0\n",
      "episode 1993 score 7.2 average_score -29.5\n",
      "episode 1994 score -31.8 average_score -29.6\n",
      "episode 1995 score -20.5 average_score -29.7\n",
      "episode 1996 score 2.8 average_score -29.4\n",
      "episode 1997 score -30.2 average_score -29.1\n",
      "episode 1998 score 5.8 average_score -28.8\n",
      "episode 1999 score -30.2 average_score -28.8\n",
      "episode 2000 score -35.8 average_score -28.8\n",
      "episode 2001 score 0.8 average_score -28.4\n",
      "episode 2002 score -43.0 average_score -28.3\n",
      "episode 2003 score -48.5 average_score -28.6\n",
      "episode 2004 score -42.0 average_score -28.6\n",
      "episode 2005 score -26.0 average_score -28.8\n",
      "episode 2006 score -51.8 average_score -29.2\n",
      "episode 2007 score -50.2 average_score -29.2\n",
      "episode 2008 score -7.0 average_score -29.1\n",
      "episode 2009 score -51.8 average_score -29.5\n",
      "episode 2010 score -16.2 average_score -29.5\n",
      "episode 2011 score -48.5 average_score -29.5\n",
      "episode 2012 score -23.0 average_score -29.5\n",
      "episode 2013 score -22.2 average_score -29.6\n",
      "episode 2014 score -30.5 average_score -29.5\n",
      "episode 2015 score -38.5 average_score -29.9\n",
      "episode 2016 score -33.8 average_score -29.7\n",
      "episode 2017 score -43.5 average_score -29.7\n",
      "episode 2018 score -58.5 average_score -29.8\n",
      "episode 2019 score -12.0 average_score -29.7\n",
      "episode 2020 score -42.5 average_score -29.7\n",
      "episode 2021 score -31.2 average_score -29.4\n",
      "episode 2022 score -27.5 average_score -29.5\n",
      "episode 2023 score -33.2 average_score -29.4\n",
      "episode 2024 score -36.0 average_score -29.5\n",
      "episode 2025 score -29.0 average_score -29.7\n",
      "episode 2026 score -20.8 average_score -29.9\n",
      "episode 2027 score -23.5 average_score -29.5\n",
      "episode 2028 score -2.0 average_score -29.1\n",
      "episode 2029 score 4.2 average_score -28.7\n",
      "episode 2030 score -31.8 average_score -29.0\n",
      "episode 2031 score 11.8 average_score -28.5\n",
      "episode 2032 score -56.8 average_score -28.8\n",
      "episode 2033 score -29.8 average_score -28.6\n",
      "episode 2034 score -44.5 average_score -28.8\n",
      "episode 2035 score -41.8 average_score -28.8\n",
      "episode 2036 score -12.5 average_score -28.7\n",
      "episode 2037 score -28.5 average_score -28.7\n",
      "episode 2038 score -40.8 average_score -28.7\n",
      "episode 2039 score -16.0 average_score -28.7\n",
      "episode 2040 score -16.8 average_score -28.6\n",
      "episode 2041 score -27.2 average_score -28.3\n",
      "episode 2042 score -23.8 average_score -28.3\n",
      "episode 2043 score -13.2 average_score -28.0\n",
      "episode 2044 score -21.0 average_score -27.8\n",
      "episode 2045 score -35.2 average_score -27.7\n",
      "episode 2046 score -12.8 average_score -27.5\n",
      "episode 2047 score -49.8 average_score -27.7\n",
      "episode 2048 score -30.2 average_score -27.2\n",
      "episode 2049 score -35.5 average_score -27.2\n",
      "episode 2050 score -21.5 average_score -27.1\n",
      "episode 2051 score -33.2 average_score -27.2\n",
      "episode 2052 score -35.8 average_score -27.3\n",
      "episode 2053 score -35.5 average_score -27.2\n",
      "episode 2054 score -51.5 average_score -27.3\n",
      "episode 2055 score -17.8 average_score -27.3\n",
      "episode 2056 score -28.0 average_score -27.1\n",
      "episode 2057 score -27.0 average_score -27.0\n",
      "episode 2058 score -20.0 average_score -26.9\n",
      "episode 2059 score 0.8 average_score -26.6\n",
      "episode 2060 score -11.2 average_score -26.3\n",
      "episode 2061 score -29.2 average_score -26.5\n",
      "episode 2062 score -29.0 average_score -26.3\n",
      "episode 2063 score -34.0 average_score -26.3\n",
      "episode 2064 score -36.2 average_score -26.4\n",
      "episode 2065 score -45.8 average_score -26.7\n",
      "episode 2066 score -64.2 average_score -27.1\n",
      "episode 2067 score -39.2 average_score -27.3\n",
      "episode 2068 score -37.5 average_score -27.4\n",
      "episode 2069 score -25.2 average_score -27.7\n",
      "episode 2070 score -57.2 average_score -28.0\n",
      "episode 2071 score -13.5 average_score -28.0\n",
      "episode 2072 score -36.2 average_score -28.3\n",
      "episode 2073 score -49.0 average_score -28.6\n",
      "episode 2074 score -28.2 average_score -29.1\n",
      "episode 2075 score -13.2 average_score -29.1\n",
      "episode 2076 score -24.2 average_score -28.9\n",
      "episode 2077 score -3.2 average_score -28.6\n",
      "episode 2078 score -51.2 average_score -29.0\n",
      "episode 2079 score -13.0 average_score -28.7\n",
      "episode 2080 score 5.0 average_score -28.2\n",
      "episode 2081 score -46.2 average_score -28.5\n",
      "episode 2082 score -21.8 average_score -28.1\n",
      "episode 2083 score -12.2 average_score -28.1\n",
      "episode 2084 score -20.0 average_score -28.0\n",
      "episode 2085 score -28.0 average_score -27.7\n",
      "episode 2086 score -7.0 average_score -27.6\n",
      "episode 2087 score -29.2 average_score -27.3\n",
      "episode 2088 score -26.0 average_score -27.2\n",
      "episode 2089 score -25.0 average_score -27.2\n",
      "episode 2090 score -29.8 average_score -27.4\n",
      "episode 2091 score -31.2 average_score -27.7\n",
      "episode 2092 score -12.0 average_score -27.7\n",
      "episode 2093 score -42.0 average_score -28.2\n",
      "episode 2094 score -38.5 average_score -28.3\n",
      "episode 2095 score -14.8 average_score -28.2\n",
      "episode 2096 score -28.2 average_score -28.5\n",
      "episode 2097 score -29.5 average_score -28.5\n",
      "episode 2098 score 5.0 average_score -28.5\n",
      "episode 2099 score -21.5 average_score -28.4\n",
      "episode 2100 score -29.5 average_score -28.4\n",
      "episode 2101 score -32.8 average_score -28.7\n",
      "episode 2102 score -30.0 average_score -28.6\n",
      "episode 2103 score -33.8 average_score -28.4\n",
      "episode 2104 score -24.2 average_score -28.2\n",
      "episode 2105 score -52.2 average_score -28.5\n",
      "episode 2106 score -8.8 average_score -28.1\n",
      "episode 2107 score -21.5 average_score -27.8\n",
      "episode 2108 score -43.5 average_score -28.2\n",
      "episode 2109 score -27.8 average_score -27.9\n",
      "episode 2110 score -35.2 average_score -28.1\n",
      "episode 2111 score -31.0 average_score -27.9\n",
      "episode 2112 score -21.8 average_score -27.9\n",
      "episode 2113 score -17.2 average_score -27.9\n",
      "episode 2114 score -45.0 average_score -28.0\n",
      "episode 2115 score -27.2 average_score -27.9\n",
      "episode 2116 score -24.2 average_score -27.8\n",
      "episode 2117 score -38.0 average_score -27.8\n",
      "episode 2118 score -48.0 average_score -27.6\n",
      "episode 2119 score -33.0 average_score -27.9\n",
      "episode 2120 score -38.5 average_score -27.8\n",
      "episode 2121 score -46.8 average_score -28.0\n",
      "episode 2122 score -46.8 average_score -28.2\n",
      "episode 2123 score -33.8 average_score -28.2\n",
      "episode 2124 score -8.0 average_score -27.9\n",
      "episode 2125 score -18.5 average_score -27.8\n",
      "episode 2126 score -31.8 average_score -27.9\n",
      "episode 2127 score -45.2 average_score -28.1\n",
      "episode 2128 score -32.8 average_score -28.4\n",
      "episode 2129 score -60.0 average_score -29.1\n",
      "episode 2130 score -37.8 average_score -29.1\n",
      "episode 2131 score -32.5 average_score -29.6\n",
      "episode 2132 score -21.2 average_score -29.2\n",
      "episode 2133 score -9.5 average_score -29.0\n",
      "episode 2134 score -42.2 average_score -29.0\n",
      "episode 2135 score -39.0 average_score -29.0\n",
      "episode 2136 score -6.0 average_score -28.9\n",
      "episode 2137 score -10.5 average_score -28.7\n",
      "episode 2138 score -39.2 average_score -28.7\n",
      "episode 2139 score -14.0 average_score -28.7\n",
      "episode 2140 score -33.5 average_score -28.8\n",
      "episode 2141 score -16.0 average_score -28.7\n",
      "episode 2142 score -15.2 average_score -28.6\n",
      "episode 2143 score -44.2 average_score -29.0\n",
      "episode 2144 score -67.2 average_score -29.4\n",
      "episode 2145 score -18.2 average_score -29.2\n",
      "episode 2146 score -33.5 average_score -29.5\n",
      "episode 2147 score -48.5 average_score -29.4\n",
      "episode 2148 score -40.2 average_score -29.5\n",
      "episode 2149 score -7.5 average_score -29.3\n",
      "episode 2150 score -56.8 average_score -29.6\n",
      "episode 2151 score -68.8 average_score -30.0\n",
      "episode 2152 score -26.2 average_score -29.9\n",
      "episode 2153 score -31.5 average_score -29.8\n",
      "episode 2154 score -31.0 average_score -29.6\n",
      "episode 2155 score -57.8 average_score -30.0\n",
      "episode 2156 score -77.0 average_score -30.5\n",
      "episode 2157 score -34.5 average_score -30.6\n",
      "episode 2158 score -55.2 average_score -30.9\n",
      "episode 2159 score -2.5 average_score -31.0\n",
      "episode 2160 score -36.0 average_score -31.2\n",
      "episode 2161 score -21.0 average_score -31.1\n",
      "episode 2162 score -14.5 average_score -31.0\n",
      "episode 2163 score 1.2 average_score -30.6\n",
      "episode 2164 score -31.5 average_score -30.6\n",
      "episode 2165 score -39.2 average_score -30.5\n",
      "episode 2166 score -36.8 average_score -30.3\n",
      "episode 2167 score -13.8 average_score -30.0\n",
      "episode 2168 score -35.2 average_score -30.0\n",
      "episode 2169 score -13.0 average_score -29.9\n",
      "episode 2170 score -47.2 average_score -29.8\n",
      "episode 2171 score -37.8 average_score -30.0\n",
      "episode 2172 score -34.0 average_score -30.0\n",
      "episode 2173 score -31.0 average_score -29.8\n",
      "episode 2174 score -13.2 average_score -29.7\n",
      "episode 2175 score -50.5 average_score -30.0\n",
      "episode 2176 score -23.0 average_score -30.0\n",
      "episode 2177 score -5.5 average_score -30.0\n",
      "episode 2178 score -8.0 average_score -29.6\n",
      "episode 2179 score -40.0 average_score -29.9\n",
      "episode 2180 score -47.0 average_score -30.4\n",
      "episode 2181 score -43.5 average_score -30.4\n",
      "episode 2182 score -55.8 average_score -30.7\n",
      "episode 2183 score -30.8 average_score -30.9\n",
      "episode 2184 score -21.8 average_score -30.9\n",
      "episode 2185 score -38.8 average_score -31.0\n",
      "episode 2186 score -26.5 average_score -31.2\n",
      "episode 2187 score -25.2 average_score -31.2\n",
      "episode 2188 score -50.5 average_score -31.4\n",
      "episode 2189 score -12.5 average_score -31.3\n",
      "episode 2190 score -37.8 average_score -31.4\n",
      "episode 2191 score -20.5 average_score -31.3\n",
      "episode 2192 score -23.0 average_score -31.4\n",
      "episode 2193 score -41.5 average_score -31.4\n",
      "episode 2194 score -52.5 average_score -31.5\n",
      "episode 2195 score -25.8 average_score -31.6\n",
      "episode 2196 score -41.2 average_score -31.7\n",
      "episode 2197 score -22.0 average_score -31.7\n",
      "episode 2198 score -30.2 average_score -32.0\n",
      "episode 2199 score -37.8 average_score -32.2\n",
      "episode 2200 score -31.8 average_score -32.2\n",
      "episode 2201 score -46.5 average_score -32.3\n",
      "episode 2202 score -34.2 average_score -32.4\n",
      "episode 2203 score -35.5 average_score -32.4\n",
      "episode 2204 score -20.8 average_score -32.4\n",
      "episode 2205 score -49.5 average_score -32.3\n",
      "episode 2206 score -46.5 average_score -32.7\n",
      "episode 2207 score -44.5 average_score -33.0\n",
      "episode 2208 score -25.5 average_score -32.8\n",
      "episode 2209 score -34.0 average_score -32.8\n",
      "episode 2210 score -9.2 average_score -32.6\n",
      "episode 2211 score -56.8 average_score -32.8\n",
      "episode 2212 score -70.8 average_score -33.3\n",
      "episode 2213 score -42.0 average_score -33.6\n",
      "episode 2214 score -16.0 average_score -33.3\n",
      "episode 2215 score -30.2 average_score -33.3\n",
      "episode 2216 score -13.5 average_score -33.2\n",
      "episode 2217 score 10.8 average_score -32.7\n",
      "episode 2218 score -44.2 average_score -32.7\n",
      "episode 2219 score -3.0 average_score -32.4\n",
      "episode 2220 score -32.5 average_score -32.3\n",
      "episode 2221 score -57.2 average_score -32.4\n",
      "episode 2222 score -25.0 average_score -32.2\n",
      "episode 2223 score -36.0 average_score -32.2\n",
      "episode 2224 score -53.5 average_score -32.7\n",
      "episode 2225 score -21.5 average_score -32.7\n",
      "episode 2226 score -27.2 average_score -32.7\n",
      "episode 2227 score -14.8 average_score -32.4\n",
      "episode 2228 score -62.8 average_score -32.7\n",
      "episode 2229 score -72.2 average_score -32.8\n",
      "episode 2230 score 1.2 average_score -32.4\n",
      "episode 2231 score -18.5 average_score -32.3\n",
      "episode 2232 score -55.0 average_score -32.6\n",
      "episode 2233 score -18.8 average_score -32.7\n",
      "episode 2234 score -8.0 average_score -32.3\n",
      "episode 2235 score -20.2 average_score -32.2\n",
      "episode 2236 score -35.2 average_score -32.4\n",
      "episode 2237 score -46.8 average_score -32.8\n",
      "episode 2238 score -14.8 average_score -32.6\n",
      "episode 2239 score -35.8 average_score -32.8\n",
      "episode 2240 score -1.2 average_score -32.5\n",
      "episode 2241 score -46.2 average_score -32.8\n",
      "episode 2242 score -32.2 average_score -32.9\n",
      "episode 2243 score -58.2 average_score -33.1\n",
      "episode 2244 score -38.0 average_score -32.8\n",
      "episode 2245 score -25.5 average_score -32.9\n",
      "episode 2246 score -11.8 average_score -32.6\n",
      "episode 2247 score -42.0 average_score -32.6\n",
      "episode 2248 score -21.8 average_score -32.4\n",
      "episode 2249 score -21.0 average_score -32.5\n",
      "episode 2250 score -66.5 average_score -32.6\n",
      "episode 2251 score -35.5 average_score -32.3\n",
      "episode 2252 score -50.5 average_score -32.5\n",
      "episode 2253 score -37.2 average_score -32.6\n",
      "episode 2254 score -19.2 average_score -32.5\n",
      "episode 2255 score -82.2 average_score -32.7\n",
      "episode 2256 score -14.2 average_score -32.1\n",
      "episode 2257 score -33.5 average_score -32.1\n",
      "episode 2258 score -7.5 average_score -31.6\n",
      "episode 2259 score -37.0 average_score -31.9\n",
      "episode 2260 score -30.2 average_score -31.9\n",
      "episode 2261 score -11.0 average_score -31.8\n",
      "episode 2262 score -22.2 average_score -31.9\n",
      "episode 2263 score -34.0 average_score -32.2\n",
      "episode 2264 score -28.8 average_score -32.2\n",
      "episode 2265 score -13.0 average_score -31.9\n",
      "episode 2266 score -12.8 average_score -31.7\n",
      "episode 2267 score -26.0 average_score -31.8\n",
      "episode 2268 score -29.0 average_score -31.7\n",
      "episode 2269 score -20.8 average_score -31.8\n",
      "episode 2270 score -27.5 average_score -31.6\n",
      "episode 2271 score -30.5 average_score -31.6\n",
      "episode 2272 score -21.5 average_score -31.4\n",
      "episode 2273 score -4.5 average_score -31.2\n",
      "episode 2274 score -30.8 average_score -31.3\n",
      "episode 2275 score -38.8 average_score -31.2\n",
      "episode 2276 score -32.0 average_score -31.3\n",
      "episode 2277 score -52.8 average_score -31.8\n",
      "episode 2278 score -24.2 average_score -31.9\n",
      "episode 2279 score -31.0 average_score -31.9\n",
      "episode 2280 score -7.5 average_score -31.5\n",
      "episode 2281 score -23.0 average_score -31.3\n",
      "episode 2282 score -51.5 average_score -31.2\n",
      "episode 2283 score -22.8 average_score -31.1\n",
      "episode 2284 score -35.8 average_score -31.3\n",
      "episode 2285 score -11.8 average_score -31.0\n",
      "episode 2286 score -41.0 average_score -31.1\n",
      "episode 2287 score -50.5 average_score -31.4\n",
      "episode 2288 score -21.8 average_score -31.1\n",
      "episode 2289 score -36.2 average_score -31.4\n",
      "episode 2290 score -45.0 average_score -31.4\n",
      "episode 2291 score -47.5 average_score -31.7\n",
      "episode 2292 score -43.8 average_score -31.9\n",
      "episode 2293 score -6.0 average_score -31.5\n",
      "episode 2294 score -0.5 average_score -31.0\n",
      "episode 2295 score -40.2 average_score -31.2\n",
      "episode 2296 score -36.8 average_score -31.1\n",
      "episode 2297 score -47.5 average_score -31.4\n",
      "episode 2298 score -29.8 average_score -31.4\n",
      "episode 2299 score -22.0 average_score -31.2\n",
      "episode 2300 score -35.8 average_score -31.3\n",
      "episode 2301 score -41.2 average_score -31.2\n",
      "episode 2302 score -28.8 average_score -31.1\n",
      "episode 2303 score -7.2 average_score -30.9\n",
      "episode 2304 score -18.2 average_score -30.8\n",
      "episode 2305 score -31.5 average_score -30.7\n",
      "episode 2306 score -19.2 average_score -30.4\n",
      "episode 2307 score -47.0 average_score -30.4\n",
      "episode 2308 score -15.5 average_score -30.3\n",
      "episode 2309 score -18.8 average_score -30.2\n",
      "episode 2310 score -29.0 average_score -30.4\n",
      "episode 2311 score -28.5 average_score -30.1\n",
      "episode 2312 score -38.0 average_score -29.8\n",
      "episode 2313 score -34.0 average_score -29.7\n",
      "episode 2314 score -40.8 average_score -29.9\n",
      "episode 2315 score -37.0 average_score -30.0\n",
      "episode 2316 score -38.2 average_score -30.2\n",
      "episode 2317 score -40.0 average_score -30.7\n",
      "episode 2318 score -38.2 average_score -30.7\n",
      "episode 2319 score -40.2 average_score -31.1\n",
      "episode 2320 score -44.8 average_score -31.2\n",
      "episode 2321 score -31.0 average_score -30.9\n",
      "episode 2322 score -56.0 average_score -31.2\n",
      "episode 2323 score -4.0 average_score -30.9\n",
      "episode 2324 score -28.0 average_score -30.6\n",
      "episode 2325 score -12.2 average_score -30.6\n",
      "episode 2326 score -14.0 average_score -30.4\n",
      "episode 2327 score -43.0 average_score -30.7\n",
      "episode 2328 score -12.2 average_score -30.2\n",
      "episode 2329 score -49.0 average_score -30.0\n",
      "episode 2330 score -33.0 average_score -30.3\n",
      "episode 2331 score -12.0 average_score -30.2\n",
      "episode 2332 score -17.0 average_score -29.9\n",
      "episode 2333 score -18.5 average_score -29.9\n",
      "episode 2334 score -10.5 average_score -29.9\n",
      "episode 2335 score -60.5 average_score -30.3\n",
      "episode 2336 score -42.2 average_score -30.4\n",
      "episode 2337 score -48.0 average_score -30.4\n",
      "episode 2338 score -30.2 average_score -30.5\n",
      "episode 2339 score -52.8 average_score -30.7\n",
      "episode 2340 score -48.5 average_score -31.2\n",
      "episode 2341 score -18.0 average_score -30.9\n",
      "episode 2342 score -38.5 average_score -30.9\n",
      "episode 2343 score -23.0 average_score -30.6\n",
      "episode 2344 score -34.0 average_score -30.6\n",
      "episode 2345 score -29.5 average_score -30.6\n",
      "episode 2346 score -25.5 average_score -30.7\n",
      "episode 2347 score -35.8 average_score -30.7\n",
      "episode 2348 score -18.8 average_score -30.6\n",
      "episode 2349 score -48.0 average_score -30.9\n",
      "episode 2350 score 0.2 average_score -30.2\n",
      "episode 2351 score -29.5 average_score -30.2\n",
      "episode 2352 score -6.8 average_score -29.7\n",
      "episode 2353 score -45.0 average_score -29.8\n",
      "episode 2354 score -24.2 average_score -29.9\n",
      "episode 2355 score -71.2 average_score -29.8\n",
      "episode 2356 score -36.0 average_score -30.0\n",
      "episode 2357 score -6.5 average_score -29.7\n",
      "episode 2358 score -57.0 average_score -30.2\n",
      "episode 2359 score -26.8 average_score -30.1\n",
      "episode 2360 score -27.0 average_score -30.1\n",
      "episode 2361 score -24.0 average_score -30.2\n",
      "episode 2362 score -28.2 average_score -30.3\n",
      "episode 2363 score -20.8 average_score -30.1\n",
      "episode 2364 score -10.5 average_score -29.9\n",
      "episode 2365 score -38.8 average_score -30.2\n",
      "episode 2366 score 13.2 average_score -29.9\n",
      "episode 2367 score -42.0 average_score -30.1\n",
      "episode 2368 score -49.0 average_score -30.3\n",
      "episode 2369 score -58.2 average_score -30.7\n",
      "episode 2370 score -28.8 average_score -30.7\n",
      "episode 2371 score -30.8 average_score -30.7\n",
      "episode 2372 score -18.0 average_score -30.7\n",
      "episode 2373 score -60.5 average_score -31.2\n",
      "episode 2374 score 3.2 average_score -30.9\n",
      "episode 2375 score -12.2 average_score -30.6\n",
      "episode 2376 score -32.5 average_score -30.6\n",
      "episode 2377 score -5.8 average_score -30.1\n",
      "episode 2378 score -37.2 average_score -30.3\n",
      "episode 2379 score -14.2 average_score -30.1\n",
      "episode 2380 score -6.2 average_score -30.1\n",
      "episode 2381 score -34.2 average_score -30.2\n",
      "episode 2382 score -31.5 average_score -30.0\n",
      "episode 2383 score -25.8 average_score -30.0\n",
      "episode 2384 score -48.8 average_score -30.2\n",
      "episode 2385 score -32.2 average_score -30.4\n",
      "episode 2386 score -46.8 average_score -30.4\n",
      "episode 2387 score -24.5 average_score -30.2\n",
      "episode 2388 score -44.8 average_score -30.4\n",
      "episode 2389 score -27.0 average_score -30.3\n",
      "episode 2390 score -48.8 average_score -30.4\n",
      "episode 2391 score -16.8 average_score -30.0\n",
      "episode 2392 score -33.2 average_score -29.9\n",
      "episode 2393 score -29.2 average_score -30.2\n",
      "episode 2394 score -14.8 average_score -30.3\n",
      "episode 2395 score -32.8 average_score -30.2\n",
      "episode 2396 score -19.5 average_score -30.1\n",
      "episode 2397 score -42.0 average_score -30.0\n",
      "episode 2398 score -41.5 average_score -30.1\n",
      "episode 2399 score -18.8 average_score -30.1\n",
      "episode 2400 score -18.2 average_score -29.9\n",
      "episode 2401 score -33.5 average_score -29.8\n",
      "episode 2402 score -52.0 average_score -30.1\n",
      "episode 2403 score -10.8 average_score -30.1\n",
      "episode 2404 score -28.8 average_score -30.2\n",
      "episode 2405 score -34.5 average_score -30.2\n",
      "episode 2406 score -0.8 average_score -30.1\n",
      "episode 2407 score -26.5 average_score -29.9\n",
      "episode 2408 score -43.8 average_score -30.1\n",
      "episode 2409 score -66.8 average_score -30.6\n",
      "episode 2410 score -19.5 average_score -30.5\n",
      "episode 2411 score -35.8 average_score -30.6\n",
      "episode 2412 score -4.8 average_score -30.3\n",
      "episode 2413 score -31.0 average_score -30.2\n",
      "episode 2414 score -12.2 average_score -29.9\n",
      "episode 2415 score -18.2 average_score -29.8\n",
      "episode 2416 score -38.5 average_score -29.8\n",
      "episode 2417 score -54.8 average_score -29.9\n",
      "episode 2418 score -16.5 average_score -29.7\n",
      "episode 2419 score -54.2 average_score -29.8\n",
      "episode 2420 score -27.8 average_score -29.7\n",
      "episode 2421 score -42.5 average_score -29.8\n",
      "episode 2422 score -38.2 average_score -29.6\n",
      "episode 2423 score -27.8 average_score -29.8\n",
      "episode 2424 score -54.5 average_score -30.1\n",
      "episode 2425 score -52.2 average_score -30.5\n",
      "episode 2426 score -4.0 average_score -30.4\n",
      "episode 2427 score -50.0 average_score -30.5\n",
      "episode 2428 score -19.5 average_score -30.5\n",
      "episode 2429 score -54.8 average_score -30.6\n",
      "episode 2430 score -12.2 average_score -30.4\n",
      "episode 2431 score -49.5 average_score -30.8\n",
      "episode 2432 score -26.8 average_score -30.9\n",
      "episode 2433 score -31.2 average_score -31.0\n",
      "episode 2434 score -11.2 average_score -31.0\n",
      "episode 2435 score -30.5 average_score -30.7\n",
      "episode 2436 score -22.0 average_score -30.5\n",
      "episode 2437 score -31.2 average_score -30.3\n",
      "episode 2438 score -36.5 average_score -30.4\n",
      "episode 2439 score -26.8 average_score -30.1\n",
      "episode 2440 score -17.2 average_score -29.8\n",
      "episode 2441 score -25.8 average_score -29.9\n",
      "episode 2442 score -26.8 average_score -29.8\n",
      "episode 2443 score -24.0 average_score -29.8\n",
      "episode 2444 score -37.2 average_score -29.8\n",
      "episode 2445 score 0.0 average_score -29.5\n",
      "episode 2446 score -37.8 average_score -29.7\n",
      "episode 2447 score 14.2 average_score -29.2\n",
      "episode 2448 score -81.8 average_score -29.8\n",
      "episode 2449 score -27.2 average_score -29.6\n",
      "episode 2450 score -44.8 average_score -30.0\n",
      "episode 2451 score -57.8 average_score -30.3\n",
      "episode 2452 score -38.5 average_score -30.6\n",
      "episode 2453 score -21.2 average_score -30.4\n",
      "episode 2454 score -34.5 average_score -30.5\n",
      "episode 2455 score -50.0 average_score -30.3\n",
      "episode 2456 score -33.8 average_score -30.3\n",
      "episode 2457 score -32.2 average_score -30.5\n",
      "episode 2458 score -30.2 average_score -30.2\n",
      "episode 2459 score -22.0 average_score -30.2\n",
      "episode 2460 score -24.5 average_score -30.2\n",
      "episode 2461 score 3.0 average_score -29.9\n",
      "episode 2462 score -40.0 average_score -30.0\n",
      "episode 2463 score -16.8 average_score -30.0\n",
      "episode 2464 score -25.2 average_score -30.1\n",
      "episode 2465 score -42.8 average_score -30.2\n",
      "episode 2466 score -36.0 average_score -30.7\n",
      "episode 2467 score -42.0 average_score -30.7\n",
      "episode 2468 score -18.2 average_score -30.4\n",
      "episode 2469 score -29.2 average_score -30.1\n",
      "episode 2470 score -19.5 average_score -30.0\n",
      "episode 2471 score -43.8 average_score -30.1\n",
      "episode 2472 score -55.5 average_score -30.5\n",
      "episode 2473 score -4.8 average_score -29.9\n",
      "episode 2474 score -19.5 average_score -30.1\n",
      "episode 2475 score -66.5 average_score -30.7\n",
      "episode 2476 score -13.8 average_score -30.5\n",
      "episode 2477 score -37.2 average_score -30.8\n",
      "episode 2478 score -47.0 average_score -30.9\n",
      "episode 2479 score -35.2 average_score -31.1\n",
      "episode 2480 score -23.0 average_score -31.3\n",
      "episode 2481 score -8.5 average_score -31.0\n",
      "episode 2482 score -19.2 average_score -30.9\n",
      "episode 2483 score -31.5 average_score -31.0\n",
      "episode 2484 score -48.8 average_score -31.0\n",
      "episode 2485 score -16.0 average_score -30.8\n",
      "episode 2486 score -30.2 average_score -30.6\n",
      "episode 2487 score -23.5 average_score -30.6\n",
      "episode 2488 score -27.5 average_score -30.5\n",
      "episode 2489 score 15.2 average_score -30.0\n",
      "episode 2490 score -25.2 average_score -29.8\n",
      "episode 2491 score -8.0 average_score -29.7\n",
      "episode 2492 score -16.0 average_score -29.5\n",
      "episode 2493 score -58.2 average_score -29.8\n",
      "episode 2494 score -16.5 average_score -29.8\n",
      "episode 2495 score -2.5 average_score -29.5\n",
      "episode 2496 score -25.0 average_score -29.6\n",
      "episode 2497 score -19.2 average_score -29.4\n",
      "episode 2498 score -18.2 average_score -29.1\n",
      "episode 2499 score -30.0 average_score -29.3\n"
     ]
    }
   ],
   "source": [
    "agent = Agent(ALPHA=0.001, GAMMA=0.99, n_actions=5,\n",
    "                  layer1_size=32, layer2_size=16,\n",
    "                  layer3_size=8, fname='model.h5')\n",
    "env = ontime_dataset_env(df)\n",
    "score_history = []\n",
    "\n",
    "n_episodes = 2500\n",
    "\n",
    "for i in range(n_episodes):\n",
    "    done = False\n",
    "    score = 0\n",
    "    observation = env.reset()\n",
    "    while not done:\n",
    "        action = agent.choose_action(observation)\n",
    "        observation_, reward, done, info = env.step(action)\n",
    "        agent.store_transition(observation, action, reward)\n",
    "        observation = observation_\n",
    "        score += reward\n",
    "    score_history.append(score)\n",
    "    agent.learn()\n",
    "    print('episode', i, 'score %.1f' % score, 'average_score %.1f' %\n",
    "         np.mean(score_history[-100:]))\n",
    "\n",
    "agent.save_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Trying the same agent on a gym environment\n",
    "Below is the same model defined above, just edited to take the new input shapes from Gym's LunarLander-v2 environment. the model was able to be trained on this environment and was able to get positive rewards."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from gym_model.Agent import GymAgent\n",
    "from gym_model.gym_env import run_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mostafakm/Documents/School/Practical Data Science/Assignment 7/gym_model/Agent.py:41: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=[<tf.Tenso..., outputs=[<tf.Tenso...)`\n",
      "  policy = Model(input=[input, advantages], output=[probs])\n",
      "/home/mostafakm/Documents/School/Practical Data Science/Assignment 7/gym_model/Agent.py:43: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=[<tf.Tenso..., outputs=[<tf.Tenso...)`\n",
      "  predict = Model(input=[input], output=[probs])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 0 score -321.2 average_score -321.2\n",
      "episode 1 score -111.1 average_score -216.1\n",
      "episode 2 score -157.6 average_score -196.6\n",
      "episode 3 score -222.1 average_score -203.0\n",
      "episode 4 score -142.3 average_score -190.8\n",
      "episode 5 score -291.4 average_score -207.6\n",
      "episode 6 score -25.4 average_score -181.6\n",
      "episode 7 score -28.4 average_score -162.4\n",
      "episode 8 score -261.0 average_score -173.4\n",
      "episode 9 score -103.2 average_score -166.4\n",
      "episode 10 score -10.5 average_score -152.2\n",
      "episode 11 score -153.5 average_score -152.3\n",
      "episode 12 score -287.2 average_score -162.7\n",
      "episode 13 score -168.1 average_score -163.1\n",
      "episode 14 score -300.6 average_score -172.2\n",
      "episode 15 score -225.9 average_score -175.6\n",
      "episode 16 score -133.4 average_score -173.1\n",
      "episode 17 score -141.5 average_score -171.4\n",
      "episode 18 score -143.3 average_score -169.9\n",
      "episode 19 score -109.8 average_score -166.9\n",
      "episode 20 score -149.8 average_score -166.1\n",
      "episode 21 score -106.6 average_score -163.4\n",
      "episode 22 score -72.3 average_score -159.4\n",
      "episode 23 score -170.1 average_score -159.8\n",
      "episode 24 score -161.9 average_score -159.9\n",
      "episode 25 score -295.5 average_score -165.1\n",
      "episode 26 score -199.0 average_score -166.4\n",
      "episode 27 score -181.8 average_score -166.9\n",
      "episode 28 score -178.7 average_score -167.3\n",
      "episode 29 score -46.8 average_score -163.3\n",
      "episode 30 score 11.9 average_score -157.7\n",
      "episode 31 score -134.3 average_score -156.9\n",
      "episode 32 score -90.4 average_score -154.9\n",
      "episode 33 score -312.2 average_score -159.6\n",
      "episode 34 score -297.6 average_score -163.5\n",
      "episode 35 score 38.0 average_score -157.9\n",
      "episode 36 score -371.6 average_score -163.7\n",
      "episode 37 score -127.7 average_score -162.7\n",
      "episode 38 score -74.7 average_score -160.5\n",
      "episode 39 score -170.0 average_score -160.7\n",
      "episode 40 score -290.9 average_score -163.9\n",
      "episode 41 score -69.9 average_score -161.6\n",
      "episode 42 score -153.5 average_score -161.5\n",
      "episode 43 score -142.8 average_score -161.0\n",
      "episode 44 score -107.7 average_score -159.8\n",
      "episode 45 score -131.0 average_score -159.2\n",
      "episode 46 score -73.7 average_score -157.4\n",
      "episode 47 score -80.4 average_score -155.8\n",
      "episode 48 score -222.6 average_score -157.2\n",
      "episode 49 score -233.2 average_score -158.7\n",
      "episode 50 score -244.0 average_score -160.4\n",
      "episode 51 score -337.2 average_score -163.8\n",
      "episode 52 score -74.1 average_score -162.1\n",
      "episode 53 score -156.8 average_score -162.0\n",
      "episode 54 score -91.8 average_score -160.7\n",
      "episode 55 score -76.0 average_score -159.2\n",
      "episode 56 score -85.7 average_score -157.9\n",
      "episode 57 score -201.6 average_score -158.6\n",
      "episode 58 score -61.3 average_score -157.0\n",
      "episode 59 score -174.3 average_score -157.3\n",
      "episode 60 score -85.3 average_score -156.1\n",
      "episode 61 score -160.6 average_score -156.2\n",
      "episode 62 score -152.4 average_score -156.1\n",
      "episode 63 score -38.4 average_score -154.3\n",
      "episode 64 score -119.7 average_score -153.7\n",
      "episode 65 score -92.5 average_score -152.8\n",
      "episode 66 score -83.9 average_score -151.8\n",
      "episode 67 score -2.7 average_score -149.6\n",
      "episode 68 score -391.1 average_score -153.1\n",
      "episode 69 score -172.2 average_score -153.4\n",
      "episode 70 score -218.5 average_score -154.3\n",
      "episode 71 score -161.7 average_score -154.4\n",
      "episode 72 score -94.3 average_score -153.6\n",
      "episode 73 score -167.3 average_score -153.7\n",
      "episode 74 score -114.7 average_score -153.2\n",
      "episode 75 score -366.0 average_score -156.0\n",
      "episode 76 score -96.4 average_score -155.3\n",
      "episode 77 score -150.1 average_score -155.2\n",
      "episode 78 score -262.2 average_score -156.5\n",
      "episode 79 score -67.9 average_score -155.4\n",
      "episode 80 score -167.4 average_score -155.6\n",
      "episode 81 score -57.5 average_score -154.4\n",
      "episode 82 score -69.0 average_score -153.4\n",
      "episode 83 score -75.3 average_score -152.4\n",
      "episode 84 score -120.8 average_score -152.1\n",
      "episode 85 score -104.5 average_score -151.5\n",
      "episode 86 score -137.8 average_score -151.3\n",
      "episode 87 score -155.0 average_score -151.4\n",
      "episode 88 score -202.1 average_score -152.0\n",
      "episode 89 score -256.4 average_score -153.1\n",
      "episode 90 score -102.3 average_score -152.6\n",
      "episode 91 score -71.0 average_score -151.7\n",
      "episode 92 score -150.1 average_score -151.7\n",
      "episode 93 score -311.7 average_score -153.4\n",
      "episode 94 score -225.9 average_score -154.1\n",
      "episode 95 score -123.4 average_score -153.8\n",
      "episode 96 score -102.0 average_score -153.3\n",
      "episode 97 score -142.9 average_score -153.2\n",
      "episode 98 score -42.9 average_score -152.0\n",
      "episode 99 score -113.6 average_score -151.7\n",
      "episode 100 score -126.1 average_score -149.7\n",
      "episode 101 score -132.5 average_score -149.9\n",
      "episode 102 score -203.0 average_score -150.4\n",
      "episode 103 score -125.6 average_score -149.4\n",
      "episode 104 score -186.2 average_score -149.9\n",
      "episode 105 score -134.3 average_score -148.3\n",
      "episode 106 score -43.3 average_score -148.5\n",
      "episode 107 score -286.3 average_score -151.0\n",
      "episode 108 score -87.8 average_score -149.3\n",
      "episode 109 score -74.4 average_score -149.0\n",
      "episode 110 score -104.8 average_score -150.0\n",
      "episode 111 score -118.9 average_score -149.6\n",
      "episode 112 score -106.3 average_score -147.8\n",
      "episode 113 score -217.9 average_score -148.3\n",
      "episode 114 score -119.6 average_score -146.5\n",
      "episode 115 score -249.3 average_score -146.7\n",
      "episode 116 score -117.6 average_score -146.6\n",
      "episode 117 score -112.2 average_score -146.3\n",
      "episode 118 score -111.5 average_score -146.0\n",
      "episode 119 score -85.9 average_score -145.7\n",
      "episode 120 score -140.0 average_score -145.6\n",
      "episode 121 score -73.6 average_score -145.3\n",
      "episode 122 score -292.5 average_score -147.5\n",
      "episode 123 score -170.7 average_score -147.5\n",
      "episode 124 score -24.1 average_score -146.1\n",
      "episode 125 score 39.5 average_score -142.8\n",
      "episode 126 score -87.8 average_score -141.7\n",
      "episode 127 score -41.5 average_score -140.3\n",
      "episode 128 score -87.3 average_score -139.3\n",
      "episode 129 score -49.0 average_score -139.4\n",
      "episode 130 score -58.1 average_score -140.1\n",
      "episode 131 score -221.7 average_score -140.9\n",
      "episode 132 score -210.5 average_score -142.1\n",
      "episode 133 score -147.7 average_score -140.5\n",
      "episode 134 score -62.3 average_score -138.1\n",
      "episode 135 score -274.9 average_score -141.3\n",
      "episode 136 score -92.0 average_score -138.5\n",
      "episode 137 score -114.1 average_score -138.3\n",
      "episode 138 score -243.5 average_score -140.0\n",
      "episode 139 score -127.2 average_score -139.6\n",
      "episode 140 score -56.1 average_score -137.3\n",
      "episode 141 score -141.1 average_score -138.0\n",
      "episode 142 score -112.4 average_score -137.6\n",
      "episode 143 score -176.2 average_score -137.9\n",
      "episode 144 score -75.5 average_score -137.6\n",
      "episode 145 score -122.7 average_score -137.5\n",
      "episode 146 score -316.5 average_score -139.9\n",
      "episode 147 score -82.4 average_score -139.9\n",
      "episode 148 score -83.8 average_score -138.5\n",
      "episode 149 score -60.6 average_score -136.8\n",
      "episode 150 score -125.3 average_score -135.6\n",
      "episode 151 score -121.9 average_score -133.5\n",
      "episode 152 score -64.4 average_score -133.4\n",
      "episode 153 score -87.8 average_score -132.7\n",
      "episode 154 score 18.9 average_score -131.6\n",
      "episode 155 score -117.3 average_score -132.0\n",
      "episode 156 score -116.0 average_score -132.3\n",
      "episode 157 score -161.2 average_score -131.9\n",
      "episode 158 score -385.4 average_score -135.1\n",
      "episode 159 score -44.6 average_score -133.8\n",
      "episode 160 score -206.1 average_score -135.1\n",
      "episode 161 score -145.0 average_score -134.9\n",
      "episode 162 score -34.6 average_score -133.7\n",
      "episode 163 score -78.2 average_score -134.1\n",
      "episode 164 score -186.9 average_score -134.8\n",
      "episode 165 score -74.5 average_score -134.6\n",
      "episode 166 score -97.0 average_score -134.7\n",
      "episode 167 score -81.5 average_score -135.5\n",
      "episode 168 score -127.4 average_score -132.9\n",
      "episode 169 score -177.9 average_score -132.9\n",
      "episode 170 score -110.6 average_score -131.9\n",
      "episode 171 score -78.8 average_score -131.0\n",
      "episode 172 score -104.5 average_score -131.1\n",
      "episode 173 score -364.0 average_score -133.1\n",
      "episode 174 score -139.2 average_score -133.3\n",
      "episode 175 score -117.6 average_score -130.9\n",
      "episode 176 score -484.6 average_score -134.7\n",
      "episode 177 score -80.4 average_score -134.0\n",
      "episode 178 score -84.2 average_score -132.3\n",
      "episode 179 score -63.7 average_score -132.2\n",
      "episode 180 score -189.7 average_score -132.5\n",
      "episode 181 score -216.6 average_score -134.0\n",
      "episode 182 score -145.7 average_score -134.8\n",
      "episode 183 score -172.7 average_score -135.8\n",
      "episode 184 score -221.7 average_score -136.8\n",
      "episode 185 score -75.1 average_score -136.5\n",
      "episode 186 score -61.5 average_score -135.7\n",
      "episode 187 score -17.8 average_score -134.4\n",
      "episode 188 score -181.3 average_score -134.2\n",
      "episode 189 score -97.3 average_score -132.6\n",
      "episode 190 score -293.7 average_score -134.5\n",
      "episode 191 score -65.9 average_score -134.4\n",
      "episode 192 score -183.1 average_score -134.8\n",
      "episode 193 score -78.7 average_score -132.4\n",
      "episode 194 score -346.4 average_score -133.6\n",
      "episode 195 score -143.5 average_score -133.8\n",
      "episode 196 score -60.9 average_score -133.4\n",
      "episode 197 score -110.3 average_score -133.1\n",
      "episode 198 score -115.0 average_score -133.8\n",
      "episode 199 score -326.4 average_score -135.9\n",
      "episode 200 score -268.0 average_score -137.4\n",
      "episode 201 score -73.4 average_score -136.8\n",
      "episode 202 score -175.9 average_score -136.5\n",
      "episode 203 score -129.7 average_score -136.5\n",
      "episode 204 score -414.1 average_score -138.8\n",
      "episode 205 score -54.1 average_score -138.0\n",
      "episode 206 score -178.8 average_score -139.4\n",
      "episode 207 score -31.8 average_score -136.8\n",
      "episode 208 score -128.9 average_score -137.2\n",
      "episode 209 score -349.9 average_score -140.0\n",
      "episode 210 score -104.0 average_score -140.0\n",
      "episode 211 score -90.9 average_score -139.7\n",
      "episode 212 score -175.8 average_score -140.4\n",
      "episode 213 score -89.4 average_score -139.1\n",
      "episode 214 score -120.8 average_score -139.1\n",
      "episode 215 score -54.2 average_score -137.2\n",
      "episode 216 score -18.9 average_score -136.2\n",
      "episode 217 score -41.4 average_score -135.5\n",
      "episode 218 score -124.2 average_score -135.6\n",
      "episode 219 score -54.9 average_score -135.3\n",
      "episode 220 score -63.8 average_score -134.5\n",
      "episode 221 score -138.7 average_score -135.2\n",
      "episode 222 score -85.9 average_score -133.1\n",
      "episode 223 score -165.6 average_score -133.1\n",
      "episode 224 score -115.1 average_score -134.0\n",
      "episode 225 score -104.0 average_score -135.4\n",
      "episode 226 score -72.9 average_score -135.3\n",
      "episode 227 score -57.9 average_score -135.4\n",
      "episode 228 score -320.6 average_score -137.8\n",
      "episode 229 score -296.0 average_score -140.2\n",
      "episode 230 score -161.0 average_score -141.3\n",
      "episode 231 score -19.3 average_score -139.2\n",
      "episode 232 score -212.1 average_score -139.3\n",
      "episode 233 score -72.1 average_score -138.5\n",
      "episode 234 score -74.5 average_score -138.6\n",
      "episode 235 score -51.6 average_score -136.4\n",
      "episode 236 score -258.2 average_score -138.1\n",
      "episode 237 score -264.3 average_score -139.6\n",
      "episode 238 score -100.2 average_score -138.1\n",
      "episode 239 score -78.2 average_score -137.6\n",
      "episode 240 score -143.5 average_score -138.5\n",
      "episode 241 score -52.2 average_score -137.6\n",
      "episode 242 score -105.8 average_score -137.6\n",
      "episode 243 score -83.1 average_score -136.6\n",
      "episode 244 score -9.9 average_score -136.0\n",
      "episode 245 score -78.1 average_score -135.5\n",
      "episode 246 score -108.2 average_score -133.4\n",
      "episode 247 score -44.0 average_score -133.1\n",
      "episode 248 score -146.9 average_score -133.7\n",
      "episode 249 score -103.0 average_score -134.1\n",
      "episode 250 score -86.5 average_score -133.7\n",
      "episode 251 score -71.7 average_score -133.2\n",
      "episode 252 score -142.3 average_score -134.0\n",
      "episode 253 score -164.5 average_score -134.8\n",
      "episode 254 score -173.3 average_score -136.7\n",
      "episode 255 score -88.1 average_score -136.4\n",
      "episode 256 score -145.5 average_score -136.7\n",
      "episode 257 score -32.6 average_score -135.4\n",
      "episode 258 score -308.6 average_score -134.6\n",
      "episode 259 score -148.9 average_score -135.7\n",
      "episode 260 score -73.2 average_score -134.4\n",
      "episode 261 score -266.9 average_score -135.6\n",
      "episode 262 score -261.3 average_score -137.8\n",
      "episode 263 score -230.9 average_score -139.4\n",
      "episode 264 score -95.0 average_score -138.4\n",
      "episode 265 score -158.0 average_score -139.3\n",
      "episode 266 score -330.4 average_score -141.6\n",
      "episode 267 score -103.5 average_score -141.8\n",
      "episode 268 score -91.8 average_score -141.5\n",
      "episode 269 score -299.5 average_score -142.7\n",
      "episode 270 score -68.9 average_score -142.3\n",
      "episode 271 score -103.4 average_score -142.5\n",
      "episode 272 score -119.2 average_score -142.7\n",
      "episode 273 score -95.2 average_score -140.0\n",
      "episode 274 score -48.0 average_score -139.1\n",
      "episode 275 score -325.9 average_score -141.2\n",
      "episode 276 score -43.5 average_score -136.7\n",
      "episode 277 score -355.9 average_score -139.5\n",
      "episode 278 score -96.4 average_score -139.6\n",
      "episode 279 score -75.9 average_score -139.7\n",
      "episode 280 score -204.4 average_score -139.9\n",
      "episode 281 score -163.9 average_score -139.4\n",
      "episode 282 score -77.1 average_score -138.7\n",
      "episode 283 score -76.5 average_score -137.7\n",
      "episode 284 score -381.1 average_score -139.3\n",
      "episode 285 score 14.9 average_score -138.4\n",
      "episode 286 score -80.4 average_score -138.6\n",
      "episode 287 score -215.8 average_score -140.6\n",
      "episode 288 score -4.5 average_score -138.8\n",
      "episode 289 score -105.9 average_score -138.9\n",
      "episode 290 score -12.7 average_score -136.1\n",
      "episode 291 score -120.7 average_score -136.6\n",
      "episode 292 score -221.4 average_score -137.0\n",
      "episode 293 score -106.3 average_score -137.3\n",
      "episode 294 score -253.0 average_score -136.4\n",
      "episode 295 score -41.0 average_score -135.3\n",
      "episode 296 score -95.9 average_score -135.7\n",
      "episode 297 score -60.8 average_score -135.2\n",
      "episode 298 score -54.4 average_score -134.6\n",
      "episode 299 score -391.2 average_score -135.2\n",
      "episode 300 score -117.2 average_score -133.7\n",
      "episode 301 score -170.5 average_score -134.7\n",
      "episode 302 score -44.5 average_score -133.4\n",
      "episode 303 score -33.0 average_score -132.4\n",
      "episode 304 score -29.9 average_score -128.6\n",
      "episode 305 score -140.7 average_score -129.4\n",
      "episode 306 score -142.4 average_score -129.1\n",
      "episode 307 score -103.8 average_score -129.8\n",
      "episode 308 score -207.3 average_score -130.6\n",
      "episode 309 score -53.7 average_score -127.6\n",
      "episode 310 score -20.9 average_score -126.8\n",
      "episode 311 score -112.7 average_score -127.0\n",
      "episode 312 score -82.8 average_score -126.1\n",
      "episode 313 score -248.5 average_score -127.7\n",
      "episode 314 score -64.7 average_score -127.1\n",
      "episode 315 score -101.5 average_score -127.6\n",
      "episode 316 score -39.4 average_score -127.8\n",
      "episode 317 score -73.8 average_score -128.1\n",
      "episode 318 score -108.9 average_score -128.0\n",
      "episode 319 score -149.3 average_score -128.9\n",
      "episode 320 score -254.0 average_score -130.8\n",
      "episode 321 score -69.9 average_score -130.1\n",
      "episode 322 score -140.3 average_score -130.7\n",
      "episode 323 score -28.1 average_score -129.3\n",
      "episode 324 score -43.3 average_score -128.6\n",
      "episode 325 score -281.3 average_score -130.3\n",
      "episode 326 score -56.5 average_score -130.2\n",
      "episode 327 score -62.5 average_score -130.2\n",
      "episode 328 score -3.4 average_score -127.0\n",
      "episode 329 score -378.6 average_score -127.9\n",
      "episode 330 score -94.8 average_score -127.2\n",
      "episode 331 score -225.5 average_score -129.3\n",
      "episode 332 score -74.0 average_score -127.9\n",
      "episode 333 score -58.0 average_score -127.7\n",
      "episode 334 score -54.2 average_score -127.5\n",
      "episode 335 score -226.2 average_score -129.3\n",
      "episode 336 score -480.8 average_score -131.5\n",
      "episode 337 score -133.1 average_score -130.2\n",
      "episode 338 score -225.1 average_score -131.5\n",
      "episode 339 score -50.4 average_score -131.2\n",
      "episode 340 score -338.8 average_score -133.1\n",
      "episode 341 score -86.3 average_score -133.5\n",
      "episode 342 score -78.1 average_score -133.2\n",
      "episode 343 score -251.7 average_score -134.9\n",
      "episode 344 score -331.4 average_score -138.1\n",
      "episode 345 score -81.8 average_score -138.1\n",
      "episode 346 score -99.3 average_score -138.0\n",
      "episode 347 score -122.3 average_score -138.8\n",
      "episode 348 score -107.8 average_score -138.4\n",
      "episode 349 score -48.8 average_score -137.9\n",
      "episode 350 score -139.3 average_score -138.4\n",
      "episode 351 score -260.2 average_score -140.3\n",
      "episode 352 score -56.1 average_score -139.4\n",
      "episode 353 score -178.1 average_score -139.6\n",
      "episode 354 score -27.2 average_score -138.1\n",
      "episode 355 score -424.6 average_score -141.5\n",
      "episode 356 score -130.6 average_score -141.3\n",
      "episode 357 score -215.0 average_score -143.2\n",
      "episode 358 score -63.5 average_score -140.7\n",
      "episode 359 score -85.1 average_score -140.1\n",
      "episode 360 score -284.3 average_score -142.2\n",
      "episode 361 score -79.8 average_score -140.3\n",
      "episode 362 score -82.1 average_score -138.5\n",
      "episode 363 score -221.7 average_score -138.4\n",
      "episode 364 score -93.6 average_score -138.4\n",
      "episode 365 score -79.0 average_score -137.6\n",
      "episode 366 score -51.2 average_score -134.8\n",
      "episode 367 score -71.6 average_score -134.5\n",
      "episode 368 score -74.6 average_score -134.3\n",
      "episode 369 score -167.6 average_score -133.0\n",
      "episode 370 score -194.5 average_score -134.3\n",
      "episode 371 score 29.9 average_score -132.9\n",
      "episode 372 score -117.2 average_score -132.9\n",
      "episode 373 score -267.8 average_score -134.6\n",
      "episode 374 score -52.6 average_score -134.7\n",
      "episode 375 score -91.7 average_score -132.4\n",
      "episode 376 score -260.8 average_score -134.5\n",
      "episode 377 score -170.7 average_score -132.7\n",
      "episode 378 score -179.8 average_score -133.5\n",
      "episode 379 score 33.3 average_score -132.4\n",
      "episode 380 score -203.3 average_score -132.4\n",
      "episode 381 score -126.4 average_score -132.0\n",
      "episode 382 score -74.4 average_score -132.0\n",
      "episode 383 score -81.4 average_score -132.0\n",
      "episode 384 score -221.2 average_score -130.5\n",
      "episode 385 score -83.9 average_score -131.4\n",
      "episode 386 score -267.4 average_score -133.3\n",
      "episode 387 score -91.3 average_score -132.1\n",
      "episode 388 score -59.2 average_score -132.6\n",
      "episode 389 score -87.6 average_score -132.4\n",
      "episode 390 score -73.1 average_score -133.0\n",
      "episode 391 score -53.5 average_score -132.4\n",
      "episode 392 score -158.6 average_score -131.7\n",
      "episode 393 score -67.7 average_score -131.3\n",
      "episode 394 score -80.5 average_score -129.6\n",
      "episode 395 score -126.9 average_score -130.5\n",
      "episode 396 score -105.5 average_score -130.6\n",
      "episode 397 score -364.6 average_score -133.6\n",
      "episode 398 score -43.6 average_score -133.5\n",
      "episode 399 score -374.1 average_score -133.3\n",
      "episode 400 score -23.2 average_score -132.4\n",
      "episode 401 score -98.8 average_score -131.7\n",
      "episode 402 score -114.8 average_score -132.4\n",
      "episode 403 score -167.8 average_score -133.7\n",
      "episode 404 score -379.6 average_score -137.2\n",
      "episode 405 score -27.6 average_score -136.1\n",
      "episode 406 score -105.0 average_score -135.7\n",
      "episode 407 score -75.2 average_score -135.4\n",
      "episode 408 score -129.1 average_score -134.7\n",
      "episode 409 score -85.5 average_score -135.0\n",
      "episode 410 score -296.3 average_score -137.7\n",
      "episode 411 score -64.5 average_score -137.2\n",
      "episode 412 score -108.5 average_score -137.5\n",
      "episode 413 score -154.8 average_score -136.6\n",
      "episode 414 score -95.6 average_score -136.9\n",
      "episode 415 score -164.4 average_score -137.5\n",
      "episode 416 score -32.5 average_score -137.4\n",
      "episode 417 score -16.1 average_score -136.9\n",
      "episode 418 score -122.5 average_score -137.0\n",
      "episode 419 score -58.4 average_score -136.1\n",
      "episode 420 score -50.8 average_score -134.0\n",
      "episode 421 score -253.5 average_score -135.9\n",
      "episode 422 score -86.1 average_score -135.3\n",
      "episode 423 score -70.0 average_score -135.8\n",
      "episode 424 score -217.8 average_score -137.5\n",
      "episode 425 score -269.2 average_score -137.4\n",
      "episode 426 score -126.1 average_score -138.1\n",
      "episode 427 score -124.4 average_score -138.7\n",
      "episode 428 score -130.3 average_score -140.0\n",
      "episode 429 score -28.5 average_score -136.5\n",
      "episode 430 score -50.8 average_score -136.0\n",
      "episode 431 score -159.7 average_score -135.4\n",
      "episode 432 score -63.8 average_score -135.3\n",
      "episode 433 score -110.6 average_score -135.8\n",
      "episode 434 score -123.0 average_score -136.5\n",
      "episode 435 score -33.6 average_score -134.6\n",
      "episode 436 score -64.8 average_score -130.4\n",
      "episode 437 score -31.2 average_score -129.4\n",
      "episode 438 score -376.7 average_score -130.9\n",
      "episode 439 score -39.1 average_score -130.8\n",
      "episode 440 score -125.5 average_score -128.6\n",
      "episode 441 score -92.7 average_score -128.7\n",
      "episode 442 score -150.7 average_score -129.4\n",
      "episode 443 score -71.9 average_score -127.6\n",
      "episode 444 score -225.5 average_score -126.6\n",
      "episode 445 score -67.9 average_score -126.4\n",
      "episode 446 score -16.8 average_score -125.6\n",
      "episode 447 score -88.8 average_score -125.3\n",
      "episode 448 score -41.8 average_score -124.6\n",
      "episode 449 score -53.0 average_score -124.7\n",
      "episode 450 score -3.9 average_score -123.3\n",
      "episode 451 score -15.6 average_score -120.9\n",
      "episode 452 score -198.4 average_score -122.3\n",
      "episode 453 score -5.7 average_score -120.6\n",
      "episode 454 score -105.6 average_score -121.3\n",
      "episode 455 score -109.2 average_score -118.2\n",
      "episode 456 score -90.5 average_score -117.8\n",
      "episode 457 score -191.4 average_score -117.6\n",
      "episode 458 score -131.2 average_score -118.2\n",
      "episode 459 score -82.0 average_score -118.2\n",
      "episode 460 score -239.0 average_score -117.7\n",
      "episode 461 score 2.1 average_score -116.9\n",
      "episode 462 score -93.5 average_score -117.0\n",
      "episode 463 score -49.2 average_score -115.3\n",
      "episode 464 score -100.9 average_score -115.4\n",
      "episode 465 score -50.8 average_score -115.1\n",
      "episode 466 score -28.6 average_score -114.9\n",
      "episode 467 score -165.7 average_score -115.8\n",
      "episode 468 score -386.7 average_score -118.9\n",
      "episode 469 score -81.9 average_score -118.1\n",
      "episode 470 score -33.5 average_score -116.5\n",
      "episode 471 score -127.5 average_score -118.1\n",
      "episode 472 score -117.6 average_score -118.1\n",
      "episode 473 score -237.4 average_score -117.8\n",
      "episode 474 score -295.1 average_score -120.2\n",
      "episode 475 score -94.5 average_score -120.2\n",
      "episode 476 score -183.6 average_score -119.4\n",
      "episode 477 score -40.0 average_score -118.1\n",
      "episode 478 score -203.5 average_score -118.4\n",
      "episode 479 score -129.2 average_score -120.0\n",
      "episode 480 score -47.8 average_score -118.4\n",
      "episode 481 score -304.8 average_score -120.2\n",
      "episode 482 score -151.0 average_score -121.0\n",
      "episode 483 score -65.5 average_score -120.8\n",
      "episode 484 score -57.4 average_score -119.2\n",
      "episode 485 score -453.8 average_score -122.9\n",
      "episode 486 score -134.0 average_score -121.6\n",
      "episode 487 score -103.3 average_score -121.7\n",
      "episode 488 score -230.3 average_score -123.4\n",
      "episode 489 score -256.8 average_score -125.1\n",
      "episode 490 score -22.2 average_score -124.6\n",
      "episode 491 score -351.0 average_score -127.5\n",
      "episode 492 score -63.5 average_score -126.6\n",
      "episode 493 score -86.8 average_score -126.8\n",
      "episode 494 score -285.2 average_score -128.8\n",
      "episode 495 score -164.0 average_score -129.2\n",
      "episode 496 score -213.2 average_score -130.3\n",
      "episode 497 score -57.6 average_score -127.2\n",
      "episode 498 score -106.6 average_score -127.8\n",
      "episode 499 score -45.8 average_score -124.6\n",
      "episode 500 score -97.1 average_score -125.3\n",
      "episode 501 score -77.2 average_score -125.1\n",
      "episode 502 score -40.4 average_score -124.3\n",
      "episode 503 score 23.6 average_score -122.4\n",
      "episode 504 score -200.1 average_score -120.6\n",
      "episode 505 score -17.0 average_score -120.5\n",
      "episode 506 score -158.3 average_score -121.1\n",
      "episode 507 score 30.7 average_score -120.0\n",
      "episode 508 score 67.6 average_score -118.0\n",
      "episode 509 score -15.6 average_score -117.3\n",
      "episode 510 score -81.1 average_score -115.2\n",
      "episode 511 score -83.1 average_score -115.4\n",
      "episode 512 score -317.2 average_score -117.4\n",
      "episode 513 score -45.7 average_score -116.4\n",
      "episode 514 score -194.0 average_score -117.3\n",
      "episode 515 score -26.5 average_score -116.0\n",
      "episode 516 score -75.7 average_score -116.4\n",
      "episode 517 score -6.0 average_score -116.3\n",
      "episode 518 score -65.7 average_score -115.7\n",
      "episode 519 score -213.4 average_score -117.3\n",
      "episode 520 score -269.1 average_score -119.5\n",
      "episode 521 score -51.1 average_score -117.4\n",
      "episode 522 score -66.3 average_score -117.2\n",
      "episode 523 score -60.8 average_score -117.1\n",
      "episode 524 score 15.6 average_score -114.8\n",
      "episode 525 score -24.3 average_score -112.4\n",
      "episode 526 score -76.0 average_score -111.9\n",
      "episode 527 score -78.8 average_score -111.4\n",
      "episode 528 score 2.7 average_score -110.1\n",
      "episode 529 score -212.8 average_score -111.9\n",
      "episode 530 score -66.9 average_score -112.1\n",
      "episode 531 score -64.6 average_score -111.1\n",
      "episode 532 score -197.3 average_score -112.5\n",
      "episode 533 score -37.4 average_score -111.7\n",
      "episode 534 score -334.0 average_score -113.8\n",
      "episode 535 score -46.8 average_score -114.0\n",
      "episode 536 score -74.0 average_score -114.1\n",
      "episode 537 score -84.8 average_score -114.6\n",
      "episode 538 score -264.4 average_score -113.5\n",
      "episode 539 score -107.3 average_score -114.2\n",
      "episode 540 score -44.9 average_score -113.3\n",
      "episode 541 score -36.9 average_score -112.8\n",
      "episode 542 score -47.7 average_score -111.8\n",
      "episode 543 score -281.7 average_score -113.9\n",
      "episode 544 score -78.6 average_score -112.4\n",
      "episode 545 score -311.9 average_score -114.8\n",
      "episode 546 score 3.4 average_score -114.6\n",
      "episode 547 score -283.0 average_score -116.6\n",
      "episode 548 score -22.4 average_score -116.4\n",
      "episode 549 score -81.7 average_score -116.7\n",
      "episode 550 score -28.3 average_score -116.9\n",
      "episode 551 score -130.4 average_score -118.1\n",
      "episode 552 score -131.8 average_score -117.4\n",
      "episode 553 score -126.9 average_score -118.6\n",
      "episode 554 score 25.7 average_score -117.3\n",
      "episode 555 score -53.1 average_score -116.7\n",
      "episode 556 score 1.6 average_score -115.8\n",
      "episode 557 score -119.5 average_score -115.1\n",
      "episode 558 score -92.3 average_score -114.7\n",
      "episode 559 score -12.1 average_score -114.0\n",
      "episode 560 score -96.5 average_score -112.6\n",
      "episode 561 score -102.9 average_score -113.6\n",
      "episode 562 score -86.8 average_score -113.6\n",
      "episode 563 score -35.1 average_score -113.4\n",
      "episode 564 score -230.1 average_score -114.7\n",
      "episode 565 score -71.2 average_score -114.9\n",
      "episode 566 score -5.8 average_score -114.7\n",
      "episode 567 score -64.2 average_score -113.7\n",
      "episode 568 score -53.0 average_score -110.3\n",
      "episode 569 score -77.4 average_score -110.3\n",
      "episode 570 score -28.7 average_score -110.2\n",
      "episode 571 score -77.9 average_score -109.7\n",
      "episode 572 score -47.1 average_score -109.0\n",
      "episode 573 score -45.1 average_score -107.1\n",
      "episode 574 score -233.0 average_score -106.5\n",
      "episode 575 score -87.8 average_score -106.4\n",
      "episode 576 score 9.7 average_score -104.5\n",
      "episode 577 score -18.6 average_score -104.3\n",
      "episode 578 score -53.5 average_score -102.8\n",
      "episode 579 score -46.3 average_score -102.0\n",
      "episode 580 score -100.8 average_score -102.5\n",
      "episode 581 score -63.9 average_score -100.1\n",
      "episode 582 score -3.1 average_score -98.6\n",
      "episode 583 score -59.8 average_score -98.5\n",
      "episode 584 score -94.1 average_score -98.9\n",
      "episode 585 score -56.3 average_score -94.9\n",
      "episode 586 score -86.4 average_score -94.5\n",
      "episode 587 score -56.7 average_score -94.0\n",
      "episode 588 score -29.5 average_score -92.0\n",
      "episode 589 score -163.2 average_score -91.0\n",
      "episode 590 score -4.0 average_score -90.9\n",
      "episode 591 score -240.8 average_score -89.8\n",
      "episode 592 score -143.6 average_score -90.6\n",
      "episode 593 score -60.4 average_score -90.3\n",
      "episode 594 score -174.0 average_score -89.2\n",
      "episode 595 score -77.3 average_score -88.3\n",
      "episode 596 score -132.5 average_score -87.5\n",
      "episode 597 score -72.9 average_score -87.7\n",
      "episode 598 score -75.9 average_score -87.4\n",
      "episode 599 score -140.2 average_score -88.3\n",
      "episode 600 score 23.3 average_score -87.1\n",
      "episode 601 score -34.8 average_score -86.7\n",
      "episode 602 score -52.2 average_score -86.8\n",
      "episode 603 score -74.4 average_score -87.8\n",
      "episode 604 score -24.8 average_score -86.0\n",
      "episode 605 score -41.7 average_score -86.3\n",
      "episode 606 score -97.9 average_score -85.7\n",
      "episode 607 score 9.1 average_score -85.9\n",
      "episode 608 score -175.8 average_score -88.3\n",
      "episode 609 score -139.6 average_score -89.5\n",
      "episode 610 score -114.3 average_score -89.9\n",
      "episode 611 score -66.2 average_score -89.7\n",
      "episode 612 score -10.8 average_score -86.6\n",
      "episode 613 score -42.6 average_score -86.6\n",
      "episode 614 score -15.6 average_score -84.8\n",
      "episode 615 score -62.6 average_score -85.2\n",
      "episode 616 score -19.4 average_score -84.6\n",
      "episode 617 score -49.0 average_score -85.1\n",
      "episode 618 score -20.0 average_score -84.6\n",
      "episode 619 score -53.8 average_score -83.0\n",
      "episode 620 score -262.2 average_score -82.9\n",
      "episode 621 score -176.0 average_score -84.2\n",
      "episode 622 score -29.2 average_score -83.8\n",
      "episode 623 score -38.8 average_score -83.6\n",
      "episode 624 score -219.9 average_score -86.0\n",
      "episode 625 score -86.8 average_score -86.6\n",
      "episode 626 score -146.3 average_score -87.3\n",
      "episode 627 score -44.3 average_score -86.9\n",
      "episode 628 score 35.1 average_score -86.6\n",
      "episode 629 score -306.8 average_score -87.6\n",
      "episode 630 score -32.3 average_score -87.2\n",
      "episode 631 score 64.4 average_score -85.9\n",
      "episode 632 score -74.7 average_score -84.7\n",
      "episode 633 score -61.0 average_score -84.9\n",
      "episode 634 score -124.3 average_score -82.8\n",
      "episode 635 score -22.9 average_score -82.6\n",
      "episode 636 score -9.1 average_score -81.9\n",
      "episode 637 score 30.2 average_score -80.8\n",
      "episode 638 score -31.3 average_score -78.5\n",
      "episode 639 score -43.2 average_score -77.8\n",
      "episode 640 score -144.8 average_score -78.8\n",
      "episode 641 score -8.8 average_score -78.5\n",
      "episode 642 score -9.0 average_score -78.1\n",
      "episode 643 score -48.4 average_score -75.8\n",
      "episode 644 score -36.6 average_score -75.4\n",
      "episode 645 score -14.1 average_score -72.4\n",
      "episode 646 score -93.5 average_score -73.4\n",
      "episode 647 score -13.5 average_score -70.7\n",
      "episode 648 score -52.4 average_score -71.0\n",
      "episode 649 score -5.3 average_score -70.2\n",
      "episode 650 score -183.5 average_score -71.8\n",
      "episode 651 score -178.8 average_score -72.3\n",
      "episode 652 score 1.4 average_score -70.9\n",
      "episode 653 score -139.5 average_score -71.1\n",
      "episode 654 score -135.0 average_score -72.7\n",
      "episode 655 score -60.8 average_score -72.7\n",
      "episode 656 score -64.9 average_score -73.4\n",
      "episode 657 score -95.2 average_score -73.2\n",
      "episode 658 score -38.8 average_score -72.6\n",
      "episode 659 score 42.0 average_score -72.1\n",
      "episode 660 score -57.3 average_score -71.7\n",
      "episode 661 score -40.3 average_score -71.1\n",
      "episode 662 score -54.3 average_score -70.7\n",
      "episode 663 score -76.9 average_score -71.2\n",
      "episode 664 score -75.6 average_score -69.6\n",
      "episode 665 score -46.5 average_score -69.4\n",
      "episode 666 score -172.1 average_score -71.0\n",
      "episode 667 score -89.9 average_score -71.3\n",
      "episode 668 score -17.4 average_score -70.9\n",
      "episode 669 score 35.2 average_score -69.8\n",
      "episode 670 score -17.6 average_score -69.7\n",
      "episode 671 score -16.8 average_score -69.1\n",
      "episode 672 score -117.2 average_score -69.8\n",
      "episode 673 score -62.4 average_score -70.0\n",
      "episode 674 score -58.1 average_score -68.2\n",
      "episode 675 score -36.5 average_score -67.7\n",
      "episode 676 score -55.6 average_score -68.3\n",
      "episode 677 score -141.4 average_score -69.6\n",
      "episode 678 score -17.9 average_score -69.2\n",
      "episode 679 score -27.1 average_score -69.0\n",
      "episode 680 score -8.9 average_score -68.1\n",
      "episode 681 score -25.1 average_score -67.7\n",
      "episode 682 score -83.4 average_score -68.5\n",
      "episode 683 score 21.6 average_score -67.7\n",
      "episode 684 score -51.0 average_score -67.3\n",
      "episode 685 score -17.4 average_score -66.9\n",
      "episode 686 score -75.8 average_score -66.8\n",
      "episode 687 score -138.2 average_score -67.6\n",
      "episode 688 score -32.6 average_score -67.6\n",
      "episode 689 score -283.5 average_score -68.8\n",
      "episode 690 score -92.8 average_score -69.7\n",
      "episode 691 score -204.7 average_score -69.4\n",
      "episode 692 score -144.1 average_score -69.4\n",
      "episode 693 score -76.4 average_score -69.5\n",
      "episode 694 score -44.4 average_score -68.2\n",
      "episode 695 score -57.7 average_score -68.0\n",
      "episode 696 score -115.8 average_score -67.9\n",
      "episode 697 score -47.8 average_score -67.6\n",
      "episode 698 score 5.7 average_score -66.8\n",
      "episode 699 score -113.0 average_score -66.5\n",
      "episode 700 score -55.2 average_score -67.3\n",
      "episode 701 score -18.6 average_score -67.1\n",
      "episode 702 score -94.1 average_score -67.6\n",
      "episode 703 score -209.2 average_score -68.9\n",
      "episode 704 score 11.8 average_score -68.6\n",
      "episode 705 score 71.9 average_score -67.4\n",
      "episode 706 score -66.2 average_score -67.1\n",
      "episode 707 score -23.0 average_score -67.4\n",
      "episode 708 score 37.5 average_score -65.3\n",
      "episode 709 score 21.9 average_score -63.7\n",
      "episode 710 score 3.6 average_score -62.5\n",
      "episode 711 score -19.3 average_score -62.0\n",
      "episode 712 score -42.4 average_score -62.3\n",
      "episode 713 score -59.3 average_score -62.5\n",
      "episode 714 score -6.6 average_score -62.4\n",
      "episode 715 score -33.1 average_score -62.1\n",
      "episode 716 score -4.2 average_score -62.0\n",
      "episode 717 score -42.4 average_score -61.9\n",
      "episode 718 score -7.2 average_score -61.8\n",
      "episode 719 score 24.9 average_score -61.0\n",
      "episode 720 score -38.7 average_score -58.8\n",
      "episode 721 score -58.4 average_score -57.6\n",
      "episode 722 score 7.9 average_score -57.2\n",
      "episode 723 score 1.1 average_score -56.8\n",
      "episode 724 score -262.4 average_score -57.2\n",
      "episode 725 score 56.9 average_score -55.8\n",
      "episode 726 score -68.9 average_score -55.0\n",
      "episode 727 score -77.0 average_score -55.3\n",
      "episode 728 score -161.0 average_score -57.3\n",
      "episode 729 score -57.3 average_score -54.8\n",
      "episode 730 score -16.2 average_score -54.7\n",
      "episode 731 score -166.9 average_score -57.0\n",
      "episode 732 score -127.3 average_score -57.5\n",
      "episode 733 score -241.4 average_score -59.3\n",
      "episode 734 score 5.6 average_score -58.0\n",
      "episode 735 score -193.4 average_score -59.7\n",
      "episode 736 score -90.0 average_score -60.5\n",
      "episode 737 score 19.4 average_score -60.6\n",
      "episode 738 score -75.8 average_score -61.1\n",
      "episode 739 score -63.8 average_score -61.3\n",
      "episode 740 score 6.6 average_score -59.8\n",
      "episode 741 score -59.1 average_score -60.3\n",
      "episode 742 score -371.8 average_score -63.9\n",
      "episode 743 score -68.9 average_score -64.1\n",
      "episode 744 score -20.6 average_score -63.9\n",
      "episode 745 score -61.8 average_score -64.4\n",
      "episode 746 score -2.2 average_score -63.5\n",
      "episode 747 score 46.8 average_score -62.9\n",
      "episode 748 score -61.6 average_score -63.0\n",
      "episode 749 score -68.4 average_score -63.6\n",
      "episode 750 score -75.5 average_score -62.5\n",
      "episode 751 score -8.3 average_score -60.8\n",
      "episode 752 score -165.7 average_score -62.5\n",
      "episode 753 score -52.7 average_score -61.6\n",
      "episode 754 score -14.8 average_score -60.4\n",
      "episode 755 score -24.9 average_score -60.1\n",
      "episode 756 score -18.1 average_score -59.6\n",
      "episode 757 score -27.8 average_score -58.9\n",
      "episode 758 score -5.1 average_score -58.6\n",
      "episode 759 score -74.4 average_score -59.8\n",
      "episode 760 score -8.7 average_score -59.3\n",
      "episode 761 score 17.3 average_score -58.7\n",
      "episode 762 score -148.0 average_score -59.6\n",
      "episode 763 score -89.4 average_score -59.8\n",
      "episode 764 score -139.2 average_score -60.4\n",
      "episode 765 score -102.9 average_score -61.0\n",
      "episode 766 score 28.6 average_score -58.9\n",
      "episode 767 score -16.0 average_score -58.2\n",
      "episode 768 score -42.1 average_score -58.5\n",
      "episode 769 score -4.1 average_score -58.9\n",
      "episode 770 score -30.1 average_score -59.0\n",
      "episode 771 score -14.9 average_score -59.0\n",
      "episode 772 score -35.9 average_score -58.1\n",
      "episode 773 score -227.7 average_score -59.8\n",
      "episode 774 score 22.9 average_score -59.0\n",
      "episode 775 score -40.3 average_score -59.0\n",
      "episode 776 score -141.5 average_score -59.9\n",
      "episode 777 score -128.2 average_score -59.8\n",
      "episode 778 score -11.5 average_score -59.7\n",
      "episode 779 score -24.0 average_score -59.7\n",
      "episode 780 score -35.9 average_score -59.9\n",
      "episode 781 score 16.1 average_score -59.5\n",
      "episode 782 score -282.1 average_score -61.5\n",
      "episode 783 score -78.8 average_score -62.5\n",
      "episode 784 score 23.9 average_score -61.8\n",
      "episode 785 score 6.1 average_score -61.5\n",
      "episode 786 score 3.2 average_score -60.7\n",
      "episode 787 score -263.0 average_score -62.0\n",
      "episode 788 score -40.9 average_score -62.1\n",
      "episode 789 score -81.0 average_score -60.0\n",
      "episode 790 score 13.4 average_score -59.0\n",
      "episode 791 score -64.2 average_score -57.6\n",
      "episode 792 score -84.8 average_score -57.0\n",
      "episode 793 score -37.3 average_score -56.6\n",
      "episode 794 score -3.0 average_score -56.2\n",
      "episode 795 score -2.1 average_score -55.6\n",
      "episode 796 score -52.8 average_score -55.0\n",
      "episode 797 score -62.7 average_score -55.1\n",
      "episode 798 score -91.1 average_score -56.1\n",
      "episode 799 score -8.3 average_score -55.1\n",
      "episode 800 score -161.4 average_score -56.1\n",
      "episode 801 score -72.8 average_score -56.7\n",
      "episode 802 score 20.9 average_score -55.5\n",
      "episode 803 score -28.1 average_score -53.7\n",
      "episode 804 score -152.5 average_score -55.3\n",
      "episode 805 score -174.5 average_score -57.8\n",
      "episode 806 score 17.9 average_score -57.0\n",
      "episode 807 score -42.1 average_score -57.2\n",
      "episode 808 score -43.9 average_score -58.0\n",
      "episode 809 score -230.2 average_score -60.5\n",
      "episode 810 score -28.0 average_score -60.8\n",
      "episode 811 score -10.3 average_score -60.7\n",
      "episode 812 score -190.5 average_score -62.2\n",
      "episode 813 score -63.3 average_score -62.2\n",
      "episode 814 score -48.0 average_score -62.7\n",
      "episode 815 score -1.4 average_score -62.3\n",
      "episode 816 score -69.1 average_score -63.0\n",
      "episode 817 score -87.9 average_score -63.4\n",
      "episode 818 score -19.0 average_score -63.6\n",
      "episode 819 score 20.5 average_score -63.6\n",
      "episode 820 score 41.6 average_score -62.8\n",
      "episode 821 score -224.3 average_score -64.5\n",
      "episode 822 score -377.9 average_score -68.3\n",
      "episode 823 score -169.2 average_score -70.0\n",
      "episode 824 score -76.4 average_score -68.2\n",
      "episode 825 score -132.1 average_score -70.1\n",
      "episode 826 score -338.0 average_score -72.7\n",
      "episode 827 score -68.9 average_score -72.7\n",
      "episode 828 score -49.1 average_score -71.5\n",
      "episode 829 score 28.3 average_score -70.7\n",
      "episode 830 score -199.5 average_score -72.5\n",
      "episode 831 score -278.5 average_score -73.6\n",
      "episode 832 score 20.8 average_score -72.2\n",
      "episode 833 score -28.9 average_score -70.0\n",
      "episode 834 score 51.9 average_score -69.6\n",
      "episode 835 score -26.4 average_score -67.9\n",
      "episode 836 score -26.5 average_score -67.3\n",
      "episode 837 score -245.6 average_score -69.9\n",
      "episode 838 score -241.4 average_score -71.6\n",
      "episode 839 score 71.9 average_score -70.2\n",
      "episode 840 score -194.4 average_score -72.2\n",
      "episode 841 score -147.5 average_score -73.1\n",
      "episode 842 score 20.1 average_score -69.2\n",
      "episode 843 score -14.9 average_score -68.6\n",
      "episode 844 score -114.4 average_score -69.6\n",
      "episode 845 score 24.8 average_score -68.7\n",
      "episode 846 score -24.9 average_score -68.9\n",
      "episode 847 score -85.9 average_score -70.3\n",
      "episode 848 score -38.2 average_score -70.0\n",
      "episode 849 score -35.6 average_score -69.7\n",
      "episode 850 score -160.5 average_score -70.6\n",
      "episode 851 score -14.4 average_score -70.6\n",
      "episode 852 score 23.4 average_score -68.7\n",
      "episode 853 score 51.1 average_score -67.7\n",
      "episode 854 score -277.4 average_score -70.3\n",
      "episode 855 score -36.4 average_score -70.4\n",
      "episode 856 score -174.2 average_score -72.0\n",
      "episode 857 score -63.5 average_score -72.4\n",
      "episode 858 score -121.8 average_score -73.5\n",
      "episode 859 score -198.8 average_score -74.8\n",
      "episode 860 score 18.7 average_score -74.5\n",
      "episode 861 score -157.6 average_score -76.2\n",
      "episode 862 score -99.7 average_score -75.8\n",
      "episode 863 score -54.7 average_score -75.4\n",
      "episode 864 score 26.8 average_score -73.7\n",
      "episode 865 score 48.7 average_score -72.2\n",
      "episode 866 score -78.0 average_score -73.3\n",
      "episode 867 score -44.5 average_score -73.6\n",
      "episode 868 score -210.2 average_score -75.3\n",
      "episode 869 score -63.2 average_score -75.9\n",
      "episode 870 score 46.5 average_score -75.1\n",
      "episode 871 score -290.4 average_score -77.8\n",
      "episode 872 score 6.8 average_score -77.4\n",
      "episode 873 score 57.7 average_score -74.6\n",
      "episode 874 score 11.7 average_score -74.7\n",
      "episode 875 score 9.5 average_score -74.2\n",
      "episode 876 score -75.4 average_score -73.5\n",
      "episode 877 score -99.0 average_score -73.2\n",
      "episode 878 score -20.5 average_score -73.3\n",
      "episode 879 score -163.9 average_score -74.7\n",
      "episode 880 score -5.4 average_score -74.4\n",
      "episode 881 score 32.4 average_score -74.2\n",
      "episode 882 score 17.4 average_score -71.3\n",
      "episode 883 score 57.1 average_score -69.9\n",
      "episode 884 score 66.9 average_score -69.5\n",
      "episode 885 score -9.5 average_score -69.6\n",
      "episode 886 score -47.7 average_score -70.1\n",
      "episode 887 score -45.6 average_score -68.0\n",
      "episode 888 score -39.0 average_score -67.9\n",
      "episode 889 score -27.4 average_score -67.4\n",
      "episode 890 score 3.6 average_score -67.5\n",
      "episode 891 score 19.4 average_score -66.7\n",
      "episode 892 score -219.5 average_score -68.0\n",
      "episode 893 score 61.3 average_score -67.0\n",
      "episode 894 score 48.4 average_score -66.5\n",
      "episode 895 score -10.2 average_score -66.6\n",
      "episode 896 score -52.1 average_score -66.6\n",
      "episode 897 score -312.9 average_score -69.1\n",
      "episode 898 score 17.1 average_score -68.0\n",
      "episode 899 score -48.7 average_score -68.4\n",
      "episode 900 score 19.1 average_score -66.6\n",
      "episode 901 score 18.2 average_score -65.7\n",
      "episode 902 score -10.1 average_score -66.0\n",
      "episode 903 score 18.8 average_score -65.5\n",
      "episode 904 score 34.3 average_score -63.7\n",
      "episode 905 score -13.0 average_score -62.0\n",
      "episode 906 score 1.3 average_score -62.2\n",
      "episode 907 score -4.0 average_score -61.8\n",
      "episode 908 score -193.8 average_score -63.3\n",
      "episode 909 score -76.9 average_score -61.8\n",
      "episode 910 score -34.1 average_score -61.9\n",
      "episode 911 score -57.3 average_score -62.3\n",
      "episode 912 score -15.4 average_score -60.6\n",
      "episode 913 score -29.7 average_score -60.2\n",
      "episode 914 score 22.9 average_score -59.5\n",
      "episode 915 score 12.3 average_score -59.4\n",
      "episode 916 score 13.6 average_score -58.6\n",
      "episode 917 score -236.9 average_score -60.1\n",
      "episode 918 score -179.2 average_score -61.7\n",
      "episode 919 score 24.5 average_score -61.6\n",
      "episode 920 score 28.4 average_score -61.8\n",
      "episode 921 score -35.7 average_score -59.9\n",
      "episode 922 score 30.6 average_score -55.8\n",
      "episode 923 score -24.2 average_score -54.3\n",
      "episode 924 score -24.9 average_score -53.8\n",
      "episode 925 score 0.4 average_score -52.5\n",
      "episode 926 score 17.7 average_score -48.9\n",
      "episode 927 score 44.7 average_score -47.8\n",
      "episode 928 score -33.4 average_score -47.6\n",
      "episode 929 score 12.3 average_score -47.8\n",
      "episode 930 score 47.3 average_score -45.3\n",
      "episode 931 score -7.0 average_score -42.6\n",
      "episode 932 score -33.2 average_score -43.2\n",
      "episode 933 score -18.2 average_score -43.0\n",
      "episode 934 score -102.3 average_score -44.6\n",
      "episode 935 score -97.0 average_score -45.3\n",
      "episode 936 score -39.3 average_score -45.4\n",
      "episode 937 score -38.1 average_score -43.4\n",
      "episode 938 score -13.1 average_score -41.1\n",
      "episode 939 score -9.4 average_score -41.9\n",
      "episode 940 score 16.3 average_score -39.8\n",
      "episode 941 score 8.1 average_score -38.2\n",
      "episode 942 score -38.5 average_score -38.8\n",
      "episode 943 score 25.1 average_score -38.4\n",
      "episode 944 score 26.5 average_score -37.0\n",
      "episode 945 score 30.5 average_score -36.9\n",
      "episode 946 score 31.4 average_score -36.4\n",
      "episode 947 score 3.8 average_score -35.5\n",
      "episode 948 score 2.4 average_score -35.1\n",
      "episode 949 score 1.6 average_score -34.7\n",
      "episode 950 score -4.2 average_score -33.1\n",
      "episode 951 score -8.9 average_score -33.1\n",
      "episode 952 score 21.3 average_score -33.1\n",
      "episode 953 score -87.4 average_score -34.5\n",
      "episode 954 score -177.6 average_score -33.5\n",
      "episode 955 score -65.5 average_score -33.8\n",
      "episode 956 score 17.9 average_score -31.9\n",
      "episode 957 score -92.7 average_score -32.2\n",
      "episode 958 score 37.0 average_score -30.6\n",
      "episode 959 score 36.1 average_score -28.2\n",
      "episode 960 score -30.7 average_score -28.7\n",
      "episode 961 score -18.0 average_score -27.3\n",
      "episode 962 score 33.6 average_score -26.0\n",
      "episode 963 score 55.7 average_score -24.9\n",
      "episode 964 score -15.2 average_score -25.3\n",
      "episode 965 score -17.3 average_score -26.0\n",
      "episode 966 score -4.8 average_score -25.2\n",
      "episode 967 score -27.4 average_score -25.1\n",
      "episode 968 score -13.6 average_score -23.1\n",
      "episode 969 score 14.8 average_score -22.3\n",
      "episode 970 score 3.3 average_score -22.7\n",
      "episode 971 score 8.4 average_score -19.8\n",
      "episode 972 score 45.3 average_score -19.4\n",
      "episode 973 score -90.5 average_score -20.9\n",
      "episode 974 score -54.4 average_score -21.5\n",
      "episode 975 score -102.7 average_score -22.6\n",
      "episode 976 score 98.7 average_score -20.9\n",
      "episode 977 score -134.4 average_score -21.2\n",
      "episode 978 score 26.9 average_score -20.8\n",
      "episode 979 score 16.9 average_score -19.0\n",
      "episode 980 score 12.1 average_score -18.8\n",
      "episode 981 score 55.1 average_score -18.6\n",
      "episode 982 score -53.3 average_score -19.3\n",
      "episode 983 score -15.6 average_score -20.0\n",
      "episode 984 score 3.9 average_score -20.6\n",
      "episode 985 score -228.3 average_score -22.8\n",
      "episode 986 score 10.4 average_score -22.2\n",
      "episode 987 score -70.2 average_score -22.5\n",
      "episode 988 score 61.2 average_score -21.5\n",
      "episode 989 score -60.7 average_score -21.8\n",
      "episode 990 score -13.9 average_score -22.0\n",
      "episode 991 score -13.1 average_score -22.3\n",
      "episode 992 score -2.6 average_score -20.1\n",
      "episode 993 score -32.8 average_score -21.1\n",
      "episode 994 score 37.5 average_score -21.2\n",
      "episode 995 score 29.1 average_score -20.8\n",
      "episode 996 score 12.0 average_score -20.2\n",
      "episode 997 score 15.4 average_score -16.9\n",
      "episode 998 score 74.1 average_score -16.3\n",
      "episode 999 score 49.8 average_score -15.3\n",
      "episode 1000 score 39.5 average_score -15.1\n",
      "episode 1001 score 55.4 average_score -14.7\n",
      "episode 1002 score -8.3 average_score -14.7\n",
      "episode 1003 score -24.5 average_score -15.2\n",
      "episode 1004 score -0.9 average_score -15.5\n",
      "episode 1005 score -56.6 average_score -15.9\n",
      "episode 1006 score 6.1 average_score -15.9\n",
      "episode 1007 score -42.9 average_score -16.3\n",
      "episode 1008 score -25.7 average_score -14.6\n",
      "episode 1009 score -50.8 average_score -14.3\n",
      "episode 1010 score -12.6 average_score -14.1\n",
      "episode 1011 score -27.7 average_score -13.8\n",
      "episode 1012 score -36.6 average_score -14.1\n",
      "episode 1013 score 9.1 average_score -13.7\n",
      "episode 1014 score 92.4 average_score -13.0\n",
      "episode 1015 score 10.6 average_score -13.0\n",
      "episode 1016 score 22.2 average_score -12.9\n",
      "episode 1017 score 2.0 average_score -10.5\n",
      "episode 1018 score -115.0 average_score -9.9\n",
      "episode 1019 score -10.8 average_score -10.2\n",
      "episode 1020 score -42.3 average_score -10.9\n",
      "episode 1021 score 31.4 average_score -10.3\n",
      "episode 1022 score 25.6 average_score -10.3\n",
      "episode 1023 score 48.1 average_score -9.6\n",
      "episode 1024 score -14.0 average_score -9.5\n",
      "episode 1025 score -23.1 average_score -9.7\n",
      "episode 1026 score -107.2 average_score -11.0\n",
      "episode 1027 score 10.4 average_score -11.3\n",
      "episode 1028 score 76.1 average_score -10.2\n",
      "episode 1029 score -37.9 average_score -10.7\n",
      "episode 1030 score -1.0 average_score -11.2\n",
      "episode 1031 score -9.1 average_score -11.2\n",
      "episode 1032 score -17.7 average_score -11.1\n",
      "episode 1033 score -1.4 average_score -10.9\n",
      "episode 1034 score 8.5 average_score -9.8\n",
      "episode 1035 score -7.0 average_score -8.9\n",
      "episode 1036 score 61.0 average_score -7.9\n",
      "episode 1037 score -87.8 average_score -8.4\n",
      "episode 1038 score 5.6 average_score -8.2\n",
      "episode 1039 score 15.3 average_score -7.9\n",
      "episode 1040 score 49.0 average_score -7.6\n",
      "episode 1041 score 23.3 average_score -7.5\n",
      "episode 1042 score 25.5 average_score -6.8\n",
      "episode 1043 score 78.3 average_score -6.3\n",
      "episode 1044 score 70.1 average_score -5.9\n",
      "episode 1045 score -32.5 average_score -6.5\n",
      "episode 1046 score -14.3 average_score -6.9\n",
      "episode 1047 score 26.4 average_score -6.7\n",
      "episode 1048 score -63.1 average_score -7.4\n",
      "episode 1049 score -18.5 average_score -7.6\n",
      "episode 1050 score -18.5 average_score -7.7\n",
      "episode 1051 score -78.8 average_score -8.4\n",
      "episode 1052 score -11.9 average_score -8.7\n",
      "episode 1053 score 79.8 average_score -7.1\n",
      "episode 1054 score -90.1 average_score -6.2\n",
      "episode 1055 score 11.8 average_score -5.4\n",
      "episode 1056 score -19.6 average_score -5.8\n",
      "episode 1057 score 14.8 average_score -4.7\n",
      "episode 1058 score -126.4 average_score -6.4\n",
      "episode 1059 score 26.0 average_score -6.5\n",
      "episode 1060 score 79.7 average_score -5.4\n",
      "episode 1061 score -74.3 average_score -5.9\n",
      "episode 1062 score -56.5 average_score -6.8\n",
      "episode 1063 score 49.7 average_score -6.9\n",
      "episode 1064 score -23.4 average_score -7.0\n",
      "episode 1065 score -4.8 average_score -6.8\n",
      "episode 1066 score -48.9 average_score -7.3\n",
      "episode 1067 score 66.1 average_score -6.3\n",
      "episode 1068 score -202.1 average_score -8.2\n",
      "episode 1069 score -53.7 average_score -8.9\n",
      "episode 1070 score -37.3 average_score -9.3\n",
      "episode 1071 score -48.2 average_score -9.9\n",
      "episode 1072 score -2.5 average_score -10.4\n",
      "episode 1073 score 93.8 average_score -8.5\n",
      "episode 1074 score -0.4 average_score -8.0\n",
      "episode 1075 score 27.7 average_score -6.7\n",
      "episode 1076 score 41.4 average_score -7.2\n",
      "episode 1077 score 8.6 average_score -5.8\n",
      "episode 1078 score 0.9 average_score -6.1\n",
      "episode 1079 score 59.0 average_score -5.7\n",
      "episode 1080 score 34.2 average_score -5.4\n",
      "episode 1081 score 70.4 average_score -5.3\n",
      "episode 1082 score -155.9 average_score -6.3\n",
      "episode 1083 score -106.2 average_score -7.2\n",
      "episode 1084 score -19.0 average_score -7.4\n",
      "episode 1085 score 37.4 average_score -4.8\n",
      "episode 1086 score -177.3 average_score -6.7\n",
      "episode 1087 score -58.5 average_score -6.5\n",
      "episode 1088 score 82.9 average_score -6.3\n",
      "episode 1089 score -194.5 average_score -7.7\n",
      "episode 1090 score -13.1 average_score -7.7\n",
      "episode 1091 score 23.9 average_score -7.3\n",
      "episode 1092 score 23.1 average_score -7.0\n",
      "episode 1093 score -154.7 average_score -8.3\n",
      "episode 1094 score -1.4 average_score -8.6\n",
      "episode 1095 score 69.2 average_score -8.2\n",
      "episode 1096 score 57.3 average_score -7.8\n",
      "episode 1097 score -178.2 average_score -9.7\n",
      "episode 1098 score -123.4 average_score -11.7\n",
      "episode 1099 score -25.6 average_score -12.5\n",
      "episode 1100 score -49.9 average_score -13.3\n",
      "episode 1101 score 61.8 average_score -13.3\n",
      "episode 1102 score 55.1 average_score -12.6\n",
      "episode 1103 score -248.1 average_score -14.9\n",
      "episode 1104 score 5.3 average_score -14.8\n",
      "episode 1105 score -23.7 average_score -14.5\n",
      "episode 1106 score 57.9 average_score -14.0\n",
      "episode 1107 score 71.6 average_score -12.8\n",
      "episode 1108 score -176.3 average_score -14.3\n",
      "episode 1109 score -14.1 average_score -14.0\n",
      "episode 1110 score -173.6 average_score -15.6\n",
      "episode 1111 score -75.5 average_score -16.1\n",
      "episode 1112 score 21.7 average_score -15.5\n",
      "episode 1113 score 66.5 average_score -14.9\n",
      "episode 1114 score -181.3 average_score -17.6\n",
      "episode 1115 score 79.9 average_score -16.9\n",
      "episode 1116 score -7.8 average_score -17.2\n",
      "episode 1117 score -21.8 average_score -17.5\n",
      "episode 1118 score -9.7 average_score -16.4\n",
      "episode 1119 score -71.1 average_score -17.0\n",
      "episode 1120 score -10.0 average_score -16.7\n",
      "episode 1121 score -16.6 average_score -17.2\n",
      "episode 1122 score 7.4 average_score -17.4\n",
      "episode 1123 score -122.6 average_score -19.1\n",
      "episode 1124 score 54.5 average_score -18.4\n",
      "episode 1125 score 63.6 average_score -17.5\n",
      "episode 1126 score 59.5 average_score -15.9\n",
      "episode 1127 score -33.2 average_score -16.3\n",
      "episode 1128 score 28.5 average_score -16.8\n",
      "episode 1129 score 85.1 average_score -15.5\n",
      "episode 1130 score -12.5 average_score -15.7\n",
      "episode 1131 score -14.8 average_score -15.7\n",
      "episode 1132 score 26.5 average_score -15.3\n",
      "episode 1133 score 47.3 average_score -14.8\n",
      "episode 1134 score 0.8 average_score -14.9\n",
      "episode 1135 score -1.2 average_score -14.8\n",
      "episode 1136 score 26.5 average_score -15.1\n",
      "episode 1137 score 22.7 average_score -14.0\n",
      "episode 1138 score 57.4 average_score -13.5\n",
      "episode 1139 score 66.2 average_score -13.0\n",
      "episode 1140 score 116.7 average_score -12.3\n",
      "episode 1141 score 98.4 average_score -11.6\n",
      "episode 1142 score -13.0 average_score -12.0\n",
      "episode 1143 score 110.6 average_score -11.6\n",
      "episode 1144 score 110.8 average_score -11.2\n",
      "episode 1145 score 36.5 average_score -10.6\n",
      "episode 1146 score -1.7 average_score -10.4\n",
      "episode 1147 score 120.2 average_score -9.5\n",
      "episode 1148 score -52.8 average_score -9.4\n",
      "episode 1149 score 19.2 average_score -9.0\n",
      "episode 1150 score 94.8 average_score -7.9\n",
      "episode 1151 score 42.6 average_score -6.7\n",
      "episode 1152 score -7.6 average_score -6.6\n",
      "episode 1153 score 17.3 average_score -7.2\n",
      "episode 1154 score 89.0 average_score -5.4\n",
      "episode 1155 score 74.9 average_score -4.8\n",
      "episode 1156 score 57.3 average_score -4.0\n",
      "episode 1157 score -17.1 average_score -4.4\n",
      "episode 1158 score 56.9 average_score -2.5\n",
      "episode 1159 score 99.2 average_score -1.8\n",
      "episode 1160 score 66.7 average_score -1.9\n",
      "episode 1161 score -33.6 average_score -1.5\n",
      "episode 1162 score -141.2 average_score -2.4\n",
      "episode 1163 score 120.1 average_score -1.7\n",
      "episode 1164 score 79.3 average_score -0.6\n",
      "episode 1165 score 5.2 average_score -0.5\n",
      "episode 1166 score 3.0 average_score -0.0\n",
      "episode 1167 score 120.7 average_score 0.5\n",
      "episode 1168 score -2.5 average_score 2.5\n",
      "episode 1169 score -30.0 average_score 2.8\n",
      "episode 1170 score 84.4 average_score 4.0\n",
      "episode 1171 score 110.4 average_score 5.6\n",
      "episode 1172 score -180.3 average_score 3.8\n",
      "episode 1173 score 82.9 average_score 3.7\n",
      "episode 1174 score -11.6 average_score 3.6\n",
      "episode 1175 score 1.5 average_score 3.3\n",
      "episode 1176 score 46.0 average_score 3.3\n",
      "episode 1177 score 48.8 average_score 3.7\n",
      "episode 1178 score 35.8 average_score 4.1\n",
      "episode 1179 score 3.3 average_score 3.5\n",
      "episode 1180 score -24.8 average_score 2.9\n",
      "episode 1181 score -42.2 average_score 1.8\n",
      "episode 1182 score 28.6 average_score 3.7\n",
      "episode 1183 score 12.9 average_score 4.9\n",
      "episode 1184 score 4.7 average_score 5.1\n",
      "episode 1185 score -21.0 average_score 4.5\n",
      "episode 1186 score 42.7 average_score 6.7\n",
      "episode 1187 score 137.0 average_score 8.7\n",
      "episode 1188 score -14.9 average_score 7.7\n",
      "episode 1189 score -12.8 average_score 9.5\n",
      "episode 1190 score -17.3 average_score 9.5\n",
      "episode 1191 score 2.9 average_score 9.3\n",
      "episode 1192 score -1.3 average_score 9.0\n",
      "episode 1193 score 46.6 average_score 11.0\n",
      "episode 1194 score -30.3 average_score 10.7\n",
      "episode 1195 score 10.8 average_score 10.2\n",
      "episode 1196 score -18.4 average_score 9.4\n",
      "episode 1197 score 15.6 average_score 11.3\n",
      "episode 1198 score 32.5 average_score 12.9\n",
      "episode 1199 score -66.5 average_score 12.5\n",
      "episode 1200 score -35.6 average_score 12.6\n",
      "episode 1201 score -8.1 average_score 11.9\n",
      "episode 1202 score 83.5 average_score 12.2\n",
      "episode 1203 score -28.8 average_score 14.4\n",
      "episode 1204 score -89.7 average_score 13.5\n",
      "episode 1205 score 8.0 average_score 13.8\n",
      "episode 1206 score -39.0 average_score 12.8\n",
      "episode 1207 score 138.9 average_score 13.5\n",
      "episode 1208 score 5.4 average_score 15.3\n",
      "episode 1209 score -64.6 average_score 14.8\n",
      "episode 1210 score 39.3 average_score 16.9\n",
      "episode 1211 score 0.5 average_score 17.7\n",
      "episode 1212 score 24.0 average_score 17.7\n",
      "episode 1213 score -58.1 average_score 16.5\n",
      "episode 1214 score -13.5 average_score 18.1\n",
      "episode 1215 score -42.3 average_score 16.9\n",
      "episode 1216 score -77.2 average_score 16.2\n",
      "episode 1217 score -60.8 average_score 15.8\n",
      "episode 1218 score 59.3 average_score 16.5\n",
      "episode 1219 score 38.7 average_score 17.6\n",
      "episode 1220 score -158.2 average_score 16.1\n",
      "episode 1221 score -107.5 average_score 15.2\n",
      "episode 1222 score 45.8 average_score 15.6\n",
      "episode 1223 score 50.6 average_score 17.3\n",
      "episode 1224 score 120.6 average_score 18.0\n",
      "episode 1225 score -37.0 average_score 17.0\n",
      "episode 1226 score -25.1 average_score 16.1\n",
      "episode 1227 score 6.3 average_score 16.5\n",
      "episode 1228 score 42.4 average_score 16.7\n",
      "episode 1229 score -45.8 average_score 15.4\n",
      "episode 1230 score 130.7 average_score 16.8\n",
      "episode 1231 score 25.0 average_score 17.2\n",
      "episode 1232 score -17.0 average_score 16.8\n",
      "episode 1233 score 69.4 average_score 17.0\n",
      "episode 1234 score -88.9 average_score 16.1\n",
      "episode 1235 score -0.7 average_score 16.1\n",
      "episode 1236 score 22.3 average_score 16.1\n",
      "episode 1237 score 1.5 average_score 15.8\n",
      "episode 1238 score -19.1 average_score 15.1\n",
      "episode 1239 score -4.8 average_score 14.4\n",
      "episode 1240 score 3.5 average_score 13.2\n",
      "episode 1241 score 117.1 average_score 13.4\n",
      "episode 1242 score 45.6 average_score 14.0\n",
      "episode 1243 score -75.4 average_score 12.1\n",
      "episode 1244 score -1.0 average_score 11.0\n",
      "episode 1245 score -244.4 average_score 8.2\n",
      "episode 1246 score 102.5 average_score 9.3\n",
      "episode 1247 score 24.5 average_score 8.3\n",
      "episode 1248 score 141.7 average_score 10.2\n",
      "episode 1249 score -46.8 average_score 9.6\n",
      "episode 1250 score 131.1 average_score 10.0\n",
      "episode 1251 score 31.9 average_score 9.8\n",
      "episode 1252 score 66.6 average_score 10.6\n",
      "episode 1253 score -114.3 average_score 9.3\n",
      "episode 1254 score 125.8 average_score 9.6\n",
      "episode 1255 score 74.0 average_score 9.6\n",
      "episode 1256 score 99.4 average_score 10.0\n",
      "episode 1257 score -46.1 average_score 9.8\n",
      "episode 1258 score 74.1 average_score 9.9\n",
      "episode 1259 score 28.5 average_score 9.2\n",
      "episode 1260 score -15.5 average_score 8.4\n",
      "episode 1261 score 32.8 average_score 9.1\n",
      "episode 1262 score 41.8 average_score 10.9\n",
      "episode 1263 score -20.5 average_score 9.5\n",
      "episode 1264 score 126.3 average_score 10.0\n",
      "episode 1265 score -52.0 average_score 9.4\n",
      "episode 1266 score -41.4 average_score 8.9\n",
      "episode 1267 score 77.1 average_score 8.5\n",
      "episode 1268 score 118.5 average_score 9.7\n",
      "episode 1269 score 132.7 average_score 11.3\n",
      "episode 1270 score -6.0 average_score 10.4\n",
      "episode 1271 score 87.0 average_score 10.2\n",
      "episode 1272 score -55.3 average_score 11.5\n",
      "episode 1273 score -0.5 average_score 10.6\n",
      "episode 1274 score 18.9 average_score 10.9\n",
      "episode 1275 score -21.9 average_score 10.7\n",
      "episode 1276 score -17.4 average_score 10.1\n",
      "episode 1277 score -70.6 average_score 8.9\n",
      "episode 1278 score 60.4 average_score 9.1\n",
      "episode 1279 score -132.0 average_score 7.8\n",
      "episode 1280 score -99.1 average_score 7.0\n",
      "episode 1281 score -106.0 average_score 6.4\n",
      "episode 1282 score -19.9 average_score 5.9\n",
      "episode 1283 score 96.9 average_score 6.7\n",
      "episode 1284 score -14.3 average_score 6.5\n",
      "episode 1285 score 39.5 average_score 7.1\n",
      "episode 1286 score 122.8 average_score 7.9\n",
      "episode 1287 score 56.6 average_score 7.1\n",
      "episode 1288 score -79.7 average_score 6.5\n",
      "episode 1289 score -16.2 average_score 6.5\n",
      "episode 1290 score -58.7 average_score 6.0\n",
      "episode 1291 score -58.6 average_score 5.4\n",
      "episode 1292 score -105.4 average_score 4.4\n",
      "episode 1293 score 127.8 average_score 5.2\n",
      "episode 1294 score 100.4 average_score 6.5\n",
      "episode 1295 score 100.9 average_score 7.4\n",
      "episode 1296 score 100.9 average_score 8.6\n",
      "episode 1297 score -34.4 average_score 8.1\n",
      "episode 1298 score -142.9 average_score 6.4\n",
      "episode 1299 score 35.5 average_score 7.4\n",
      "episode 1300 score 38.4 average_score 8.1\n",
      "episode 1301 score -13.0 average_score 8.1\n",
      "episode 1302 score 25.6 average_score 7.5\n",
      "episode 1303 score 7.3 average_score 7.8\n",
      "episode 1304 score 7.0 average_score 8.8\n",
      "episode 1305 score -9.0 average_score 8.6\n",
      "episode 1306 score -32.2 average_score 8.7\n",
      "episode 1307 score -0.2 average_score 7.3\n",
      "episode 1308 score 12.2 average_score 7.4\n",
      "episode 1309 score 35.8 average_score 8.4\n",
      "episode 1310 score 82.8 average_score 8.8\n",
      "episode 1311 score 32.2 average_score 9.1\n",
      "episode 1312 score -0.9 average_score 8.9\n",
      "episode 1313 score -7.7 average_score 9.4\n",
      "episode 1314 score 26.9 average_score 9.8\n",
      "episode 1315 score -18.3 average_score 10.0\n",
      "episode 1316 score -14.3 average_score 10.7\n",
      "episode 1317 score 56.7 average_score 11.8\n",
      "episode 1318 score -17.0 average_score 11.1\n",
      "episode 1319 score 27.4 average_score 11.0\n",
      "episode 1320 score -3.6 average_score 12.5\n",
      "episode 1321 score 115.0 average_score 14.7\n",
      "episode 1322 score 16.2 average_score 14.4\n",
      "episode 1323 score -2.3 average_score 13.9\n",
      "episode 1324 score 30.9 average_score 13.0\n",
      "episode 1325 score -9.8 average_score 13.3\n",
      "episode 1326 score 8.0 average_score 13.6\n",
      "episode 1327 score 22.4 average_score 13.8\n",
      "episode 1328 score 33.5 average_score 13.7\n",
      "episode 1329 score 77.1 average_score 14.9\n",
      "episode 1330 score 24.2 average_score 13.9\n",
      "episode 1331 score 160.5 average_score 15.2\n",
      "episode 1332 score 14.5 average_score 15.5\n",
      "episode 1333 score 5.6 average_score 14.9\n",
      "episode 1334 score 76.9 average_score 16.5\n",
      "episode 1335 score 33.3 average_score 16.9\n",
      "episode 1336 score 14.5 average_score 16.8\n",
      "episode 1337 score 29.0 average_score 17.1\n",
      "episode 1338 score 43.8 average_score 17.7\n",
      "episode 1339 score 141.0 average_score 19.2\n",
      "episode 1340 score -16.4 average_score 19.0\n",
      "episode 1341 score 44.8 average_score 18.2\n",
      "episode 1342 score -4.2 average_score 17.8\n",
      "episode 1343 score -5.2 average_score 18.5\n",
      "episode 1344 score 13.7 average_score 18.6\n",
      "episode 1345 score 211.0 average_score 23.2\n",
      "episode 1346 score -24.2 average_score 21.9\n",
      "episode 1347 score 36.2 average_score 22.0\n",
      "episode 1348 score 153.5 average_score 22.1\n",
      "episode 1349 score 23.5 average_score 22.8\n",
      "episode 1350 score -42.3 average_score 21.1\n",
      "episode 1351 score 18.3 average_score 21.0\n",
      "episode 1352 score 117.2 average_score 21.5\n",
      "episode 1353 score 15.8 average_score 22.8\n",
      "episode 1354 score 11.5 average_score 21.6\n",
      "episode 1355 score 48.3 average_score 21.4\n",
      "episode 1356 score 1.4 average_score 20.4\n",
      "episode 1357 score 53.1 average_score 21.4\n",
      "episode 1358 score -4.6 average_score 20.6\n",
      "episode 1359 score 64.7 average_score 20.9\n",
      "episode 1360 score 15.0 average_score 21.3\n",
      "episode 1361 score 45.9 average_score 21.4\n",
      "episode 1362 score 64.1 average_score 21.6\n",
      "episode 1363 score 39.1 average_score 22.2\n",
      "episode 1364 score 141.9 average_score 22.4\n",
      "episode 1365 score -18.0 average_score 22.7\n",
      "episode 1366 score 21.9 average_score 23.3\n",
      "episode 1367 score 2.4 average_score 22.6\n",
      "episode 1368 score 18.1 average_score 21.6\n",
      "episode 1369 score 27.9 average_score 20.5\n",
      "episode 1370 score -10.6 average_score 20.5\n",
      "episode 1371 score -12.3 average_score 19.5\n",
      "episode 1372 score -2.9 average_score 20.0\n",
      "episode 1373 score 26.0 average_score 20.3\n",
      "episode 1374 score -44.1 average_score 19.7\n",
      "episode 1375 score 6.2 average_score 19.9\n",
      "episode 1376 score -86.9 average_score 19.2\n",
      "episode 1377 score 97.2 average_score 20.9\n",
      "episode 1378 score 2.7 average_score 20.3\n",
      "episode 1379 score -7.5 average_score 21.6\n",
      "episode 1380 score -95.2 average_score 21.6\n",
      "episode 1381 score -9.0 average_score 22.6\n",
      "episode 1382 score 37.0 average_score 23.2\n",
      "episode 1383 score 5.5 average_score 22.3\n",
      "episode 1384 score 35.4 average_score 22.7\n",
      "episode 1385 score 20.9 average_score 22.6\n",
      "episode 1386 score -2.2 average_score 21.3\n",
      "episode 1387 score -113.7 average_score 19.6\n",
      "episode 1388 score 27.1 average_score 20.7\n",
      "episode 1389 score -0.7 average_score 20.8\n",
      "episode 1390 score 27.2 average_score 21.7\n",
      "episode 1391 score 19.1 average_score 22.5\n",
      "episode 1392 score 38.3 average_score 23.9\n",
      "episode 1393 score 176.3 average_score 24.4\n",
      "episode 1394 score 54.3 average_score 23.9\n",
      "episode 1395 score 5.2 average_score 23.0\n",
      "episode 1396 score 23.3 average_score 22.2\n",
      "episode 1397 score 20.3 average_score 22.7\n",
      "episode 1398 score -61.1 average_score 23.6\n",
      "episode 1399 score -13.7 average_score 23.1\n",
      "episode 1400 score 23.4 average_score 22.9\n",
      "episode 1401 score -33.4 average_score 22.7\n",
      "episode 1402 score 21.5 average_score 22.7\n",
      "episode 1403 score 2.4 average_score 22.6\n",
      "episode 1404 score -12.9 average_score 22.4\n",
      "episode 1405 score 3.7 average_score 22.6\n",
      "episode 1406 score 163.7 average_score 24.5\n",
      "episode 1407 score 4.3 average_score 24.6\n",
      "episode 1408 score 31.7 average_score 24.8\n",
      "episode 1409 score -7.5 average_score 24.3\n",
      "episode 1410 score 136.6 average_score 24.9\n",
      "episode 1411 score -17.3 average_score 24.4\n",
      "episode 1412 score 22.2 average_score 24.6\n",
      "episode 1413 score 40.6 average_score 25.1\n",
      "episode 1414 score 146.1 average_score 26.3\n",
      "episode 1415 score 11.3 average_score 26.6\n",
      "episode 1416 score 136.1 average_score 28.1\n",
      "episode 1417 score -13.8 average_score 27.4\n",
      "episode 1418 score 13.1 average_score 27.7\n",
      "episode 1419 score 32.2 average_score 27.7\n",
      "episode 1420 score 105.4 average_score 28.8\n",
      "episode 1421 score -53.6 average_score 27.1\n",
      "episode 1422 score 24.3 average_score 27.2\n",
      "episode 1423 score 161.4 average_score 28.8\n",
      "episode 1424 score 34.1 average_score 28.9\n",
      "episode 1425 score 0.4 average_score 29.0\n",
      "episode 1426 score 6.9 average_score 29.0\n",
      "episode 1427 score 20.2 average_score 28.9\n",
      "episode 1428 score 14.5 average_score 28.7\n",
      "episode 1429 score 1.0 average_score 28.0\n",
      "episode 1430 score -37.3 average_score 27.4\n",
      "episode 1431 score -49.3 average_score 25.3\n",
      "episode 1432 score -22.3 average_score 24.9\n",
      "episode 1433 score 147.5 average_score 26.3\n",
      "episode 1434 score 29.1 average_score 25.8\n",
      "episode 1435 score 9.2 average_score 25.6\n",
      "episode 1436 score 152.9 average_score 27.0\n",
      "episode 1437 score -21.6 average_score 26.5\n",
      "episode 1438 score 72.1 average_score 26.8\n",
      "episode 1439 score -206.5 average_score 23.3\n",
      "episode 1440 score 0.7 average_score 23.5\n",
      "episode 1441 score -32.7 average_score 22.7\n",
      "episode 1442 score 35.5 average_score 23.1\n",
      "episode 1443 score 35.0 average_score 23.5\n",
      "episode 1444 score 174.2 average_score 25.1\n",
      "episode 1445 score 15.3 average_score 23.1\n",
      "episode 1446 score -12.0 average_score 23.3\n",
      "episode 1447 score 263.3 average_score 25.5\n",
      "episode 1448 score 55.4 average_score 24.5\n",
      "episode 1449 score -27.2 average_score 24.0\n",
      "episode 1450 score -31.3 average_score 24.2\n",
      "episode 1451 score 39.6 average_score 24.4\n",
      "episode 1452 score 37.9 average_score 23.6\n",
      "episode 1453 score -20.2 average_score 23.2\n",
      "episode 1454 score 101.4 average_score 24.1\n",
      "episode 1455 score -8.1 average_score 23.5\n",
      "episode 1456 score 142.3 average_score 25.0\n",
      "episode 1457 score 78.3 average_score 25.2\n",
      "episode 1458 score 47.4 average_score 25.7\n",
      "episode 1459 score 104.3 average_score 26.1\n",
      "episode 1460 score -58.1 average_score 25.4\n",
      "episode 1461 score 180.4 average_score 26.7\n",
      "episode 1462 score -20.1 average_score 25.9\n",
      "episode 1463 score -18.5 average_score 25.3\n",
      "episode 1464 score -12.7 average_score 23.8\n",
      "episode 1465 score 153.1 average_score 25.5\n",
      "episode 1466 score 8.1 average_score 25.3\n",
      "episode 1467 score 29.5 average_score 25.6\n",
      "episode 1468 score 13.8 average_score 25.6\n",
      "episode 1469 score -48.1 average_score 24.8\n",
      "episode 1470 score -27.4 average_score 24.6\n",
      "episode 1471 score -27.5 average_score 24.5\n",
      "episode 1472 score -91.8 average_score 23.6\n",
      "episode 1473 score 26.4 average_score 23.6\n",
      "episode 1474 score 78.3 average_score 24.8\n",
      "episode 1475 score 115.1 average_score 25.9\n",
      "episode 1476 score -27.0 average_score 26.5\n",
      "episode 1477 score 103.3 average_score 26.6\n",
      "episode 1478 score -46.6 average_score 26.1\n",
      "episode 1479 score -111.9 average_score 25.0\n",
      "episode 1480 score 153.6 average_score 27.5\n",
      "episode 1481 score 22.0 average_score 27.8\n",
      "episode 1482 score 87.5 average_score 28.3\n",
      "episode 1483 score 35.6 average_score 28.6\n",
      "episode 1484 score -8.3 average_score 28.2\n",
      "episode 1485 score 30.7 average_score 28.3\n",
      "episode 1486 score -1.8 average_score 28.3\n",
      "episode 1487 score 19.1 average_score 29.6\n",
      "episode 1488 score 147.7 average_score 30.8\n",
      "episode 1489 score 3.5 average_score 30.9\n",
      "episode 1490 score -10.7 average_score 30.5\n",
      "episode 1491 score 100.6 average_score 31.3\n",
      "episode 1492 score 135.1 average_score 32.3\n",
      "episode 1493 score 44.4 average_score 31.0\n",
      "episode 1494 score 105.6 average_score 31.5\n",
      "episode 1495 score 23.1 average_score 31.7\n",
      "episode 1496 score 141.2 average_score 32.8\n",
      "episode 1497 score 151.4 average_score 34.2\n",
      "episode 1498 score 46.3 average_score 35.2\n",
      "episode 1499 score -16.8 average_score 35.2\n",
      "episode 1500 score 63.9 average_score 35.6\n",
      "episode 1501 score 24.3 average_score 36.2\n",
      "episode 1502 score -67.3 average_score 35.3\n",
      "episode 1503 score 31.4 average_score 35.6\n",
      "episode 1504 score -12.9 average_score 35.6\n",
      "episode 1505 score 2.3 average_score 35.6\n",
      "episode 1506 score 2.8 average_score 34.0\n",
      "episode 1507 score 9.3 average_score 34.0\n",
      "episode 1508 score -9.4 average_score 33.6\n",
      "episode 1509 score -5.5 average_score 33.6\n",
      "episode 1510 score 264.3 average_score 34.9\n",
      "episode 1511 score 2.6 average_score 35.1\n",
      "episode 1512 score -87.9 average_score 34.0\n",
      "episode 1513 score 17.1 average_score 33.8\n",
      "episode 1514 score 158.0 average_score 33.9\n",
      "episode 1515 score 17.9 average_score 33.9\n",
      "episode 1516 score 266.1 average_score 35.2\n",
      "episode 1517 score -6.4 average_score 35.3\n",
      "episode 1518 score 26.1 average_score 35.4\n",
      "episode 1519 score 253.3 average_score 37.7\n",
      "episode 1520 score 23.2 average_score 36.8\n",
      "episode 1521 score 33.2 average_score 37.7\n",
      "episode 1522 score 35.6 average_score 37.8\n",
      "episode 1523 score 167.5 average_score 37.9\n",
      "episode 1524 score 0.9 average_score 37.5\n",
      "episode 1525 score 250.5 average_score 40.0\n",
      "episode 1526 score 227.3 average_score 42.2\n",
      "episode 1527 score 6.5 average_score 42.1\n",
      "episode 1528 score 38.8 average_score 42.4\n",
      "episode 1529 score -10.4 average_score 42.2\n",
      "episode 1530 score 6.3 average_score 42.7\n",
      "episode 1531 score 150.9 average_score 44.7\n",
      "episode 1532 score -6.2 average_score 44.8\n",
      "episode 1533 score 18.8 average_score 43.6\n",
      "episode 1534 score 203.2 average_score 45.3\n",
      "episode 1535 score 243.1 average_score 47.6\n",
      "episode 1536 score 278.9 average_score 48.9\n",
      "episode 1537 score 263.0 average_score 51.7\n",
      "episode 1538 score 259.6 average_score 53.6\n",
      "episode 1539 score 60.3 average_score 56.3\n",
      "episode 1540 score -31.3 average_score 56.0\n",
      "episode 1541 score 29.6 average_score 56.6\n",
      "episode 1542 score 40.7 average_score 56.6\n",
      "episode 1543 score -17.8 average_score 56.1\n",
      "episode 1544 score -179.5 average_score 52.6\n",
      "episode 1545 score 41.0 average_score 52.8\n",
      "episode 1546 score -167.4 average_score 51.3\n",
      "episode 1547 score -92.9 average_score 47.7\n",
      "episode 1548 score 220.5 average_score 49.4\n",
      "episode 1549 score -36.2 average_score 49.3\n",
      "episode 1550 score 17.5 average_score 49.8\n",
      "episode 1551 score 252.2 average_score 51.9\n",
      "episode 1552 score 230.3 average_score 53.8\n",
      "episode 1553 score 297.9 average_score 57.0\n",
      "episode 1554 score 14.9 average_score 56.1\n",
      "episode 1555 score 255.2 average_score 58.8\n",
      "episode 1556 score 292.9 average_score 60.3\n",
      "episode 1557 score 260.9 average_score 62.1\n",
      "episode 1558 score -4.9 average_score 61.6\n",
      "episode 1559 score -185.0 average_score 58.7\n",
      "episode 1560 score 203.2 average_score 61.3\n",
      "episode 1561 score -47.0 average_score 59.0\n",
      "episode 1562 score 273.4 average_score 62.0\n",
      "episode 1563 score 37.2 average_score 62.5\n",
      "episode 1564 score 243.3 average_score 65.1\n",
      "episode 1565 score -11.9 average_score 63.4\n",
      "episode 1566 score 273.6 average_score 66.1\n",
      "episode 1567 score -7.3 average_score 65.7\n",
      "episode 1568 score 38.5 average_score 66.0\n",
      "episode 1569 score 75.2 average_score 67.2\n",
      "episode 1570 score 303.3 average_score 70.5\n",
      "episode 1571 score -23.5 average_score 70.5\n",
      "episode 1572 score 247.1 average_score 73.9\n",
      "episode 1573 score -3.1 average_score 73.6\n",
      "episode 1574 score -41.7 average_score 72.4\n",
      "episode 1575 score 20.2 average_score 71.5\n",
      "episode 1576 score -6.2 average_score 71.7\n",
      "episode 1577 score 268.8 average_score 73.3\n",
      "episode 1578 score 282.4 average_score 76.6\n",
      "episode 1579 score -51.3 average_score 77.2\n",
      "episode 1580 score -42.1 average_score 75.3\n",
      "episode 1581 score 263.7 average_score 77.7\n",
      "episode 1582 score -15.5 average_score 76.7\n",
      "episode 1583 score -51.8 average_score 75.8\n",
      "episode 1584 score 13.4 average_score 76.0\n",
      "episode 1585 score 178.0 average_score 77.5\n",
      "episode 1586 score 202.7 average_score 79.5\n",
      "episode 1587 score 14.1 average_score 79.5\n",
      "episode 1588 score -78.5 average_score 77.2\n",
      "episode 1589 score -72.7 average_score 76.5\n",
      "episode 1590 score -43.1 average_score 76.1\n",
      "episode 1591 score 195.1 average_score 77.1\n",
      "episode 1592 score -133.6 average_score 74.4\n",
      "episode 1593 score 17.4 average_score 74.1\n",
      "episode 1594 score 13.8 average_score 73.2\n",
      "episode 1595 score 23.0 average_score 73.2\n",
      "episode 1596 score 191.7 average_score 73.7\n",
      "episode 1597 score -11.5 average_score 72.1\n",
      "episode 1598 score 28.8 average_score 71.9\n",
      "episode 1599 score 252.4 average_score 74.6\n",
      "episode 1600 score 207.8 average_score 76.0\n",
      "episode 1601 score -21.4 average_score 75.6\n",
      "episode 1602 score 27.2 average_score 76.5\n",
      "episode 1603 score 186.6 average_score 78.1\n",
      "episode 1604 score 37.2 average_score 78.6\n",
      "episode 1605 score 19.1 average_score 78.7\n",
      "episode 1606 score -8.6 average_score 78.6\n",
      "episode 1607 score -53.1 average_score 78.0\n",
      "episode 1608 score 271.7 average_score 80.8\n",
      "episode 1609 score -137.2 average_score 79.5\n",
      "episode 1610 score 14.8 average_score 77.0\n",
      "episode 1611 score 237.9 average_score 79.4\n",
      "episode 1612 score 36.0 average_score 80.6\n",
      "episode 1613 score 4.5 average_score 80.5\n",
      "episode 1614 score 214.8 average_score 81.0\n",
      "episode 1615 score -18.4 average_score 80.7\n",
      "episode 1616 score 31.7 average_score 78.3\n",
      "episode 1617 score -21.2 average_score 78.2\n",
      "episode 1618 score 40.4 average_score 78.3\n",
      "episode 1619 score -55.8 average_score 75.2\n",
      "episode 1620 score -15.3 average_score 74.8\n",
      "episode 1621 score -9.6 average_score 74.4\n",
      "episode 1622 score -60.6 average_score 73.5\n",
      "episode 1623 score 0.8 average_score 71.8\n",
      "episode 1624 score 255.4 average_score 74.3\n",
      "episode 1625 score 29.3 average_score 72.1\n",
      "episode 1626 score 199.2 average_score 71.8\n",
      "episode 1627 score -7.4 average_score 71.7\n",
      "episode 1628 score 260.6 average_score 73.9\n",
      "episode 1629 score -134.3 average_score 72.7\n",
      "episode 1630 score 14.8 average_score 72.8\n",
      "episode 1631 score 235.1 average_score 73.6\n",
      "episode 1632 score 29.8 average_score 74.0\n",
      "episode 1633 score -54.5 average_score 73.2\n",
      "episode 1634 score 27.1 average_score 71.5\n",
      "episode 1635 score 27.4 average_score 69.3\n",
      "episode 1636 score 267.1 average_score 69.2\n",
      "episode 1637 score 222.4 average_score 68.8\n",
      "episode 1638 score 21.6 average_score 66.4\n",
      "episode 1639 score 31.2 average_score 66.1\n",
      "episode 1640 score 221.5 average_score 68.7\n",
      "episode 1641 score 239.7 average_score 70.8\n",
      "episode 1642 score 281.7 average_score 73.2\n",
      "episode 1643 score 292.2 average_score 76.3\n",
      "episode 1644 score 254.3 average_score 80.6\n",
      "episode 1645 score 273.7 average_score 82.9\n",
      "episode 1646 score 218.8 average_score 86.8\n",
      "episode 1647 score -34.7 average_score 87.4\n",
      "episode 1648 score -1.3 average_score 85.2\n",
      "episode 1649 score 2.9 average_score 85.5\n",
      "episode 1650 score 20.7 average_score 85.6\n",
      "episode 1651 score 52.8 average_score 83.6\n",
      "episode 1652 score -111.9 average_score 80.2\n",
      "episode 1653 score 207.7 average_score 79.3\n",
      "episode 1654 score 30.7 average_score 79.4\n",
      "episode 1655 score 274.4 average_score 79.6\n",
      "episode 1656 score 18.4 average_score 76.9\n",
      "episode 1657 score 231.5 average_score 76.6\n",
      "episode 1658 score -51.2 average_score 76.1\n",
      "episode 1659 score 32.5 average_score 78.3\n",
      "episode 1660 score 244.8 average_score 78.7\n",
      "episode 1661 score 240.9 average_score 81.6\n",
      "episode 1662 score -66.9 average_score 78.2\n",
      "episode 1663 score 289.6 average_score 80.7\n",
      "episode 1664 score 242.3 average_score 80.7\n",
      "episode 1665 score 284.5 average_score 83.6\n",
      "episode 1666 score 262.0 average_score 83.5\n",
      "episode 1667 score 57.4 average_score 84.2\n",
      "episode 1668 score 273.9 average_score 86.5\n",
      "episode 1669 score -45.5 average_score 85.3\n",
      "episode 1670 score 270.1 average_score 85.0\n",
      "episode 1671 score 251.7 average_score 87.7\n",
      "episode 1672 score -105.5 average_score 84.2\n",
      "episode 1673 score 4.0 average_score 84.3\n",
      "episode 1674 score -13.5 average_score 84.6\n",
      "episode 1675 score 277.8 average_score 87.1\n",
      "episode 1676 score -63.9 average_score 86.6\n",
      "episode 1677 score 205.7 average_score 85.9\n",
      "episode 1678 score 266.1 average_score 85.8\n",
      "episode 1679 score 244.4 average_score 88.7\n",
      "episode 1680 score 272.9 average_score 91.9\n",
      "episode 1681 score -17.6 average_score 89.1\n",
      "episode 1682 score 245.6 average_score 91.7\n",
      "episode 1683 score 2.0 average_score 92.2\n",
      "episode 1684 score -60.0 average_score 91.5\n",
      "episode 1685 score 10.7 average_score 89.8\n",
      "episode 1686 score 249.4 average_score 90.3\n",
      "episode 1687 score 15.0 average_score 90.3\n",
      "episode 1688 score 30.0 average_score 91.4\n",
      "episode 1689 score 22.5 average_score 92.3\n",
      "episode 1690 score 215.3 average_score 94.9\n",
      "episode 1691 score 241.4 average_score 95.4\n",
      "episode 1692 score -61.7 average_score 96.1\n",
      "episode 1693 score 254.3 average_score 98.5\n",
      "episode 1694 score 243.2 average_score 100.8\n",
      "episode 1695 score 47.2 average_score 101.0\n",
      "episode 1696 score 45.9 average_score 99.5\n",
      "episode 1697 score -51.0 average_score 99.1\n",
      "episode 1698 score 42.0 average_score 99.3\n",
      "episode 1699 score 5.4 average_score 96.8\n",
      "episode 1700 score 73.4 average_score 95.5\n",
      "episode 1701 score 23.7 average_score 95.9\n",
      "episode 1702 score 290.0 average_score 98.5\n",
      "episode 1703 score 18.6 average_score 96.9\n",
      "episode 1704 score 279.1 average_score 99.3\n",
      "episode 1705 score 18.8 average_score 99.3\n",
      "episode 1706 score -14.9 average_score 99.2\n",
      "episode 1707 score 275.4 average_score 102.5\n",
      "episode 1708 score -15.8 average_score 99.6\n",
      "episode 1709 score -21.3 average_score 100.8\n",
      "episode 1710 score 9.3 average_score 100.7\n",
      "episode 1711 score 263.6 average_score 101.0\n",
      "episode 1712 score 287.5 average_score 103.5\n",
      "episode 1713 score 32.2 average_score 103.8\n",
      "episode 1714 score -7.1 average_score 101.6\n",
      "episode 1715 score -15.2 average_score 101.6\n",
      "episode 1716 score 3.2 average_score 101.3\n",
      "episode 1717 score -12.6 average_score 101.4\n",
      "episode 1718 score 264.1 average_score 103.6\n",
      "episode 1719 score 267.5 average_score 106.9\n",
      "episode 1720 score -62.8 average_score 106.4\n",
      "episode 1721 score -3.6 average_score 106.5\n",
      "episode 1722 score 10.9 average_score 107.2\n",
      "episode 1723 score 269.0 average_score 109.9\n",
      "episode 1724 score -50.2 average_score 106.8\n",
      "episode 1725 score 270.1 average_score 109.2\n",
      "episode 1726 score -161.6 average_score 105.6\n",
      "episode 1727 score -3.5 average_score 105.6\n",
      "episode 1728 score -47.6 average_score 102.5\n",
      "episode 1729 score -19.7 average_score 103.7\n",
      "episode 1730 score 32.9 average_score 103.9\n",
      "episode 1731 score 55.2 average_score 102.1\n",
      "episode 1732 score 8.8 average_score 101.9\n",
      "episode 1733 score 248.4 average_score 104.9\n",
      "episode 1734 score -48.4 average_score 104.1\n",
      "episode 1735 score 132.4 average_score 105.2\n",
      "episode 1736 score 238.9 average_score 104.9\n",
      "episode 1737 score 290.0 average_score 105.6\n",
      "episode 1738 score -141.5 average_score 104.0\n",
      "episode 1739 score -40.1 average_score 103.2\n",
      "episode 1740 score -29.4 average_score 100.7\n",
      "episode 1741 score -88.7 average_score 97.4\n",
      "episode 1742 score 16.9 average_score 94.8\n",
      "episode 1743 score -8.4 average_score 91.8\n",
      "episode 1744 score -51.8 average_score 88.7\n",
      "episode 1745 score 262.0 average_score 88.6\n",
      "episode 1746 score 210.0 average_score 88.5\n",
      "episode 1747 score 7.1 average_score 88.9\n",
      "episode 1748 score -183.3 average_score 87.1\n",
      "episode 1749 score 217.5 average_score 89.3\n",
      "episode 1750 score -47.0 average_score 88.6\n",
      "episode 1751 score -4.7 average_score 88.0\n",
      "episode 1752 score -22.5 average_score 88.9\n",
      "episode 1753 score 244.4 average_score 89.3\n",
      "episode 1754 score 173.5 average_score 90.7\n",
      "episode 1755 score -42.1 average_score 87.5\n",
      "episode 1756 score 267.9 average_score 90.0\n",
      "episode 1757 score 250.1 average_score 90.2\n",
      "episode 1758 score -7.3 average_score 90.7\n",
      "episode 1759 score 5.4 average_score 90.4\n",
      "episode 1760 score 12.2 average_score 88.1\n",
      "episode 1761 score -68.2 average_score 85.0\n",
      "episode 1762 score -9.2 average_score 85.6\n",
      "episode 1763 score 13.2 average_score 82.8\n",
      "episode 1764 score 215.6 average_score 82.5\n",
      "episode 1765 score 259.1 average_score 82.3\n",
      "episode 1766 score -33.9 average_score 79.3\n",
      "episode 1767 score 259.4 average_score 81.3\n",
      "episode 1768 score 179.8 average_score 80.4\n",
      "episode 1769 score -6.5 average_score 80.8\n",
      "episode 1770 score 171.8 average_score 79.8\n",
      "episode 1771 score -102.2 average_score 76.3\n",
      "episode 1772 score 249.4 average_score 79.8\n",
      "episode 1773 score -24.1 average_score 79.5\n",
      "episode 1774 score -10.5 average_score 79.6\n",
      "episode 1775 score 210.8 average_score 78.9\n",
      "episode 1776 score 208.9 average_score 81.6\n",
      "episode 1777 score 7.6 average_score 79.6\n",
      "episode 1778 score -57.3 average_score 76.4\n",
      "episode 1779 score 286.7 average_score 76.8\n",
      "episode 1780 score 23.5 average_score 74.3\n",
      "episode 1781 score -18.1 average_score 74.3\n",
      "episode 1782 score -5.8 average_score 71.8\n",
      "episode 1783 score -42.2 average_score 71.4\n",
      "episode 1784 score 284.3 average_score 74.8\n",
      "episode 1785 score 278.4 average_score 77.5\n",
      "episode 1786 score 252.5 average_score 77.5\n",
      "episode 1787 score 41.7 average_score 77.8\n",
      "episode 1788 score 261.4 average_score 80.1\n",
      "episode 1789 score 24.9 average_score 80.1\n",
      "episode 1790 score -24.6 average_score 77.7\n",
      "episode 1791 score -13.2 average_score 75.2\n",
      "episode 1792 score 184.1 average_score 77.6\n",
      "episode 1793 score -21.3 average_score 74.9\n",
      "episode 1794 score 241.7 average_score 74.9\n",
      "episode 1795 score 222.0 average_score 76.6\n",
      "episode 1796 score 54.8 average_score 76.7\n",
      "episode 1797 score -34.2 average_score 76.9\n",
      "episode 1798 score -80.7 average_score 75.6\n",
      "episode 1799 score -92.1 average_score 74.7\n",
      "episode 1800 score -47.5 average_score 73.5\n",
      "episode 1801 score 210.6 average_score 75.3\n",
      "episode 1802 score -42.3 average_score 72.0\n",
      "episode 1803 score -9.1 average_score 71.7\n",
      "episode 1804 score -24.6 average_score 68.7\n",
      "episode 1805 score 226.1 average_score 70.8\n",
      "episode 1806 score 288.7 average_score 73.8\n",
      "episode 1807 score 196.9 average_score 73.0\n",
      "episode 1808 score -9.5 average_score 73.1\n",
      "episode 1809 score 8.9 average_score 73.4\n",
      "episode 1810 score 4.5 average_score 73.3\n",
      "episode 1811 score 24.2 average_score 70.9\n",
      "episode 1812 score -1.7 average_score 68.0\n",
      "episode 1813 score 233.2 average_score 70.1\n",
      "episode 1814 score 14.2 average_score 70.3\n",
      "episode 1815 score 237.6 average_score 72.8\n",
      "episode 1816 score -0.4 average_score 72.8\n",
      "episode 1817 score -89.5 average_score 72.0\n",
      "episode 1818 score -34.5 average_score 69.0\n",
      "episode 1819 score -10.7 average_score 66.2\n",
      "episode 1820 score 266.7 average_score 69.5\n",
      "episode 1821 score -55.2 average_score 69.0\n",
      "episode 1822 score 256.4 average_score 71.5\n",
      "episode 1823 score -0.2 average_score 68.8\n",
      "episode 1824 score -6.2 average_score 69.2\n",
      "episode 1825 score 251.8 average_score 69.0\n",
      "episode 1826 score 262.5 average_score 73.3\n",
      "episode 1827 score 300.4 average_score 76.3\n",
      "episode 1828 score 15.0 average_score 76.9\n",
      "episode 1829 score 207.9 average_score 79.2\n",
      "episode 1830 score 255.5 average_score 81.4\n",
      "episode 1831 score -152.6 average_score 79.3\n",
      "episode 1832 score -12.2 average_score 79.1\n",
      "episode 1833 score 255.2 average_score 79.2\n",
      "episode 1834 score 267.1 average_score 82.4\n",
      "episode 1835 score 248.2 average_score 83.5\n",
      "episode 1836 score 8.2 average_score 81.2\n",
      "episode 1837 score 201.5 average_score 80.3\n",
      "episode 1838 score 237.1 average_score 84.1\n",
      "episode 1839 score 216.1 average_score 86.7\n",
      "episode 1840 score 279.8 average_score 89.8\n",
      "episode 1841 score 172.3 average_score 92.4\n",
      "episode 1842 score 48.4 average_score 92.7\n",
      "episode 1843 score 233.5 average_score 95.1\n",
      "episode 1844 score -58.7 average_score 95.0\n",
      "episode 1845 score 253.2 average_score 95.0\n",
      "episode 1846 score 260.7 average_score 95.5\n",
      "episode 1847 score 145.9 average_score 96.8\n",
      "episode 1848 score 246.7 average_score 101.1\n",
      "episode 1849 score 301.4 average_score 102.0\n",
      "episode 1850 score 301.4 average_score 105.5\n",
      "episode 1851 score 244.7 average_score 108.0\n",
      "episode 1852 score -17.7 average_score 108.0\n",
      "episode 1853 score 53.4 average_score 106.1\n",
      "episode 1854 score -103.1 average_score 103.3\n",
      "episode 1855 score 274.5 average_score 106.5\n",
      "episode 1856 score 263.6 average_score 106.5\n",
      "episode 1857 score -7.1 average_score 103.9\n",
      "episode 1858 score 21.0 average_score 104.2\n",
      "episode 1859 score 260.6 average_score 106.7\n",
      "episode 1860 score -42.1 average_score 106.2\n",
      "episode 1861 score 304.4 average_score 109.9\n",
      "episode 1862 score 303.7 average_score 113.0\n",
      "episode 1863 score 28.0 average_score 113.2\n",
      "episode 1864 score 30.0 average_score 111.3\n",
      "episode 1865 score 261.4 average_score 111.4\n",
      "episode 1866 score 202.2 average_score 113.7\n",
      "episode 1867 score -50.8 average_score 110.6\n",
      "episode 1868 score -19.1 average_score 108.6\n",
      "episode 1869 score 211.5 average_score 110.8\n",
      "episode 1870 score -29.3 average_score 108.8\n",
      "episode 1871 score 268.2 average_score 112.5\n",
      "episode 1872 score 235.0 average_score 112.4\n",
      "episode 1873 score 277.0 average_score 115.4\n",
      "episode 1874 score 230.8 average_score 117.8\n",
      "episode 1875 score 244.5 average_score 118.1\n",
      "episode 1876 score 254.0 average_score 118.6\n",
      "episode 1877 score 261.8 average_score 121.1\n",
      "episode 1878 score 29.9 average_score 122.0\n",
      "episode 1879 score 261.1 average_score 121.7\n",
      "episode 1880 score 309.1 average_score 124.6\n",
      "episode 1881 score 45.7 average_score 125.2\n",
      "episode 1882 score -52.0 average_score 124.8\n",
      "episode 1883 score 18.7 average_score 125.4\n",
      "episode 1884 score 264.9 average_score 125.2\n",
      "episode 1885 score 44.1 average_score 122.8\n",
      "episode 1886 score 215.4 average_score 122.5\n",
      "episode 1887 score 180.4 average_score 123.8\n",
      "episode 1888 score -12.7 average_score 121.1\n",
      "episode 1889 score -71.0 average_score 120.1\n",
      "episode 1890 score 274.4 average_score 123.1\n",
      "episode 1891 score 252.8 average_score 125.8\n",
      "episode 1892 score 279.5 average_score 126.7\n",
      "episode 1893 score 233.6 average_score 129.3\n",
      "episode 1894 score 273.2 average_score 129.6\n",
      "episode 1895 score 12.4 average_score 127.5\n",
      "episode 1896 score 4.5 average_score 127.0\n",
      "episode 1897 score 166.8 average_score 129.0\n",
      "episode 1898 score 27.4 average_score 130.1\n",
      "episode 1899 score 237.8 average_score 133.4\n",
      "episode 1900 score -71.7 average_score 133.2\n",
      "episode 1901 score 24.1 average_score 131.3\n",
      "episode 1902 score 244.7 average_score 134.2\n",
      "episode 1903 score 247.9 average_score 136.7\n",
      "episode 1904 score 4.6 average_score 137.0\n",
      "episode 1905 score 260.6 average_score 137.4\n",
      "episode 1906 score 14.3 average_score 134.6\n",
      "episode 1907 score 266.2 average_score 135.3\n",
      "episode 1908 score 287.2 average_score 138.3\n",
      "episode 1909 score -13.1 average_score 138.1\n",
      "episode 1910 score 186.3 average_score 139.9\n",
      "episode 1911 score -2.3 average_score 139.6\n",
      "episode 1912 score 12.0 average_score 139.8\n",
      "episode 1913 score 19.1 average_score 137.6\n",
      "episode 1914 score 256.4 average_score 140.0\n",
      "episode 1915 score 20.7 average_score 137.9\n",
      "episode 1916 score 168.9 average_score 139.6\n",
      "episode 1917 score 230.0 average_score 142.8\n",
      "episode 1918 score -1.1 average_score 143.1\n",
      "episode 1919 score 237.0 average_score 145.6\n",
      "episode 1920 score 52.8 average_score 143.4\n",
      "episode 1921 score 222.5 average_score 146.2\n",
      "episode 1922 score 245.4 average_score 146.1\n",
      "episode 1923 score 12.8 average_score 146.2\n",
      "episode 1924 score 13.1 average_score 146.4\n",
      "episode 1925 score 22.0 average_score 144.1\n",
      "episode 1926 score 207.3 average_score 143.6\n",
      "episode 1927 score 295.6 average_score 143.5\n",
      "episode 1928 score -2.8 average_score 143.3\n",
      "episode 1929 score 55.4 average_score 141.8\n",
      "episode 1930 score 60.1 average_score 139.9\n",
      "episode 1931 score -8.9 average_score 141.3\n",
      "episode 1932 score 272.5 average_score 144.1\n",
      "episode 1933 score -29.0 average_score 141.3\n",
      "episode 1934 score 58.5 average_score 139.2\n",
      "episode 1935 score 14.8 average_score 136.9\n",
      "episode 1936 score -30.9 average_score 136.5\n",
      "episode 1937 score 255.7 average_score 137.0\n",
      "episode 1938 score 0.1 average_score 134.7\n",
      "episode 1939 score 226.0 average_score 134.8\n",
      "episode 1940 score 256.7 average_score 134.5\n",
      "episode 1941 score 20.4 average_score 133.0\n",
      "episode 1942 score 276.0 average_score 135.3\n",
      "episode 1943 score 236.4 average_score 135.3\n",
      "episode 1944 score 152.3 average_score 137.4\n",
      "episode 1945 score 58.4 average_score 135.5\n",
      "episode 1946 score 254.7 average_score 135.4\n",
      "episode 1947 score 16.2 average_score 134.1\n",
      "episode 1948 score 26.4 average_score 131.9\n",
      "episode 1949 score 11.9 average_score 129.0\n",
      "episode 1950 score -9.3 average_score 125.9\n",
      "episode 1951 score 58.4 average_score 124.1\n",
      "episode 1952 score 49.5 average_score 124.7\n",
      "episode 1953 score 274.8 average_score 126.9\n",
      "episode 1954 score 24.4 average_score 128.2\n",
      "episode 1955 score 16.7 average_score 125.6\n",
      "episode 1956 score 294.1 average_score 125.9\n",
      "episode 1957 score 277.1 average_score 128.8\n",
      "episode 1958 score 42.0 average_score 129.0\n",
      "episode 1959 score 291.2 average_score 129.3\n",
      "episode 1960 score 269.6 average_score 132.4\n",
      "episode 1961 score 257.7 average_score 132.0\n",
      "episode 1962 score 60.6 average_score 129.5\n",
      "episode 1963 score 161.1 average_score 130.9\n",
      "episode 1964 score 22.5 average_score 130.8\n",
      "episode 1965 score 22.0 average_score 128.4\n",
      "episode 1966 score 47.4 average_score 126.8\n",
      "episode 1967 score -38.8 average_score 127.0\n",
      "episode 1968 score 31.7 average_score 127.5\n",
      "episode 1969 score 274.8 average_score 128.1\n",
      "episode 1970 score 265.5 average_score 131.0\n",
      "episode 1971 score 34.1 average_score 128.7\n",
      "episode 1972 score -2.4 average_score 126.3\n",
      "episode 1973 score 220.9 average_score 125.8\n",
      "episode 1974 score 254.2 average_score 126.0\n",
      "episode 1975 score 272.5 average_score 126.3\n",
      "episode 1976 score 228.3 average_score 126.0\n",
      "episode 1977 score 286.5 average_score 126.3\n",
      "episode 1978 score 261.0 average_score 128.6\n",
      "episode 1979 score 240.5 average_score 128.4\n",
      "episode 1980 score 7.8 average_score 125.4\n",
      "episode 1981 score -7.9 average_score 124.8\n",
      "episode 1982 score 266.5 average_score 128.0\n",
      "episode 1983 score 231.7 average_score 130.1\n",
      "episode 1984 score 23.1 average_score 127.7\n",
      "episode 1985 score -127.0 average_score 126.0\n",
      "episode 1986 score 257.0 average_score 126.4\n",
      "episode 1987 score 248.7 average_score 127.1\n",
      "episode 1988 score 248.5 average_score 129.7\n",
      "episode 1989 score 34.1 average_score 130.8\n",
      "episode 1990 score -4.7 average_score 128.0\n",
      "episode 1991 score 256.1 average_score 128.0\n",
      "episode 1992 score 261.8 average_score 127.8\n",
      "episode 1993 score 203.9 average_score 127.5\n",
      "episode 1994 score 264.3 average_score 127.5\n",
      "episode 1995 score 45.5 average_score 127.8\n",
      "episode 1996 score 26.2 average_score 128.0\n",
      "episode 1997 score 40.6 average_score 126.7\n",
      "episode 1998 score 233.5 average_score 128.8\n",
      "episode 1999 score 237.6 average_score 128.8\n"
     ]
    }
   ],
   "source": [
    "run_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conclusion\n",
    "After assignment 6, this is my second attempt to apply the approach outlined the paper \"Reinforcement Learning for Taxi-out Time Prediction: An improved Q-learning Approach, to predict taxi-out time on the on-time dataset.\n",
    "\n",
    "In assignment 6, i used deep q-learning and in this assignment I tried PGM. both attempts did not yield significant results. I'm lead to believe that this dataset might not be the best candidate for the RL approach."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}