model_data_selection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Create mp3 list for genre classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('track_data_trimmed.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 147100 entries, 0 to 147099\n",
      "Data columns (total 9 columns):\n",
      "id                 147100 non-null int64\n",
      "release_artist     147100 non-null object\n",
      "release_cat_num    147097 non-null object\n",
      "release_date       147100 non-null object\n",
      "release_genre      147100 non-null object\n",
      "release_label      147100 non-null object\n",
      "release_title      147100 non-null object\n",
      "track_name         147100 non-null object\n",
      "track_url          147100 non-null object\n",
      "dtypes: int64(1), object(8)\n",
      "memory usage: 10.1+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Genre counts & map to parent genres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>release_genre</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Minimal/Tech House</td>\n",
       "      <td>29415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Progressive House</td>\n",
       "      <td>14515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Funky/Club House</td>\n",
       "      <td>13517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Deep House</td>\n",
       "      <td>13098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Techno</td>\n",
       "      <td>12394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Uplifting Trance</td>\n",
       "      <td>8606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Electro House</td>\n",
       "      <td>7169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Drum And Bass</td>\n",
       "      <td>6381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Dirty Dubstep/Trap/Grime</td>\n",
       "      <td>4164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Breakbeat</td>\n",
       "      <td>3829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Disco/Nu-Disco</td>\n",
       "      <td>2980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Balearic/Downtempo</td>\n",
       "      <td>2927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Euro Dance/Pop Dance</td>\n",
       "      <td>2643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Hip Hop/R&amp;B</td>\n",
       "      <td>2483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Hardstyle</td>\n",
       "      <td>2136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Psy/Goa Trance</td>\n",
       "      <td>2103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Dancehall/Ragga</td>\n",
       "      <td>1728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Hard Trance</td>\n",
       "      <td>1522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Indie</td>\n",
       "      <td>1369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>UK Hardcore</td>\n",
       "      <td>1322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Hard House</td>\n",
       "      <td>1239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Experimental/Electronic</td>\n",
       "      <td>1156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Pop Trance</td>\n",
       "      <td>1092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Bass</td>\n",
       "      <td>963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Broken Beat/Nu Jazz</td>\n",
       "      <td>851</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Rock</td>\n",
       "      <td>821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Gabba</td>\n",
       "      <td>741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Pop</td>\n",
       "      <td>732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>UK Garage</td>\n",
       "      <td>695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Electro</td>\n",
       "      <td>682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Deep Dubstep</td>\n",
       "      <td>596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Roots/Lovers Rock</td>\n",
       "      <td>522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Hard Techno</td>\n",
       "      <td>455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Ambient/Drone</td>\n",
       "      <td>402</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Funk</td>\n",
       "      <td>361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Scouse House</td>\n",
       "      <td>321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Dub</td>\n",
       "      <td>261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Coldwave/Synth</td>\n",
       "      <td>233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Jazz</td>\n",
       "      <td>117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>DJ Tools</td>\n",
       "      <td>104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Industrial/Noise</td>\n",
       "      <td>95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Footwork/Juke</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Classics/Ska</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>International</td>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Soul</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Soundtracks</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>Leftfield</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>50s/60s</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Rock (All)</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               release_genre     id\n",
       "0         Minimal/Tech House  29415\n",
       "1          Progressive House  14515\n",
       "2           Funky/Club House  13517\n",
       "3                 Deep House  13098\n",
       "4                     Techno  12394\n",
       "5           Uplifting Trance   8606\n",
       "6              Electro House   7169\n",
       "7              Drum And Bass   6381\n",
       "8   Dirty Dubstep/Trap/Grime   4164\n",
       "9                  Breakbeat   3829\n",
       "10            Disco/Nu-Disco   2980\n",
       "11        Balearic/Downtempo   2927\n",
       "12      Euro Dance/Pop Dance   2643\n",
       "13               Hip Hop/R&B   2483\n",
       "14                 Hardstyle   2136\n",
       "15            Psy/Goa Trance   2103\n",
       "16           Dancehall/Ragga   1728\n",
       "17               Hard Trance   1522\n",
       "18                     Indie   1369\n",
       "19               UK Hardcore   1322\n",
       "20                Hard House   1239\n",
       "21   Experimental/Electronic   1156\n",
       "22                Pop Trance   1092\n",
       "23                      Bass    963\n",
       "24       Broken Beat/Nu Jazz    851\n",
       "25                      Rock    821\n",
       "26                     Gabba    741\n",
       "27                       Pop    732\n",
       "28                 UK Garage    695\n",
       "29                   Electro    682\n",
       "30              Deep Dubstep    596\n",
       "31         Roots/Lovers Rock    522\n",
       "32               Hard Techno    455\n",
       "33             Ambient/Drone    402\n",
       "34                      Funk    361\n",
       "35              Scouse House    321\n",
       "36                       Dub    261\n",
       "37            Coldwave/Synth    233\n",
       "38                      Jazz    117\n",
       "39                  DJ Tools    104\n",
       "40          Industrial/Noise     95\n",
       "41             Footwork/Juke     87\n",
       "42              Classics/Ska     72\n",
       "43             International     66\n",
       "44                      Soul     57\n",
       "45               Soundtracks     39\n",
       "46                 Leftfield     20\n",
       "47                   50s/60s     18\n",
       "48                Rock (All)      1"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "track_genres = df.groupby('release_genre')['id'].count().sort_values(ascending=False).reset_index()\n",
    "track_genres"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Lots of genres, and lots with too few songs."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "I'm going to map some of these smaller genres to a bigger parent genre and then remove ones that I'm not going to use."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def parent_genre(s):\n",
    "    \n",
    "    parent_genres = {\n",
    "    'Minimal/Tech House':'Minimal House',\n",
    "    'Progressive House':'Progressive House',\n",
    "    'Funky/Club House':'Funky House',\n",
    "    'Deep House':'Deep House',\n",
    "    'Techno':'Techno',\n",
    "    'Uplifting Trance':'Trance',\n",
    "    'Electro House':'Electro House',\n",
    "    'Drum And Bass':'Drum And Bass',\n",
    "    'Dirty Dubstep/Trap/Grime':'Dubstep/Grime',\n",
    "    'Breakbeat':'Breakbeat',\n",
    "    'Disco/Nu-Disco':'Disco',\n",
    "    'Balearic/Downtempo':'Downtempo',\n",
    "    'Euro Dance/Pop Dance':'Euro Dance',\n",
    "    'Hip Hop/R&B':'Hip Hop/R&B',\n",
    "    'Hardstyle':'Not Needed',\n",
    "    'Psy/Goa Trance':'Not Needed',\n",
    "    'Dancehall/Ragga':'Dancehall/Ragga',\n",
    "    'Hard Trance':'Hard Trance',\n",
    "    'Indie':'Rock/Indie',\n",
    "    'UK Hardcore':'Not Needed',\n",
    "    'Hard House':'Hard House',\n",
    "    'Experimental/Electronic':'Not Needed',\n",
    "    'Pop Trance':'Not Needed',\n",
    "    'Bass':'Not Needed',\n",
    "    'Broken Beat/Nu Jazz':'Not Needed',\n",
    "    'Rock':'Rock/Indie',\n",
    "    'Gabba':'Not Needed',\n",
    "    'Pop':'Pop',\n",
    "    'UK Garage':'UK Garage',\n",
    "    'Electro':'Not Needed',\n",
    "    'Deep Dubstep':'Dubstep/Grime',\n",
    "    'Roots/Lovers Rock':'Not Needed',\n",
    "    'Hard Techno':'Not Needed',\n",
    "    'Ambient/Drone':'Not Needed',\n",
    "    'Funk':'Not Needed',\n",
    "    'Scouse House':'Not Needed',\n",
    "    'Dub':'Not Needed',\n",
    "    'Coldwave/Synth':'Not Needed',\n",
    "    'Jazz':'Not Needed',\n",
    "    'DJ Tools':'Not Needed',\n",
    "    'Industrial/Noise':'Not Needed',\n",
    "    'Footwork/Juke':'Not Needed',\n",
    "    'Classics/Ska':'Not Needed',\n",
    "    'International':'Not Needed',\n",
    "    'Soul':'Not Needed',\n",
    "    'Soundtracks':'Not Needed',\n",
    "    'Leftfield':'Not Needed',\n",
    "    '50s/60s':'Not Needed',\n",
    "    'Rock (All)':'Rock/Indie'}\n",
    "    \n",
    "    parent = parent_genres[s]\n",
    "    return parent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df['parent_genre'] = df['release_genre'].apply(parent_genre)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>release_artist</th>\n",
       "      <th>release_cat_num</th>\n",
       "      <th>release_date</th>\n",
       "      <th>release_genre</th>\n",
       "      <th>release_label</th>\n",
       "      <th>release_title</th>\n",
       "      <th>track_name</th>\n",
       "      <th>track_url</th>\n",
       "      <th>parent_genre</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>CLEAR VIEW feat JESSICA</td>\n",
       "      <td>SB 215-0</td>\n",
       "      <td>10 Sep 08</td>\n",
       "      <td>Progressive House</td>\n",
       "      <td>Songbird Holland</td>\n",
       "      <td>Tell Me</td>\n",
       "      <td>Tell Me - (6:43)</td>\n",
       "      <td>http://www.junodownload.com/MP3/SF1354749-02-0...</td>\n",
       "      <td>Progressive House</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>_LINDEN</td>\n",
       "      <td>AED 0027DL</td>\n",
       "      <td>05 Feb 16</td>\n",
       "      <td>Rock</td>\n",
       "      <td>AED</td>\n",
       "      <td>Bones/Broken Glass</td>\n",
       "      <td>Bones - (3:17)</td>\n",
       "      <td>http://www.junodownload.com/MP3/SF3007568-02-0...</td>\n",
       "      <td>Rock/Indie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>_NYQUIST</td>\n",
       "      <td>AMREC 008</td>\n",
       "      <td>23 Jan 17</td>\n",
       "      <td>Techno</td>\n",
       "      <td>Ambidextrous</td>\n",
       "      <td>Phenoxy</td>\n",
       "      <td>The Golden Sea - (5:16)</td>\n",
       "      <td>http://www.junodownload.com/MP3/SF3258293-02-0...</td>\n",
       "      <td>Techno</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>_UNSUBSCRIBE_</td>\n",
       "      <td>BNR 154D</td>\n",
       "      <td>22 Apr 16</td>\n",
       "      <td>Techno</td>\n",
       "      <td>Boysnoize Germany</td>\n",
       "      <td>Penultimate</td>\n",
       "      <td>The Opener - (6:45)</td>\n",
       "      <td>http://www.junodownload.com/MP3/SF3052441-02-0...</td>\n",
       "      <td>Techno</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>14</td>\n",
       "      <td>-PF</td>\n",
       "      <td>PF 152</td>\n",
       "      <td>23 Dec 15</td>\n",
       "      <td>Indie</td>\n",
       "      <td>Stereo Parrot</td>\n",
       "      <td>Swearing</td>\n",
       "      <td>Swearing - (2:38)</td>\n",
       "      <td>http://www.junodownload.com/MP3/SF3222101-02-0...</td>\n",
       "      <td>Rock/Indie</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id           release_artist release_cat_num release_date  \\\n",
       "0   1  CLEAR VIEW feat JESSICA        SB 215-0    10 Sep 08   \n",
       "1   3                  _LINDEN      AED 0027DL    05 Feb 16   \n",
       "2   5                 _NYQUIST       AMREC 008    23 Jan 17   \n",
       "3   9            _UNSUBSCRIBE_        BNR 154D    22 Apr 16   \n",
       "4  14                      -PF          PF 152    23 Dec 15   \n",
       "\n",
       "       release_genre      release_label       release_title  \\\n",
       "0  Progressive House   Songbird Holland             Tell Me   \n",
       "1               Rock                AED  Bones/Broken Glass   \n",
       "2             Techno       Ambidextrous             Phenoxy   \n",
       "3             Techno  Boysnoize Germany         Penultimate   \n",
       "4              Indie      Stereo Parrot            Swearing   \n",
       "\n",
       "                track_name                                          track_url  \\\n",
       "0         Tell Me - (6:43)  http://www.junodownload.com/MP3/SF1354749-02-0...   \n",
       "1           Bones - (3:17)  http://www.junodownload.com/MP3/SF3007568-02-0...   \n",
       "2  The Golden Sea - (5:16)  http://www.junodownload.com/MP3/SF3258293-02-0...   \n",
       "3      The Opener - (6:45)  http://www.junodownload.com/MP3/SF3052441-02-0...   \n",
       "4        Swearing - (2:38)  http://www.junodownload.com/MP3/SF3222101-02-0...   \n",
       "\n",
       "        parent_genre  \n",
       "0  Progressive House  \n",
       "1         Rock/Indie  \n",
       "2             Techno  \n",
       "3             Techno  \n",
       "4         Rock/Indie  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Look at the genres based on newly created parent genre..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>parent_genre</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Minimal House</td>\n",
       "      <td>29415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Progressive House</td>\n",
       "      <td>14515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Not Needed</td>\n",
       "      <td>14276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Funky House</td>\n",
       "      <td>13517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Deep House</td>\n",
       "      <td>13098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Techno</td>\n",
       "      <td>12394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Trance</td>\n",
       "      <td>8606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Electro House</td>\n",
       "      <td>7169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Drum And Bass</td>\n",
       "      <td>6381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Dubstep/Grime</td>\n",
       "      <td>4760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Breakbeat</td>\n",
       "      <td>3829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Disco</td>\n",
       "      <td>2980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Downtempo</td>\n",
       "      <td>2927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Euro Dance</td>\n",
       "      <td>2643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Hip Hop/R&amp;B</td>\n",
       "      <td>2483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Rock/Indie</td>\n",
       "      <td>2191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Dancehall/Ragga</td>\n",
       "      <td>1728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Hard Trance</td>\n",
       "      <td>1522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Hard House</td>\n",
       "      <td>1239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Pop</td>\n",
       "      <td>732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>UK Garage</td>\n",
       "      <td>695</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         parent_genre     id\n",
       "0       Minimal House  29415\n",
       "1   Progressive House  14515\n",
       "2          Not Needed  14276\n",
       "3         Funky House  13517\n",
       "4          Deep House  13098\n",
       "5              Techno  12394\n",
       "6              Trance   8606\n",
       "7       Electro House   7169\n",
       "8       Drum And Bass   6381\n",
       "9       Dubstep/Grime   4760\n",
       "10          Breakbeat   3829\n",
       "11              Disco   2980\n",
       "12          Downtempo   2927\n",
       "13         Euro Dance   2643\n",
       "14        Hip Hop/R&B   2483\n",
       "15         Rock/Indie   2191\n",
       "16    Dancehall/Ragga   1728\n",
       "17        Hard Trance   1522\n",
       "18         Hard House   1239\n",
       "19                Pop    732\n",
       "20          UK Garage    695"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('parent_genre')['id'].count().sort_values(ascending=False).reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Remove the not needed 'genre'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>parent_genre</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Minimal House</td>\n",
       "      <td>29415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Progressive House</td>\n",
       "      <td>14515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Funky House</td>\n",
       "      <td>13517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Deep House</td>\n",
       "      <td>13098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Techno</td>\n",
       "      <td>12394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Trance</td>\n",
       "      <td>8606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Electro House</td>\n",
       "      <td>7169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Drum And Bass</td>\n",
       "      <td>6381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Dubstep/Grime</td>\n",
       "      <td>4760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Breakbeat</td>\n",
       "      <td>3829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Disco</td>\n",
       "      <td>2980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Downtempo</td>\n",
       "      <td>2927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Euro Dance</td>\n",
       "      <td>2643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Hip Hop/R&amp;B</td>\n",
       "      <td>2483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Rock/Indie</td>\n",
       "      <td>2191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Dancehall/Ragga</td>\n",
       "      <td>1728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Hard Trance</td>\n",
       "      <td>1522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Hard House</td>\n",
       "      <td>1239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Pop</td>\n",
       "      <td>732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>UK Garage</td>\n",
       "      <td>695</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         parent_genre     id\n",
       "0       Minimal House  29415\n",
       "1   Progressive House  14515\n",
       "2         Funky House  13517\n",
       "3          Deep House  13098\n",
       "4              Techno  12394\n",
       "5              Trance   8606\n",
       "6       Electro House   7169\n",
       "7       Drum And Bass   6381\n",
       "8       Dubstep/Grime   4760\n",
       "9           Breakbeat   3829\n",
       "10              Disco   2980\n",
       "11          Downtempo   2927\n",
       "12         Euro Dance   2643\n",
       "13        Hip Hop/R&B   2483\n",
       "14         Rock/Indie   2191\n",
       "15    Dancehall/Ragga   1728\n",
       "16        Hard Trance   1522\n",
       "17         Hard House   1239\n",
       "18                Pop    732\n",
       "19          UK Garage    695"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "not_needed = df[df['parent_genre'] == 'Not Needed']\n",
    "df = df.drop(not_needed.index)\n",
    "df.groupby('parent_genre')['id'].count().sort_values(ascending=False).reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Sample 1,000 songs for each genre"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "I want to take a sample of 1000 songs for each genre."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>parent_genre</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Minimal House</td>\n",
       "      <td>29415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Progressive House</td>\n",
       "      <td>14515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Funky House</td>\n",
       "      <td>13517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Deep House</td>\n",
       "      <td>13098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Techno</td>\n",
       "      <td>12394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Trance</td>\n",
       "      <td>8606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Electro House</td>\n",
       "      <td>7169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Drum And Bass</td>\n",
       "      <td>6381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Dubstep/Grime</td>\n",
       "      <td>4760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Breakbeat</td>\n",
       "      <td>3829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Disco</td>\n",
       "      <td>2980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Downtempo</td>\n",
       "      <td>2927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Euro Dance</td>\n",
       "      <td>2643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Hip Hop/R&amp;B</td>\n",
       "      <td>2483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Rock/Indie</td>\n",
       "      <td>2191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Dancehall/Ragga</td>\n",
       "      <td>1728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Hard Trance</td>\n",
       "      <td>1522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Hard House</td>\n",
       "      <td>1239</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         parent_genre     id\n",
       "0       Minimal House  29415\n",
       "1   Progressive House  14515\n",
       "2         Funky House  13517\n",
       "3          Deep House  13098\n",
       "4              Techno  12394\n",
       "5              Trance   8606\n",
       "6       Electro House   7169\n",
       "7       Drum And Bass   6381\n",
       "8       Dubstep/Grime   4760\n",
       "9           Breakbeat   3829\n",
       "10              Disco   2980\n",
       "11          Downtempo   2927\n",
       "12         Euro Dance   2643\n",
       "13        Hip Hop/R&B   2483\n",
       "14         Rock/Indie   2191\n",
       "15    Dancehall/Ragga   1728\n",
       "16        Hard Trance   1522\n",
       "17         Hard House   1239"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trimmed = df[(df['parent_genre'] != 'Pop') & (df['parent_genre'] != 'UK Garage')]\n",
    "df_trimmed.groupby('parent_genre')['id'].count().sort_values(ascending=False).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "grouped = df_trimmed.groupby('parent_genre', as_index=False)\n",
    "equal_sample_df = grouped.apply(lambda x: x.sample(1000)).reset_index()\n",
    "equal_sample_df.drop(['level_0','level_1'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>parent_genre</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Trance</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Techno</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dancehall/Ragga</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Deep House</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Disco</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Downtempo</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Drum And Bass</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Dubstep/Grime</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Electro House</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Euro Dance</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Funky House</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Hard House</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Hard Trance</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Hip Hop/R&amp;B</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Minimal House</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Progressive House</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Rock/Indie</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Breakbeat</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         parent_genre    id\n",
       "0              Trance  1000\n",
       "1              Techno  1000\n",
       "2     Dancehall/Ragga  1000\n",
       "3          Deep House  1000\n",
       "4               Disco  1000\n",
       "5           Downtempo  1000\n",
       "6       Drum And Bass  1000\n",
       "7       Dubstep/Grime  1000\n",
       "8       Electro House  1000\n",
       "9          Euro Dance  1000\n",
       "10        Funky House  1000\n",
       "11         Hard House  1000\n",
       "12        Hard Trance  1000\n",
       "13        Hip Hop/R&B  1000\n",
       "14      Minimal House  1000\n",
       "15  Progressive House  1000\n",
       "16         Rock/Indie  1000\n",
       "17          Breakbeat  1000"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "equal_sample_df.groupby('parent_genre')['id'].count().sort_values(ascending=False).reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Save data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "equal_sample_df.to_pickle('final_data.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}