Collect Samples.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This tool converts a folder of samples to a big rectangular matrix with one mono sample per row.\n",
    "\n",
    "Samples should be placed in `data/mydataset/samples/`. They could be `.mp3`, `.wav`, or anything else that ffmpeg can work with. They may be all in one folder, or in nested sub-folders.\n",
    "\n",
    "Change the path below to point to the root directory, e.g., `data/mydataset/`.\n",
    "\n",
    "The samplerate `sr` is not necessarily the native samplerate of the samples, it's the samplerate you want to load them at.\n",
    "\n",
    "The output of this notebook is:\n",
    "* `data/mydataset/durations.txt`\n",
    "* `data/mydataset/filenames.txt`\n",
    "* `data/mydataset/samples.npy`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_root = 'data/drums/'\n",
    "sr = 48000\n",
    "max_length = sr*4 # ignore samples longer than 4 seconds\n",
    "fixed_length = sr/4 # trim all samples to 250 milliseconds\n",
    "limit = None # set this to 100 to only load the first 100 samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from os.path import join\n",
    "from utils import *\n",
    "from multiprocessing import Pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "files = list(list_all_files(join(data_root, 'samples'), ['.mp3']))\n",
    "len(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_sample(fn, sr=None,\n",
    "                max_length=None, fixed_length=None, normalize=True):\n",
    "    if fn == '': # ignore empty filenames\n",
    "        return None\n",
    "    audio, _ = ffmpeg_load_audio(fn, sr, mono=True)\n",
    "    duration = len(audio)\n",
    "    if duration == 0: # ignore zero-length samples\n",
    "        return None\n",
    "    if max_length and duration >= max_length: # ignore long samples\n",
    "        return None\n",
    "    if fixed_length:\n",
    "        audio.resize(fixed_length)\n",
    "    max_val = np.abs(audio).max()\n",
    "    if max_val == 0: # ignore completely silent sounds\n",
    "        return None\n",
    "    if normalize:\n",
    "        audio /= max_val\n",
    "    return (fn, audio, duration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def job(fn):\n",
    "    return load_sample(fn, sr=sr,\n",
    "                       max_length=max_length, fixed_length=fixed_length)\n",
    "pool = Pool()\n",
    "%time results = pool.map(job, files[:limit])\n",
    "print 'Processed', len(results), 'samples'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "valid = filter(None, results)\n",
    "filenames = [x[0] for x in valid]\n",
    "samples = [x[1] for x in valid]\n",
    "durations = [x[2] for x in valid]\n",
    "samples = np.asarray(samples)\n",
    "np.savetxt(join(data_root, 'filenames.txt'), filenames, fmt='%s')\n",
    "np.savetxt(join(data_root, 'durations.txt'), durations, fmt='%i')\n",
    "%time np.save(join(data_root, 'samples.npy'), samples)\n",
    "print 'Saved', len(valid), 'samples'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}