Fingerprints to t-SNE.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This converts `fingerprints.npy` to `.tsv` formatted t-SNE embeddings and plots of those embeddings in the `tsne/` and `plot/` folders respectively. If you add multiple values to `perplexity` and `initial_dims` then all combinations will be computed (in parallel). Good perplexities are in the range 1-200 with the best range around 30-100. Good `initial_dims` are in the range 30 and higher, with the dimensionality of your input data being the highest possible value (e.g., a 32x32 fingerprint would have a highest possible `initial_dims` value of 32x32=1024).\n",
    "\n",
    "Change the \"mode\" to try different t-SNE variations.\n",
    "* \"fingerprints\" will only use `fingerprints.npy`\n",
    "* \"predicted_labels\" will only use `predicted_labels.npy`\n",
    "* \"predicted_encoding\" will only use `predicted_encoding.npy`\n",
    "* \"combined\" will use all of the above data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_root = 'data/drums/'\n",
    "initial_dims = [30]\n",
    "perplexities = [30]\n",
    "mode = 'fingerprints'\n",
    "# mode = 'predicted_labels'\n",
    "# mode = 'predicted_encoding'\n",
    "# mode = 'combined'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from matplotlib import pyplot as plt\n",
    "from time import time\n",
    "from utils import *\n",
    "from os.path import join\n",
    "from multiprocessing import Pool\n",
    "import numpy as np\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def save_tsv(data, fn):\n",
    "    np.savetxt(fn, data, fmt='%.5f', delimiter='\\t')\n",
    "def tsne(data, data_root, prefix, initial_dims=30, perplexity=30):\n",
    "    mkdir_p(data_root + 'tsne')\n",
    "    mkdir_p(data_root + 'plot')\n",
    "    \n",
    "    figsize = (16,16)\n",
    "    pointsize = 2\n",
    "\n",
    "    X_2d = list(bh_tsne(data, initial_dims=initial_dims, perplexity=perplexity, no_dims=2))\n",
    "    X_2d = normalize(np.array(X_2d))\n",
    "    save_tsv(X_2d, join(data_root, 'tsne/{}.{}.{}.2d.tsv'.format(prefix, initial_dims, perplexity)))\n",
    "    \n",
    "    plt.figure(figsize=figsize)\n",
    "    plt.scatter(X_2d[:,0], X_2d[:,1], edgecolor='', s=pointsize)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(join(data_root, 'plot/{}.{}.{}.png'.format(prefix, initial_dims, perplexity)))\n",
    "    plt.close()\n",
    "    \n",
    "    X_3d = list(bh_tsne(data, initial_dims=initial_dims, perplexity=perplexity, no_dims=3))\n",
    "    X_3d = normalize(np.array(X_3d))\n",
    "    save_tsv(X_3d, join(data_root, 'tsne/{}.{}.{}.3d.tsv'.format(prefix, initial_dims, perplexity)))\n",
    "    \n",
    "    plt.figure(figsize=figsize)\n",
    "    plt.scatter(X_2d[:,0], X_2d[:,1], edgecolor='', s=pointsize, c=X_3d)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(join(data_root, 'plot/{}.{}.{}.png'.format(prefix, initial_dims, perplexity)))\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load and normalize any dataset we need\n",
    "if mode == 'fingerprints' or mode == 'combined':\n",
    "    fingerprints = np.load(join(data_root, 'fingerprints.npy'))\n",
    "    fingerprints = fingerprints.reshape(len(fingerprints), -1)\n",
    "if mode == 'predicted_labels' or mode == 'combined':\n",
    "    predicted_labels = np.load(join(data_root, 'predicted_labels.npy'))\n",
    "    predicted_labels -= predicted_labels.min()\n",
    "    predicted_labels /= predicted_labels.max()\n",
    "if mode == 'predicted_encoding' or mode == 'combined':\n",
    "    predicted_encoding = np.load(join(data_root, 'predicted_encoding.npy'))\n",
    "    std = predicted_encoding.std(axis=0)\n",
    "    predicted_encoding = predicted_encoding[:, std > 0] / std[std > 0]\n",
    "    \n",
    "if mode == 'fingerprints':\n",
    "    data = fingerprints\n",
    "if mode == 'predicted_labels':\n",
    "    data = predicted_labels\n",
    "if mode == 'predicted_encoding':\n",
    "    data = predicted_encoding\n",
    "if mode == 'combined':\n",
    "    data = np.hstack((fingerprints, predicted_labels, predicted_encoding))\n",
    "    \n",
    "print data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = data.astype(np.float64)\n",
    "def job(params):\n",
    "    start = time()\n",
    "    tsne(data, data_root, mode, initial_dims=params[0], perplexity=params[1])\n",
    "    print 'initial_dims={}, perplexity={}, {} seconds'.format(params[0], params[1], time() - start)\n",
    "params = list(itertools.product(initial_dims, perplexities))\n",
    "pool = Pool()\n",
    "pool.map(job, params)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}