ModelPlots.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Plots\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-05T23:51:16.544327Z",
     "start_time": "2019-03-05T23:51:16.531457Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Suppress the powerlaw package warnings\n",
    "# \"powerlaw.py:700: RuntimeWarning: divide by zero encountered in true_divide\"\n",
    "# \"powerlaw.py:700: RuntimeWarning: invalid value encountered in true_divide\"\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=RuntimeWarning)\n",
    "\n",
    "# 0.4x names\n",
    "\n",
    "ALPHA_HAT = 'alpha_weighted'\n",
    "ALPHA = 'alpha'\n",
    "LOG_ALPHA_NORM = 'log_alpha_norm'\n",
    "MP_SOFTRANK = 'mp_softrank'\n",
    "SPECTRAL_NORM = 'spectral_norm'\n",
    "STABLE_RANK = 'stable_rank'  \n",
    "FROBENIUS_NORM = 'norm'\n",
    "\n",
    "LOG_NORM = 'log_norm'\n",
    "LOG_SPECTRAL_NORM = 'log_spectral_norm'\n",
    "LOG_STABLE_RANK = 'log_stable_rank' \n",
    "LOG_FROBENIUS_NORM = LOG_NORM\n",
    "\n",
    "\n",
    "WW4X_COMPAT_COLUMNS = {'lognorm':LOG_NORM,\n",
    "                'logspectralnorm':LOG_SPECTRAL_NORM,\n",
    "                'spectralnorm':SPECTRAL_NORM,\n",
    "                'logpnorm':LOG_ALPHA_NORM,\n",
    "                'softrank':STABLE_RANK,\n",
    "                'softranklog':LOG_STABLE_RANK,\n",
    "                'softrank_mp':MP_SOFTRANK}\n",
    "\n",
    "\n",
    "LOG_NORM_EQN =  r\"$\\langle\\log\\Vert W\\Vert^{2}_{F}\\rangle$\"\n",
    "ALPHA_EQN = r\"$\\langle\\alpha\\rangle$\"\n",
    "ALPHA_HAT_EQN = r\"$\\hat{\\alpha}$\"\n",
    "LOG_SPECTRAL_NORM_EQN = r\"$\\langle\\log\\;\\Vert\\mathbf{W}\\Vert^{2}_{\\infty}\\rangle$\"\n",
    "LOG_STABLE_RANK_EQN = r\"$\\langle\\log\\;\\mathcal{R}_{s}\\rangle$\"\n",
    "MP_SOFTRANK_EQN = r\"$\\langle\\log\\;\\mathcal{R}_{mp}\\rangle$\"\n",
    "LOG_ALPHA_NORM_EQN = r\"$\\langle\\log\\;\\Vert\\mathbf{X}\\Vert^{\\alpha}_{\\alpha}\\rangle$\"\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy as sp\n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn import metrics\n",
    "from scipy.stats import kendalltau\n",
    "import itertools\n",
    "\n",
    "    \n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams.update({'font.size': 20})\n",
    "from pylab import rcParams\n",
    "rcParams['figure.figsize'] = 10,10\n",
    "\n",
    "MARKERS = ['o', '<', '>', 'v', '^', 's', 'D', 'X']\n",
    "MARKER_SIZE = 250\n",
    "TRANSPARENCY = 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_test_accuracy(metric, xlabel, title, series_name, \\\n",
    "                       all_names, all_summaries, top_errors, ylabel='Top1 Test Accuracy'):\n",
    "    \"\"\"Create plot of Metric vs Reported Test Accuracy, and run Linear Regression\"\"\"\n",
    "    \n",
    "    markers = itertools.cycle(MARKERS)\n",
    "                         \n",
    "    num = len(all_names)\n",
    "    xs, ys = np.empty(num), np.empty(num)\n",
    "    for im, modelname in enumerate(all_names):    \n",
    "\n",
    "        summary = all_summaries[im]\n",
    "        x = summary[metric]\n",
    "        xs[im] = x\n",
    "\n",
    "        error = top_errors[modelname]\n",
    "        y = 100.0-error\n",
    "        ys[im] = y\n",
    "\n",
    "        label = modelname\n",
    "        plt.scatter(x, y, label=label, s=MARKER_SIZE, marker=next(markers))\n",
    "\n",
    "\n",
    "    xs = xs.reshape(-1,1)\n",
    "    ys = ys.reshape(-1,1)\n",
    "    regr = LinearRegression()\n",
    "    regr.fit(xs, ys)\n",
    "    y_pred = regr.predict(xs)\n",
    "    plt.plot(xs, y_pred, color='red', linewidth=1)\n",
    "\n",
    "    rmse = np.sqrt(metrics.mean_squared_error(ys, y_pred))\n",
    "    r2 = metrics.r2_score(ys, y_pred)\n",
    "    ktau, pval  = kendalltau(xs,ys)\n",
    "    \n",
    "    title2 = r\"RMSE: {:0.2} R2: {:0.2} $\\tau$: {:0.2}\".format(rmse, r2, ktau)\n",
    "    \n",
    "    # legend moved for ResNet-1K\n",
    "    plt.legend(bbox_to_anchor=(1.04,1), loc=\"upper left\")\n",
    "    plt.title(\" Test Accuracy vs \"+title+\"\\n\"+title2)\n",
    "    plt.ylabel(ylabel)\n",
    "    plt.xlabel(xlabel);\n",
    "    \n",
    "    figname = \"img/{}_{}_accs.png\".format(series_name, metric)\n",
    "    print(\"saving {}\".format(figname))\n",
    "    plt.savefig(figname, bbox_inches=\"tight\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_metrics_histogram(metric, xlabel, title, series_name, \\\n",
    "    all_names, all_details, colors, log=False, valid_ids = []):\n",
    "                                \n",
    "    transparency = 1.0\n",
    "    \n",
    "    if len(valid_ids) == 0:\n",
    "        valid_ids = range(0,len(all_details)-1)\n",
    "        idname='all'\n",
    "    else:\n",
    "        idname='fnl'\n",
    "        \n",
    "    for im, details in enumerate(all_details):\n",
    "        if im in valid_ids:\n",
    "            vals = details[metric].to_numpy()\n",
    "            if log:\n",
    "                vals = np.log10(np.array(vals+0.000001, dtype=np.float))\n",
    "\n",
    "            mu = np.mean(vals)\n",
    "            med = np.median(vals)\n",
    "\n",
    "            label = r\"{} $\\mu=${:0.2f}\".format(all_names[im],mu)\n",
    "            plt.hist(vals, bins=100, label=label, alpha=transparency, color=colors[im], density=True)\n",
    "            transparency -= 0.15\n",
    "            print(r\"{} {} {} median = {:0.3}\".format(title, metric, all_names[im], med))\n",
    "            \n",
    "    fulltitle = \"Histogram: \"+title+\" \"+xlabel\n",
    "  \n",
    "    plt.legend()\n",
    "    plt.title(title)\n",
    "    plt.title(fulltitle)\n",
    "    plt.xlabel(xlabel)\n",
    "    \n",
    "    figname = \"img/{}_{}_{}_hist.png\".format(series_name, idname, metric)\n",
    "    print(\"saving {}\".format(figname))\n",
    "    plt.savefig(figname)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_metrics_depth(metric, ylabel, title, series_name, \\\n",
    "    all_names, all_details, colors, log=False, valid_ids = []):\n",
    "    \n",
    "    markers = itertools.cycle(MARKERS)\n",
    "\n",
    "    transparency = TRANSPARENCY\n",
    "      \n",
    "    if len(valid_ids) == 0:\n",
    "        valid_ids = range(len(all_details)-1)\n",
    "        idname='all'\n",
    "    else:\n",
    "        idname='fnl'\n",
    "        \n",
    "        \n",
    "    for im, details in enumerate(all_details):\n",
    "        if im in valid_ids:\n",
    "            \n",
    "            details = all_details[im]\n",
    "            name = all_names[im]\n",
    "            x = details.index.to_numpy()\n",
    "            y = details[metric].to_numpy()\n",
    "            if log:\n",
    "                y = np.log10(np.array(y+0.000001, dtype=np.float))\n",
    "\n",
    "            plt.scatter(x,y, label=name, color=colors[im], s=MARKER_SIZE/10.0, marker=next(markers))\n",
    "\n",
    "    # moved legend to right for ResNet\n",
    "    plt.legend()\n",
    "    plt.title(\"Depth vs \"+title+\" \"+ylabel)\n",
    "    plt.xlabel(\"Layer id\")\n",
    "    plt.ylabel(ylabel)\n",
    "    \n",
    "    figname = \"img/{}_{}_{}_depth.png\".format(series_name, idname, metric)\n",
    "    print(\"saving {}\".format(figname))\n",
    "    plt.savefig(figname)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Metrics vs Test Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_all_metrics_vs_test_accuracies( \\\n",
    "    series_name, all_names, colors, all_summaries, all_details, \\\n",
    "    top_errors):\n",
    "\n",
    "    metric = LOG_NORM\n",
    "    xlabel = LOG_NORM_EQN \n",
    "    title = \"Avg. log Frobenius Norm \"\n",
    "    plot_test_accuracy(metric, xlabel, title, series_name, \\\n",
    "                        all_names, all_summaries, top_errors)\n",
    "    \n",
    "    metric = ALPHA\n",
    "    xlabel = ALPHA_EQN \n",
    "    title = \"Avg. Alpha  \"\n",
    "    plot_test_accuracy(metric, xlabel, title, series_name, \\\n",
    "                        all_names, all_summaries, top_errors)\n",
    "        \n",
    "    metric = ALPHA_HAT\n",
    "    xlabel = ALPHA_HAT_EQN \n",
    "    title = \"Avg. Weighted Alpha\"\n",
    "    plot_test_accuracy(metric, xlabel, title, series_name, \\\n",
    "                        all_names, all_summaries, top_errors)\n",
    "        \n",
    "    metric = LOG_SPECTRAL_NORM\n",
    "    xlabel = LOG_SPECTRAL_NORM_EQN\n",
    "    title = \"Avg. log Spectral Norm\"\n",
    "    plot_test_accuracy(metric, xlabel, title, series_name, \\\n",
    "                        all_names, all_summaries, top_errors)\n",
    "        \n",
    "    metric = LOG_STABLE_RANK\n",
    "    xlabel = LOG_STABLE_RANK_EQN \n",
    "    title = \"Avg. log Stable Rank\"\n",
    "    plot_test_accuracy(metric, xlabel, title, series_name, \\\n",
    "                        all_names, all_summaries, top_errors)\n",
    "        \n",
    "\n",
    "    metric = MP_SOFTRANK\n",
    "    xlabel = MP_SOFTRANK_EQN \n",
    "    title = \"Avg. log MP Soft Rank\"\n",
    "    plot_test_accuracy(metric, xlabel, title, series_name, \\\n",
    "                        all_names, all_summaries, top_errors)\n",
    "    \n",
    "    metric = LOG_ALPHA_NORM\n",
    "    xlabel = LOG_ALPHA_NORM_EQN \n",
    "    title = r\"Avg. log $\\alpha$-Norm\"\n",
    "    plot_test_accuracy(metric, xlabel, title, series_name, \\\n",
    "                        all_names, all_summaries, top_errors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Histogram of metrics for all layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#first_n_last_ids = [0, len(all_details)-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_all_metric_histograms(\\\n",
    "    series_name, all_names, colors, all_summaries, all_details,  first_n_last_ids):\n",
    "    \n",
    "    metric = LOG_NORM\n",
    "    xlabel = LOG_NORM_EQN\n",
    "    title = series_name\n",
    "    plot_metrics_histogram(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors)                        \n",
    "    plot_metrics_histogram(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, valid_ids = first_n_last_ids)\n",
    "\n",
    "    metric = ALPHA\n",
    "    xlabel = ALPHA_EQN\n",
    "    title = series_name\n",
    "    plot_metrics_histogram(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors)                           \n",
    "    plot_metrics_histogram(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, valid_ids = first_n_last_ids)\n",
    "\n",
    "\n",
    "    metric = ALPHA_HAT\n",
    "    xlabel = ALPHA_HAT_EQN\n",
    "    title = series_name\n",
    "    plot_metrics_histogram(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors)                         \n",
    "    plot_metrics_histogram(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, valid_ids = first_n_last_ids)\n",
    "\n",
    "    \n",
    "    metric = LOG_STABLE_RANK\n",
    "    xlabel = LOG_STABLE_RANK_EQN\n",
    "    title = series_name\n",
    "    plot_metrics_histogram(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors)   \n",
    "    plot_metrics_histogram(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, valid_ids = first_n_last_ids)\n",
    "\n",
    "    metric = LOG_SPECTRAL_NORM\n",
    "    xlabel = LOG_SPECTRAL_NORM_EQN\n",
    "    title = series_name\n",
    "    plot_metrics_histogram(metric, xlabel, title,  series_name, \\\n",
    "            all_names, all_details, colors)                          \n",
    "    plot_metrics_histogram(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, valid_ids = first_n_last_ids)\n",
    "\n",
    "\n",
    "    metric = MP_SOFTRANK\n",
    "    xlabel = MP_SOFTRANK_EQN\n",
    "    title = series_name\n",
    "    plot_metrics_histogram(metric, xlabel, title,  series_name, \\\n",
    "            all_names, all_details, colors)                         \n",
    "    plot_metrics_histogram(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, valid_ids = first_n_last_ids)\n",
    "\n",
    "    \n",
    "    metric = LOG_ALPHA_NORM\n",
    "    xlabel = LOG_ALPHA_NORM_EQN\n",
    "    title = series_name\n",
    "    plot_metrics_histogram(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors)                          \n",
    "    plot_metrics_histogram(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, \\\n",
    "            valid_ids = first_n_last_ids)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Metrics as a function of depth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_all_metric_vs_depth(\\\n",
    "    series_name, all_names, colors, all_summaries, all_details, first_n_last_ids):\n",
    "\n",
    "    metric = LOG_NORM\n",
    "    xlabel = LOG_NORM_EQN\n",
    "    title = series_name\n",
    "    plot_metrics_depth(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, log=False, valid_ids = [])\n",
    "    plot_metrics_depth(metric, xlabel, title,series_name, \\\n",
    "            all_names, all_details, colors, log=False, valid_ids = first_n_last_ids)\n",
    "\n",
    "    metric = ALPHA\n",
    "    xlabel = ALPHA_EQN\n",
    "    title = series_name\n",
    "    plot_metrics_depth(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, log=False, valid_ids = [])\n",
    "    plot_metrics_depth(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, log=False, valid_ids = first_n_last_ids)\n",
    "\n",
    "    metric = ALPHA_HAT\n",
    "    xlabel = ALPHA_HAT_EQN\n",
    "    title = series_name\n",
    "    plot_metrics_depth(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, log=False, valid_ids = [])\n",
    "    plot_metrics_depth(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, log=False, valid_ids = first_n_last_ids)\n",
    "\n",
    "\n",
    "\n",
    "    metric = LOG_STABLE_RANK\n",
    "    xlabel = LOG_STABLE_RANK_EQN\n",
    "    title = series_name\n",
    "    plot_metrics_depth(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, log=False, valid_ids = [])\n",
    "    plot_metrics_depth(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, log=False, valid_ids = first_n_last_ids)\n",
    "\n",
    "    metric = LOG_SPECTRAL_NORM\n",
    "    xlabel = LOG_SPECTRAL_NORM_EQN\n",
    "    title = series_name\n",
    "    plot_metrics_depth(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, log=False, valid_ids = [])\n",
    "    plot_metrics_depth(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, log=False, valid_ids = first_n_last_ids)\n",
    "\n",
    "\n",
    "\n",
    "    metric = MP_SOFTRANK\n",
    "    xlabel = MP_SOFTRANK_EQN\n",
    "    title = series_name\n",
    "    plot_metrics_depth(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, log=False, valid_ids = [])\n",
    "    plot_metrics_depth(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, log=False, valid_ids = first_n_last_ids)\n",
    "\n",
    "    metric = LOG_ALPHA_NORM\n",
    "    xlabel = LOG_ALPHA_NORM_EQN\n",
    "    title = series_name\n",
    "    plot_metrics_depth(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, log=False, valid_ids = [])\n",
    "    plot_metrics_depth(metric, xlabel, title, series_name, \\\n",
    "            all_names, all_details, colors, log=False, valid_ids = first_n_last_ids)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_ww2x_data(all_names,all_details, all_summaries, datadir='data'):\n",
    "    \"\"\"Save ww2x dataframes and summaries\"\"\"\n",
    "    for name, df in zip(all_names,all_details):\n",
    "        filename = \"{}/{}.csv\".format(datadir,name)\n",
    "        df.to_csv(filename)\n",
    "    \n",
    "    for name, summary in zip(all_names,all_summaries):\n",
    "        filename = \"{}/{}.txt\".format(datadir,name)\n",
    "        with open(filename, 'w') as f:\n",
    "            f.write( str(summary) )\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_ww2x_data(all_names, datadir='data'):\n",
    "    \"\"\"read ww2x dataframes and summaries\"\"\"\n",
    "    all_details,all_summaries = [],[]\n",
    "    \n",
    "\n",
    "    for name in all_names:\n",
    "        filename = \"{}/{}.csv\".format(datadir,name)\n",
    "        print(\"loading details in\", filename)\n",
    "        details = pd.read_csv(filename)\n",
    "        all_details.append(details)\n",
    "    \n",
    "    for name in all_names:\n",
    "        filename = \"{}/{}.txt\".format(datadir,name)\n",
    "        print(\"loading summary in \", filename)\n",
    "        summary = eval(open(filename, 'r').read())\n",
    "        all_summaries.append(summary)\n",
    "        \n",
    "    return all_details,all_summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}