notebooks/splines.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Part of the code cames from the author of the original paper that mentioned this dataset.\n",
    "\n",
    "Raghunathan, Aditi, et al. \"Understanding and mitigating the tradeoff between robustness and accuracy.\" arXiv preprint arXiv:2002.10716 (2020).\n",
    "\"\"\"\n",
    "\n",
    "from functools import reduce\n",
    "from collections import namedtuple\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import cvxpy as cp\n",
    "import scipy\n",
    "import pandas as pd\n",
    "from scipy.optimize import minimize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_stairs = 10\n",
    "num_examples = 10\n",
    "adv_eps = (1.0 / 3)\n",
    "noise_eps = 0.1\n",
    "x_noise = 0.005\n",
    "slope = 1\n",
    "#theta = np.random.randn(feats.shape[1])\n",
    "lamda = 0.01\n",
    "np.set_printoptions(precision=5)\n",
    "\n",
    "knots = np.r_[np.arange(num_stairs), np.arange(num_stairs)-adv_eps, np.arange(num_stairs)+adv_eps]\n",
    "knots = np.sort(knots)\n",
    "\n",
    "# weights on different stairs\n",
    "weights_1 = np.asarray([1/5]*5)\n",
    "weights_2 = np.asarray([0.01]*(num_stairs-5))\n",
    "weights = np.concatenate([weights_1, weights_2])\n",
    "weights /= np.sum(weights)\n",
    "\n",
    "Result = namedtuple('Result', ['x'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_x_noise(n):    \n",
    "    # with probability x_noise there is noise\n",
    "    unit_noise = (np.random.rand(n) < 0.5).astype(float)\n",
    "    noise_mask = (np.random.rand(n) < x_noise).astype(float)\n",
    "    return noise_mask * (unit_noise*2 - 1)*adv_eps\n",
    "\n",
    "def data_gen(n, weights):\n",
    "    num_stairs = len(weights)\n",
    "    # sample from categorical distribution\n",
    "    X = np.random.choice(num_stairs, p=weights, size=n).astype(float)\n",
    "    return X\n",
    "\n",
    "def label_noise(n, eps):\n",
    "    return eps * np.random.randn(n)\n",
    "\n",
    "def get_test_set():\n",
    "    num_samples_per_stair = 10000\n",
    "    # X_test has every row a different class and columns are samples from that class\n",
    "    y_test = slope*np.arange(num_stairs).astype(float)[:, np.newaxis] * np.ones((num_stairs, num_samples_per_stair))\n",
    "    X_test = np.arange(num_stairs).astype(float)[:, np.newaxis]\n",
    "    noise = (np.random.rand(num_stairs, num_samples_per_stair)*2 - 1)*adv_eps\n",
    "    X_test = X_test + noise\n",
    "    return X_test, y_test\n",
    "\n",
    "def get_feats(X, knots):\n",
    "    X = X[:, np.newaxis]\n",
    "    M = 4\n",
    "    aug = np.arange(1, M+1)\n",
    "    knots = np.r_[aug - M - 1 - knots[0], knots, aug + knots[-1]]\n",
    "\n",
    "    K = len(knots)\n",
    "    bases = (X >= knots[:-1]).astype(np.int) * (X < knots[1:]).astype(np.int)\n",
    "    # do recursion from Hastie et al. vectorized\n",
    "    maxi = len(knots) - 1\n",
    "    for m in range(2, M+1):\n",
    "        maxi -= 1\n",
    "\n",
    "        # left sub-basis\n",
    "        num = (X - knots[:maxi])* bases[:, :maxi]\n",
    "        denom = knots[m-1 : maxi+m-1] - knots[:maxi]\n",
    "        left = num/denom\n",
    "\n",
    "        # right sub-basis\n",
    "        num = (knots[m : maxi+m] - X) * bases[:, 1:maxi+1]\n",
    "        denom = knots[m:maxi+m] - knots[1 : maxi+1]\n",
    "        right = num/denom\n",
    "\n",
    "        bases = left + right\n",
    "    return bases\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sqerr(feats, y, theta): \n",
    "    return np.square(feats.dot(theta) - y).sum()\n",
    "\n",
    "def sparse_diff(array, n=1, axis=-1):                                                                                                                 \n",
    "    if (n < 0) or (int(n) != n):                                                \n",
    "        raise ValueError('Expected order is non-negative integer, '             \n",
    "                         'but found: {}'.format(n))                             \n",
    "    if not scipy.sparse.issparse(array):                                        \n",
    "        warnings.warn('Array is not sparse. Consider using numpy.diff')         \n",
    "                                                                                \n",
    "    if n == 0:                                                                  \n",
    "        return array                                                            \n",
    "                                                                                \n",
    "    nd = array.ndim                                                             \n",
    "    slice1 = [slice(None)]*nd                                                   \n",
    "    slice2 = [slice(None)]*nd                                                   \n",
    "    slice1[axis] = slice(1, None)                                               \n",
    "    slice2[axis] = slice(None, -1)                                              \n",
    "    slice1 = tuple(slice1)                                                      \n",
    "    slice2 = tuple(slice2)                                                      \n",
    "                                                                                \n",
    "    A = sparse_diff(array, n-1, axis=axis)                                      \n",
    "    return A[slice1] - A[slice2]\n",
    "\n",
    "def first_derivative(n, order=2):\n",
    "    if n == 1:                                                                  \n",
    "        # no derivative for constant functions                                  \n",
    "        return scipy.sparse.csc_matrix(0.)                                      \n",
    "    D = sparse_diff(scipy.sparse.identity(n).tocsc(), n=order).tolil()          \n",
    "    return np.asarray(D.tocsc().todense())\n",
    "\n",
    "def derivative(n, order=2):                                                     \n",
    "    if n == 1:                                                                  \n",
    "        # no derivative for constant functions                                  \n",
    "        return scipy.sparse.csc_matrix(0.)                                      \n",
    "    D = sparse_diff(scipy.sparse.identity(n).tocsc(), n=order).tolil()          \n",
    "    return np.asarray(D.dot(D.T).tocsc().todense()) \n",
    "\n",
    "def get_P(knots, with_intercept=False):\n",
    "    P = derivative(len(knots) + 4, order=2) \n",
    "    return P\n",
    "\n",
    "def norm(theta):\n",
    "    return P.dot(theta).dot(theta)\n",
    "\n",
    "def T(x):\n",
    "    x_round = np.round(x)\n",
    "    return [x_round-adv_eps, x_round,  x_round+adv_eps]\n",
    "    \n",
    "def sqerr_adv(theta, T_feats):\n",
    "    sqerrs = [np.square(dat.dot(theta) - y) for dat in T_feats]\n",
    "    max_errs = np.sum(np.maximum.reduce(sqerrs))\n",
    "    return max_errs\n",
    "\n",
    "def test_sqerr(X_test, theta): \n",
    "    def err_y_for_x(x, e_y):                                                    \n",
    "        x_feats = get_feats(np.asarray([x]), knots)                             \n",
    "        preds_x = x_feats.dot(theta)[0]                                         \n",
    "        err_for_pt = np.square(preds_x) - 2*preds_x*e_y + noise_eps**2 + np.square(e_y)\n",
    "        return err_for_pt                                                       \n",
    "                                                                                \n",
    "    total_err = []                                                              \n",
    "    for i in range(X_test.shape[0]):                                                                                                            \n",
    "        class_i_err = 0                                                     \n",
    "        for x in [i-adv_eps, i+adv_eps]:                                    \n",
    "            class_i_err += 0.5 * x_noise * err_y_for_x(x, slope*i)                                                                \n",
    "        class_i_err += (1-x_noise)*err_y_for_x(i, slope*i)                      \n",
    "        total_err.append(class_i_err)                                           \n",
    "    total_err = np.asarray(total_err)                                           \n",
    "    return np.sum(total_err * weights)\n",
    "\n",
    "def summarize(theta, X, y, X_test):\n",
    "    T_feats = [get_feats(xx, knots) for xx in [X, X-adv_eps, X+adv_eps]]\n",
    "    feats = np.array(T_feats[0])\n",
    "    print(\"sq err train\")\n",
    "    print(sqerr(feats, y, theta)/X.shape[0])\n",
    "    print(\"sq err adv train\")\n",
    "    print(sqerr_adv(theta, T_feats)/X.shape[0])\n",
    "    print(\"sq err test\")\n",
    "    print(test_sqerr(X_test, theta))\n",
    "    print(\"generalization gap\")\n",
    "    print(test_sqerr(X_test, theta) - sqerr(feats[0], y, theta)/X.shape[0])\n",
    "    print(\"rkhs norm\")\n",
    "    print(norm(theta))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def solve_normal(f_mat, labels, l):\n",
    "    mat = f_mat.T.dot(f_mat) + l * get_P(knots)\n",
    "    theta = np.linalg.pinv(mat).dot(f_mat.T.dot(labels))\n",
    "    res_normal = Result(x=theta)\n",
    "    return res_normal\n",
    "\n",
    "def solve_gr(f_mat, y, lamda, beta=1.0):\n",
    "    def f0(w):\n",
    "        loss = np.dot(f_mat, w) - y\n",
    "        objective = (\n",
    "            np.sum(loss**2) \\\n",
    "            + lamda*np.dot(np.dot(w, P), w) \\\n",
    "            + beta / 2 * np.linalg.norm(loss.reshape(-1, 1) * f_mat, axis=1).sum()\n",
    "        )\n",
    "        return objective\n",
    "           \n",
    "    res = minimize(f0, np.zeros(f_mat.shape[1]), method='L-BFGS-B')\n",
    "    return Result(x=res.x)\n",
    "\n",
    "def solve_llr(T_feats, y, lamda, P, knots_idx, beta=1.0):\n",
    "    def f0(w):\n",
    "        sum_sq = [np.sum((np.dot(dat, w) - y)**2) for dat in T_feats]\n",
    "        temp = adv_eps * 2 * np.sum(np.dot(np.dot(D[knots_idx], w), (np.dot(T_feats[1], w) - y)))\n",
    "        l0 = sum_sq[0] - sum_sq[1] + temp\n",
    "        l1 = sum_sq[2] - sum_sq[1] - temp\n",
    "        return sum_sq[1] + beta*np.max([l0, l1]) + lamda*np.dot(np.dot(w, P), w)\n",
    "        \n",
    "    res = minimize(f0, np.zeros(T_feats[0].shape[1]))\n",
    "    return Result(x=res.x)\n",
    "    \n",
    "\n",
    "def solve_adv(T_feats, y, lamda, P):\n",
    "    theta_var = cp.Variable(T_feats[0].shape[1])\n",
    "    sum_sq = [cp.square(dat@theta_var - y) for dat in T_feats]\n",
    "    objective = cp.Minimize(cp.sum(reduce(cp.maximum, sum_sq)) + lamda*cp.quad_form(theta_var, P))\n",
    "    prob = cp.Problem(objective)\n",
    "    prob.solve()\n",
    "    return Result(x=theta_var.value)\n",
    "\n",
    "def solve_betaadv(T_feats, y, lamda, P, beta):\n",
    "    theta_var = cp.Variable(T_feats[0].shape[1])\n",
    "    mse = cp.square(T_feats[1]@theta_var - y)\n",
    "    sum_sq = [cp.square(dat@theta_var - y) for dat in T_feats]\n",
    "    objective = cp.Minimize(cp.sum(mse) + beta * cp.sum(reduce(cp.maximum, sum_sq)) + lamda*cp.quad_form(theta_var, P))\n",
    "    prob = cp.Problem(objective)\n",
    "    prob.solve()\n",
    "    return Result(x=theta_var.value)\n",
    "\n",
    "def solve_trades(T_feats, y, lamda, P, beta):\n",
    "    theta_var = cp.Variable(T_feats[0].shape[1])\n",
    "    mse = cp.square(T_feats[1]@theta_var - y)\n",
    "    sum_sq = [cp.square(dat@theta_var - T_feats[1]@theta_var) for dat in T_feats]\n",
    "    objective = cp.Minimize(cp.sum(mse) + beta * cp.sum(reduce(cp.maximum, sum_sq)) + lamda*cp.quad_form(theta_var, P))\n",
    "    prob = cp.Problem(objective)\n",
    "    prob.solve()\n",
    "    return Result(x=theta_var.value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "D = derivative(len(knots) + 4, order=1) \n",
    "P = get_P(knots)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_mse_err(theta, X, y):\n",
    "    feats = get_feats(X, knots)\n",
    "    #feats = np.array([get_feats(x, knots) for x in X])\n",
    "    mse = (np.dot(feats, theta) - y)**2\n",
    "    return mse.mean()\n",
    "\n",
    "def calc_lip(theta, X, y):\n",
    "    preds = np.array([np.dot(get_feats(x, knots), theta) for x in [X-adv_eps, X+adv_eps]])\n",
    "    lip = (np.abs(preds - y)).max(0) / adv_eps\n",
    "    return lip.mean()\n",
    "\n",
    "def adv_mse_err(theta, X, y):\n",
    "    preds = np.array([np.dot(get_feats(x, knots), theta) for x in [X-adv_eps, X, X+adv_eps]])\n",
    "    mse = ((preds - y)**2).max(0)\n",
    "    return mse.mean()\n",
    "\n",
    "def get_sol(trnX, trny, tstX, tsty, prob, beta=1, cv=\"nor\"):\n",
    "    feats = get_feats(trnX, knots)\n",
    "    T_feats = [get_feats(xx, knots) for xx in [trnX-adv_eps, trnX, trnX+adv_eps]]\n",
    "    #tst_feats = get_feats(tstX, knots)\n",
    "    \n",
    "    #lamdas = [1e2, 1e1, 1e0, 1e-1, 1e-2, 1e-3, 1e-4]\n",
    "    lamdas = [1e-2]\n",
    "    mse_err = []\n",
    "    adv_err = []\n",
    "    trn_mse_err = []\n",
    "    rets = []\n",
    "    \n",
    "    for lamda in lamdas:\n",
    "        if prob == \"normal\":\n",
    "            res = solve_normal(feats, trny, lamda)\n",
    "        elif prob == \"llr\":\n",
    "            knots_idx = np.argmin(np.abs(trnX.reshape(-1, 1) - knots.reshape(1, -1)), axis=1) + 2\n",
    "            res = solve_llr(T_feats, trny, lamda, P, knots_idx, beta=beta)\n",
    "        elif prob == \"gr\":\n",
    "            res = solve_gr(feats, trny, lamda, beta=beta)\n",
    "        elif prob == \"betaadv\":\n",
    "            res = solve_betaadv(T_feats, trny, lamda, P, beta=beta)\n",
    "        elif prob == \"adv\":\n",
    "            res = solve_adv(T_feats, trny, lamda, P)\n",
    "        elif prob == \"trades\":\n",
    "            res = solve_trades(T_feats, trny, lamda, P, beta=beta)\n",
    "        theta = res.x\n",
    "        trn_mse_err.append(calc_mse_err(theta, trnX, trny))\n",
    "        adv_err.append(adv_mse_err(theta, trnX, trny))\n",
    "        mse_err.append(calc_mse_err(theta, tstX, tsty))\n",
    "        rets.append(theta)\n",
    "\n",
    "    print(mse_err)\n",
    "    #print(adv_err)\n",
    "    if cv == \"trnor\":\n",
    "        return rets[np.argmin(trn_mse_err)]\n",
    "    elif cv == \"nor\":\n",
    "        return rets[np.argmin(mse_err)]\n",
    "    else:\n",
    "        return rets[np.argmin(adv_err)]\n",
    "\n",
    "def print_results(theta, trnX, trny, tstX, tsty):\n",
    "    trn_mse = calc_mse_err(theta, trnX, trny)\n",
    "    tst_mse = calc_mse_err(theta, tstX, tsty)\n",
    "    adv_trn_mse = adv_mse_err(theta, trnX, trny)\n",
    "    adv_tst_mse = adv_mse_err(theta, tstX, tsty)\n",
    "    trn_lip = calc_lip(theta, trnX, trny)\n",
    "    tst_lip = calc_lip(theta, tstX, tsty)\n",
    "\n",
    "    \n",
    "    #print(f\"trn mse: {trn_mse}\")\n",
    "    #print(f\"advtrn mse: {adv_trn_mse}\")\n",
    "    \n",
    "    #print(f\"tst mse: {tst_mse}\")\n",
    "    #print(f\"advtst mse: {adv_tst_mse}\")\n",
    "    \n",
    "    #print(f\"gap: {-trn_mse + tst_mse}\")\n",
    "    #print(f\"adv gap: {-adv_trn_mse + adv_tst_mse}\")\n",
    "    return trn_mse, tst_mse, adv_trn_mse, adv_tst_mse, trn_lip, tst_lip\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_examples = 40\n",
    "np.random.seed(1)\n",
    "trnX = data_gen(num_examples, weights)\n",
    "trny = slope*trnX + label_noise(trnX.shape[0], noise_eps)\n",
    "trnX += generate_x_noise(num_examples)\n",
    "\n",
    "tstX = data_gen(10000, weights)\n",
    "tsty = slope*tstX + label_noise(tstX.shape[0], noise_eps)\n",
    "tstX += generate_x_noise(10000)\n",
    "\n",
    "X_test, y_test = get_test_set()\n",
    "\n",
    "theta_normal = get_sol(trnX, trny, tstX, tsty, \"normal\", cv=\"nor\")\n",
    "theta_gr = get_sol(trnX, trny, tstX, tsty, \"gr\", beta=1e-4, cv=\"adv\")\n",
    "theta_llr = get_sol(trnX, trny, tstX, tsty, \"llr\", beta=0.01, cv=\"adv\")\n",
    "theta_adv = get_sol(trnX, trny, tstX, tsty, \"adv\", cv=\"adv\")\n",
    "theta_betaadv_5 = get_sol(trnX, trny, tstX, tsty, \"betaadv\", beta=.5, cv=\"adv\")\n",
    "theta_betaadv = get_sol(trnX, trny, tstX, tsty, \"betaadv\", beta=1, cv=\"adv\")\n",
    "theta_betaadv2 = get_sol(trnX, trny, tstX, tsty, \"betaadv\", beta=2, cv=\"adv\")\n",
    "theta_trd_10 = get_sol(trnX, trny, tstX, tsty, \"trades\", beta=6, cv=\"adv\")\n",
    "theta_trd_1 = get_sol(trnX, trny, tstX, tsty, \"trades\", beta=3, cv=\"adv\")\n",
    "theta_trd_0 = get_sol(trnX, trny, tstX, tsty, \"trades\", beta=1, cv=\"adv\")\n",
    "\n",
    "X_plot = np.linspace(-adv_eps, num_stairs-1 + adv_eps, 100)\n",
    "feats_plot = get_feats(X_plot, knots)\n",
    "normal_plot_preds = feats_plot.dot(theta_normal)\n",
    "\n",
    "#plt.scatter(X_test, y_test)\n",
    "plt.scatter(trnX, trny)\n",
    "#plt.scatter(tstX, tsty)\n",
    "\n",
    "plt.plot(X_plot, normal_plot_preds, label=\"Normal\")\n",
    "gr_plot_preds = feats_plot.dot(theta_gr)\n",
    "plt.plot(X_plot, gr_plot_preds, label=\"GR\")\n",
    "llr_plot_preds = feats_plot.dot(theta_llr)\n",
    "plt.plot(X_plot, llr_plot_preds, label=\"LLR\")\n",
    "adv_plot_preds = feats_plot.dot(theta_adv)\n",
    "plt.plot(X_plot, adv_plot_preds, label=\"AT\")\n",
    "\n",
    "badv_plot_preds = feats_plot.dot(theta_betaadv_5)\n",
    "plt.plot(X_plot, badv_plot_preds, label=\"RST(.5)\")\n",
    "badv_plot_preds = feats_plot.dot(theta_betaadv)\n",
    "plt.plot(X_plot, badv_plot_preds, label=\"RST(1)\")\n",
    "badv_plot_preds = feats_plot.dot(theta_betaadv2)\n",
    "plt.plot(X_plot, badv_plot_preds, label=\"RST(2)\")\n",
    "\n",
    "trd_plot_preds = feats_plot.dot(theta_trd_10)\n",
    "plt.plot(X_plot, trd_plot_preds, label=\"TRADES(b=6)\")\n",
    "trd_plot_preds = feats_plot.dot(theta_trd_1)\n",
    "plt.plot(X_plot, trd_plot_preds, label=\"TRADES(b=3)\")\n",
    "trd_plot_preds = feats_plot.dot(theta_trd_0)\n",
    "plt.plot(X_plot, trd_plot_preds, label=\"TRADES(b=1)\")\n",
    "#trd_plot_preds = feats_plot.dot(theta_trd_01)\n",
    "#plt.plot(X_plot, trd_plot_preds, label=\"Trades(b=0.01)\")\n",
    "plt.legend()\n",
    "plt.savefig(\"./small-stair.png\")\n",
    "#print(\"Normal\")\n",
    "normal_res = print_results(theta_normal, trnX, trny, tstX, tsty)\n",
    "gr_res = print_results(theta_gr, trnX, trny, tstX, tsty)\n",
    "llr_res = print_results(theta_llr, trnX, trny, tstX, tsty)\n",
    "adv_res = print_results(theta_adv, trnX, trny, tstX, tsty)\n",
    "betaadv_5_res = print_results(theta_betaadv_5, trnX, trny, tstX, tsty)\n",
    "betaadv_res = print_results(theta_betaadv, trnX, trny, tstX, tsty)\n",
    "betaadv2_res = print_results(theta_betaadv2, trnX, trny, tstX, tsty)\n",
    "trd10_res = print_results(theta_trd_10, trnX, trny, tstX, tsty)\n",
    "trd1_res = print_results(theta_trd_1, trnX, trny, tstX, tsty)\n",
    "trd0_res = print_results(theta_trd_0, trnX, trny, tstX, tsty)\n",
    "\n",
    "df = pd.DataFrame.from_dict({\n",
    "    \"normal\": normal_res,\n",
    "    \"GR\": gr_res,\n",
    "    \"LLR\": llr_res,\n",
    "    \"AT\": adv_res,\n",
    "    \"RST(.5)\": betaadv_5_res,\n",
    "    \"RST(1)\": betaadv_res,\n",
    "    \"RST(2)\": betaadv2_res,\n",
    "    \"TRADES(1)\": trd0_res,\n",
    "    \"TRADES(3)\": trd1_res,\n",
    "    \"TRADES(6)\": trd10_res,\n",
    "}, orient=\"index\")\n",
    "df.columns = ['tr MSE', 'ts MSE', 'adv tr MSE', 'adv ts MSE', 'tr lip(1)', 'ts lip(1)']\n",
    "df['gap'] = -(df['tr MSE'] - df['ts MSE'])\n",
    "df['adv gap'] = -(df['adv tr MSE'] - df['adv ts MSE'])\n",
    "\n",
    "df = df[['tr MSE', 'ts MSE', 'adv ts MSE', 'ts lip(1)', 'gap', 'adv gap']]\n",
    "\n",
    "print(df.to_latex(column_format=\"lc|cc|c|cc\", float_format=\"%.4f\", escape=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_examples = 30000\n",
    "np.random.seed(1)\n",
    "trnX = data_gen(num_examples, weights)\n",
    "trny = slope*trnX + label_noise(trnX.shape[0], noise_eps)\n",
    "trnX += generate_x_noise(num_examples)\n",
    "\n",
    "tstX = data_gen(15000, weights)\n",
    "tsty = slope*tstX + label_noise(tstX.shape[0], noise_eps)\n",
    "tstX += generate_x_noise(15000)\n",
    "\n",
    "\n",
    "theta_normal = get_sol(trnX, trny, tstX, tsty, \"normal\", cv=\"nor\")\n",
    "theta_gr = get_sol(trnX, trny, tstX, tsty, \"gr\", beta=1e-4, cv=\"adv\")\n",
    "theta_llr = get_sol(trnX, trny, tstX, tsty, \"llr\", beta=0.01, cv=\"adv\")\n",
    "theta_adv = get_sol(trnX, trny, tstX, tsty, \"adv\", cv=\"adv\")\n",
    "theta_betaadv_5 = get_sol(trnX, trny, tstX, tsty, \"betaadv\", beta=.5, cv=\"adv\")\n",
    "theta_betaadv = get_sol(trnX, trny, tstX, tsty, \"betaadv\", beta=1, cv=\"adv\")\n",
    "theta_betaadv2 = get_sol(trnX, trny, tstX, tsty, \"betaadv\", beta=2, cv=\"adv\")\n",
    "theta_trd_10 = get_sol(trnX, trny, tstX, tsty, \"trades\", beta=6, cv=\"adv\")\n",
    "theta_trd_1 = get_sol(trnX, trny, tstX, tsty, \"trades\", beta=3, cv=\"adv\")\n",
    "theta_trd_0 = get_sol(trnX, trny, tstX, tsty, \"trades\", beta=1, cv=\"adv\")\n",
    "\n",
    "X_plot = np.linspace(-adv_eps, num_stairs-1 + adv_eps, 100)\n",
    "feats_plot = get_feats(X_plot, knots)\n",
    "normal_plot_preds = feats_plot.dot(theta_normal)\n",
    "\n",
    "#plt.scatter(X_test, y_test)\n",
    "plt.scatter(trnX, trny)\n",
    "#plt.scatter(tstX, tsty)\n",
    "\n",
    "plt.plot(X_plot, normal_plot_preds, label=\"Normal\")\n",
    "gr_plot_preds = feats_plot.dot(theta_gr)\n",
    "plt.plot(X_plot, gr_plot_preds, label=\"GR\")\n",
    "llr_plot_preds = feats_plot.dot(theta_llr)\n",
    "plt.plot(X_plot, llr_plot_preds, label=\"LLR\")\n",
    "adv_plot_preds = feats_plot.dot(theta_adv)\n",
    "plt.plot(X_plot, adv_plot_preds, label=\"AT\")\n",
    "trd_plot_preds = feats_plot.dot(theta_trd_10)\n",
    "\n",
    "badv_plot_preds = feats_plot.dot(theta_betaadv_5)\n",
    "plt.plot(X_plot, badv_plot_preds, label=\"RST(.5)\")\n",
    "badv_plot_preds = feats_plot.dot(theta_betaadv)\n",
    "plt.plot(X_plot, badv_plot_preds, label=\"RST(1)\")\n",
    "badv_plot_preds = feats_plot.dot(theta_betaadv2)\n",
    "plt.plot(X_plot, badv_plot_preds, label=\"RST(2)\")\n",
    "plt.plot(X_plot, trd_plot_preds, label=\"TRADES(b=6)\")\n",
    "trd_plot_preds = feats_plot.dot(theta_trd_1)\n",
    "plt.plot(X_plot, trd_plot_preds, label=\"TRADES(b=3)\")\n",
    "trd_plot_preds = feats_plot.dot(theta_trd_0)\n",
    "plt.plot(X_plot, trd_plot_preds, label=\"TRADES(b=1)\")\n",
    "#trd_plot_preds = feats_plot.dot(theta_trd_01)\n",
    "#plt.plot(X_plot, trd_plot_preds, label=\"Trades(b=0.01)\")\n",
    "plt.legend()\n",
    "plt.savefig(\"./large-stair.png\")\n",
    "#print(\"Normal\")\n",
    "normal_res = print_results(theta_normal, trnX, trny, tstX, tsty)\n",
    "gr_res = print_results(theta_gr, trnX, trny, tstX, tsty)\n",
    "llr_res = print_results(theta_llr, trnX, trny, tstX, tsty)\n",
    "adv_res = print_results(theta_adv, trnX, trny, tstX, tsty)\n",
    "betaadv_5_res = print_results(theta_betaadv_5, trnX, trny, tstX, tsty)\n",
    "betaadv_res = print_results(theta_betaadv, trnX, trny, tstX, tsty)\n",
    "betaadv2_res = print_results(theta_betaadv2, trnX, trny, tstX, tsty)\n",
    "trd10_res = print_results(theta_trd_10, trnX, trny, tstX, tsty)\n",
    "trd1_res = print_results(theta_trd_1, trnX, trny, tstX, tsty)\n",
    "trd0_res = print_results(theta_trd_0, trnX, trny, tstX, tsty)\n",
    "\n",
    "df = pd.DataFrame.from_dict({\n",
    "    \"normal\": normal_res,\n",
    "    \"GR\": gr_res,\n",
    "    \"LLR\": llr_res,\n",
    "    \"AT\": adv_res,\n",
    "    \"RST(.5)\": betaadv_5_res,\n",
    "    \"RST(1)\": betaadv_res,\n",
    "    \"RST(2)\": betaadv2_res,\n",
    "    \"TRADES(6)\": trd10_res,\n",
    "    \"TRADES(3)\": trd1_res,\n",
    "    \"TRADES(1)\": trd0_res,\n",
    "}, orient=\"index\")\n",
    "df.columns = ['tr MSE', 'ts MSE', 'adv tr MSE', 'adv ts MSE', 'tr lip(1)', 'ts lip(1)']\n",
    "df['gap'] = (df['tr MSE'] - df['ts MSE'])\n",
    "df['adv gap'] = (df['adv tr MSE'] - df['adv ts MSE'])\n",
    "\n",
    "df = df[['tr MSE', 'ts MSE', 'adv ts MSE', 'ts lip(1)', 'gap', 'adv gap']]\n",
    "\n",
    "print(df.to_latex(column_format=\"lc|cc|c|cc\", float_format=\"%.3f\", escape=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}