np_split_array_based_on_running_sum.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "written for https://stackoverflow.com/q/49939762/425458"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[12, 13, 14, 5]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([3,  2,  3,  3,  1,  3,  3,  2,  4,  3,  4,  3,  2])\n",
    "time = np.array([0,  2,  4,  7, 10, 11, 12, 17, 21, 25, 29, 30, 40])\n",
    "\n",
    "def count(x, time, epoch=10):\n",
    "    ix = time.searchsorted(np.arange(0, time.max(), epoch))\n",
    "    counts = [np.sum(x[ixb:ixe+1]) for ixb,ixe in zip(ix[:-1], ix[1:])]\n",
    "    if (time[-1] - time[ix[-1]]) >= epoch:\n",
    "        return counts + [np.sum(x[ix[-1]:])]\n",
    "    else:\n",
    "        return counts\n",
    "    \n",
    "count(x,time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "signal = np.array([3,  2,  3,  3,  1,  3,  3,  2,  4,  3,  4,  3,  2])\n",
    "time = np.array([0,  2,  4,  7, 10, 11, 12, 17, 21, 25, 29, 30, 40])\n",
    "\n",
    "from itertools import chain,repeat,accumulate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9.63 µs ± 182 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)\n"
     ]
    }
   ],
   "source": [
    "    %%timeit\n",
    "    idx = np.fromiter(iter(accumulate(chain((0,), repeat(10)), lambda now, delta: time.searchsorted(time[now] + delta)).__next__, len(time)), int)\n",
    "    np.add.reduceat(signal[:idx[-1]], idx[:-1]) + signal[idx[1:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[11, 9, 14, 2]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# time = np.array([ 2,  2,  4,  7, 10, 11, 12, 17, 21, 25, 29, 30, 31, 40])\n",
    "\n",
    "# Using broadcasting you can get a 2d array of the difference of all items\n",
    "# from other items within your array\n",
    "arr = time[:, None] - time\n",
    "# Then find indices where the difference is less and equal to -10\n",
    "x, y = np.where(arr <= -10)\n",
    "# find the first occurrences of where for each item the difference is less and equal to -10 \n",
    "first_acc = np.concatenate(([0], np.where(np.diff(x) != 0)[0]  + 1, [x.size]))\n",
    "\n",
    "# use a recursive generator function to retrieve all the expected indices.\n",
    "def get_ind_rec(ind=0):\n",
    "    try:\n",
    "        ind = y[first_acc[ind]]\n",
    "        yield ind\n",
    "        yield from get_ind_rec(ind)\n",
    "    except: # IndexError\n",
    "        pass\n",
    "     \n",
    "[np.sum(x) for x in np.split(signal, list(get_ind_rec()))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[4, 8, 12]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(get_ind_rec())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30.1 µs ± 502 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n"
     ]
    }
   ],
   "source": [
    "    %%timeit\n",
    "    out=[]\n",
    "    prev=0\n",
    "    # need to reinitialize the time array since the loop eats it\n",
    "    time = np.array([0,  2,  4,  7, 10, 11, 12, 17, 21, 25, 29, 31, 40])\n",
    "    while True:\n",
    "        try:\n",
    "            idx10 = np.where(time >=10)[0][0]\n",
    "            time-=time[idx10]\n",
    "            out.append(np.sum(signal[prev:idx10+1]))\n",
    "            prev=idx10\n",
    "        except:\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[12, 13, 14]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20.6 µs ± 701 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "arr = time[:, None] - time\n",
    "x, y = np.where(arr <= -10)\n",
    "first_acc = np.concatenate(([0], np.where(np.diff(x) != 0)[0]  + 1, [x.size]))\n",
    "def get_ind_rec(ind=0):\n",
    "    try:\n",
    "        ind = y[first_acc[ind]]\n",
    "        yield ind\n",
    "        yield from get_ind_rec(ind)\n",
    "    except: # IndexError\n",
    "        pass\n",
    "     \n",
    "list(get_ind_rec())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-2-a1f375a7f66e>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-2-a1f375a7f66e>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    ims = []; for fname in train_filenames: print(loc+\"/\"+fname); ims.append(fname)\u001b[0m\n\u001b[0m                ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "np.array([])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[12, 13, 14]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "    signal = np.array([3,  2,  3,  3,  1,  3,  3,  2,  4,  3,  4,  3,  2])\n",
    "    time = np.array([0,  2,  4,  7, 10, 11, 12, 17, 21, 25, 29, 31, 40])\n",
    "\n",
    "    def count(signal, time, epoch=10):\n",
    "        counts = []\n",
    "        total = 0\n",
    "        timestart = times[0]\n",
    "\n",
    "        for x,t in zip(signal, time):\n",
    "            total += x\n",
    "\n",
    "            if t - timestart >= epoch:\n",
    "                counts.append(total)\n",
    "                total = x\n",
    "                timestart = t\n",
    "\n",
    "        return counts\n",
    "\n",
    "    count(signal, time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.88 µs ± 165 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit        \n",
    "count(xs, times)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1. ,  3.5,  3. ,  2. ])"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.interpolate import interp1d\n",
    "\n",
    "signal = np.array([3,2,3,3,1, 3, 3, 2, 4, 3, 4, 3, 2])\n",
    "time = np.array([  0,2,4,7,10,11,12,17,21,25,29,30,40])\n",
    "\n",
    "timevals = np.arange(10,time[-1]+0.0001,10,dtype=int)\n",
    "\n",
    "npinterp = np.interp(timevals,time,signal)\n",
    "\n",
    "interf = interp1d(time, signal, copy=False)\n",
    "interf(timevals)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.arange(1, 100)\n",
    "def chunk(xs, n):\n",
    "    ys = list(xs)\n",
    "    random.shuffle(ys)\n",
    "    size = len(ys) // n\n",
    "    leftovers= ys[size*n:]\n",
    "    for c, xtra in enumerate(leftovers):\n",
    "        yield ys[c*size:(c+1)*size] + [ xtra ]\n",
    "    for c in xrange(c+1,n):\n",
    "        yield ys[c*size:(c+1)*size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([67, 79, 17, 62, 12, 37, 70, 24]),\n",
       " array([98, 48, 88, 59, 47]),\n",
       " array([52, 60, 89, 23, 43, 44]),\n",
       " array([ 7, 27, 33, 74, 49,  2]),\n",
       " array([ 6, 51, 40, 13, 56, 45]),\n",
       " array([31,  3, 55, 10, 11, 46,  9, 42, 34]),\n",
       " array([53, 22, 95, 41, 19, 32,  4, 69, 86]),\n",
       " array([93, 68, 57, 65, 92, 76, 28, 63, 64, 58]),\n",
       " array([91, 66, 18, 99, 21]),\n",
       " array([36, 83, 15, 78,  1, 81, 97, 84]),\n",
       " array([61, 71, 25, 94, 87, 20, 85, 38]),\n",
       " array([ 8, 96, 75, 30, 77, 14, 72, 29]),\n",
       " array([35, 90, 82, 73, 39,  5, 26, 50, 16]),\n",
       " array([80, 54])]"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from itertools import chain\n",
    "import numpy as np\n",
    "\n",
    "a = np.arange(1, 100)\n",
    "def chunk(xs, nlow, nhigh, shuffle=True):\n",
    "    xs = np.asarray(xs)\n",
    "    if shuffle:\n",
    "        # shuffle, if you want\n",
    "        xs = xs.copy()\n",
    "        np.random.shuffle(xs)\n",
    "    \n",
    "    # get at least enough random chunk sizes in the specified range, ie nlow <= n <= nhigh\n",
    "    ns = np.random.randint(nlow, nhigh+1, size=xs.size//nlow)\n",
    "    # add up the chunk sizes to get the indices at which we'll slice up the input array\n",
    "    ixs = np.add.accumulate(ns)\n",
    "    # truncate ixs so that its contents are all valid indices with respect to xs\n",
    "    ixs = ixs[:np.searchsorted(ixs, xs.size)]\n",
    "\n",
    "    # yield slices from the input array\n",
    "    for start,end in zip(chain([None], ixs), chain(ixs, [None])):\n",
    "        yield xs[start:end]\n",
    "     \n",
    "list(chunk(a, 5, 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([10,  9,  7,  5,  9,  6,  9,  9,  7,  6,  5,  6,  5,  5,  5,  6,  6,\n",
       "        6,  9])"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlow = 5\n",
    "nhigh = 10\n",
    "xs = np.arange(1, 100)\n",
    "\n",
    "# get at least enough random chunk sizes in the specified range, ie nlow <= n <= nhigh\n",
    "ns = np.random.randint(nlow, nhigh+1, size=xs.size//nlow)\n",
    "# add up the chunk sizes to get the indices at which we'll slice up the input array\n",
    "ixs = np.add.accumulate(ns)\n",
    "# truncate ixs so that its contents are all valid indices with respect to xs\n",
    "ixs[:np.searchsorted(ixs, xs.size)]\n",
    "#     ixs = np.array([None] + ixs[:np.searchsorted(ixs, len(ys))].tolist() + [None])\n",
    "#     return ixs\n",
    "# yield slices from the input array\n",
    "# for start,end in zip(chain([None], ixs), chain(ixs, [None])):\n",
    "#     print(xs[start:end])\n",
    "\n",
    "np.random.randint(nlow, nhigh+1, size=xs.size//nlow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ixnp.searchsorted(ixs, xs.size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([None, 8, 14, 19, 25, 32, 41, 50, 59, 66, 73, 82, 87, 93, None], dtype=object)]"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ixs[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.optimize as opt\n",
    "\n",
    "x = np.array([4,8,15,29,58,116,231,462,924,1848])\n",
    "y = np.array([1.05,2.11,3.95,7.37,13.88,25.46,43.03,64.28,81.97,87.43])\n",
    "\n",
    "for x1, y1 in zip(x,y):\n",
    "    plt.plot(x1, y1, 'ro')\n",
    "\n",
    "z = np.polyfit(x, y, 3)\n",
    "f = np.poly1d(z)\n",
    "\n",
    "for x1 in np.linspace(0, 1848, 110):\n",
    "    plt.plot(x1, f(x1), 'b+')\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}