20.2 Data Operation Package_Numpy_Samples.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Numpy实用示例"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. 导入numpy包，并查看相关版本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.14.0rc1\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "print(np.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. 如何创建1维向量？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1 2 3 4 5 6 7 8 9]\n"
     ]
    }
   ],
   "source": [
    "arr = np.arange(10)\n",
    "print(arr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. 如何创建一个布尔矩阵？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ True  True  True]\n",
      " [ True  True  True]\n",
      " [ True  True  True]]\n"
     ]
    }
   ],
   "source": [
    "arr = np.full([3,3], True, dtype=bool)\n",
    "print(arr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. 如何从1维整数向量查找奇数？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1 2 3 4 5 6 7 8 9]\n",
      "[False  True False  True False  True False  True False  True]\n",
      "[1 3 5 7 9]\n",
      "[0 1 2 3 4 5 6 7 8 9]\n"
     ]
    }
   ],
   "source": [
    "arr = np.arange(10)\n",
    "print(arr)\n",
    "print(arr % 2 == 1)\n",
    "print(arr[arr % 2 == 1])\n",
    "print(arr.astype(int))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. 如何将array中的奇数都替换为-1？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0 -1  2 -1  4 -1  6 -1  8 -1]\n"
     ]
    }
   ],
   "source": [
    "arr = np.arange(10)\n",
    "arr[arr % 2 == 1] = -1\n",
    "print(arr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. 如何在不改变原array的前提下，将奇数都替换为-1？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1 2 3 4 5 6 7 8 9]\n",
      "[ 0 -1  2 -1  4 -1  6 -1  8 -1]\n"
     ]
    }
   ],
   "source": [
    "arr = np.arange(10)\n",
    "out = np.where(arr % 2 == 1, -1, arr)\n",
    "print(arr)\n",
    "print(out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. 如何reshape array?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 1 2 3 4]\n",
      " [5 6 7 8 9]]\n"
     ]
    }
   ],
   "source": [
    "arr = np.arange(10)\n",
    "print(arr.reshape([2, 5]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8. 如何将两个array垂直叠加？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "method 1: vstack\n",
      "[[0 1 2 3 4]\n",
      " [5 6 7 8 9]\n",
      " [1 1 1 1 1]\n",
      " [1 1 1 1 1]]\n",
      "method 2: concatenate\n",
      "[[0 1 2 3 4]\n",
      " [5 6 7 8 9]\n",
      " [1 1 1 1 1]\n",
      " [1 1 1 1 1]]\n",
      "method 3: r_\n",
      "[[0 1 2 3 4]\n",
      " [5 6 7 8 9]\n",
      " [1 1 1 1 1]\n",
      " [1 1 1 1 1]]\n"
     ]
    }
   ],
   "source": [
    "a = np.arange(10).reshape(2, -1)\n",
    "b = np.repeat(1, 10).reshape(2, -1)\n",
    "print('method 1: vstack')\n",
    "print(np.vstack([a, b]))\n",
    "print('method 2: concatenate')\n",
    "print(np.concatenate([a, b], axis=0))\n",
    "print('method 3: r_')\n",
    "print(np.r_[a, b])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 1],\n",
       "       [2, 3],\n",
       "       [4, 5],\n",
       "       [6, 7],\n",
       "       [8, 9]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.arange(10).reshape(-1, 2)\n",
    "a"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9. 如何将两个array水平叠加？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "method 1: hstack\n",
      "[[0 1 2 3 4 1 1 1 1 1]\n",
      " [5 6 7 8 9 1 1 1 1 1]]\n",
      "method 2: concatenate\n",
      "[[0 1 2 3 4 1 1 1 1 1]\n",
      " [5 6 7 8 9 1 1 1 1 1]]\n",
      "method 3: c_\n",
      "[[0 1 2 3 4 1 1 1 1 1]\n",
      " [5 6 7 8 9 1 1 1 1 1]]\n"
     ]
    }
   ],
   "source": [
    "a = np.arange(10).reshape(2, -1)\n",
    "b = np.repeat(1, 10).reshape(2, -1)\n",
    "print('method 1: hstack')\n",
    "print(np.hstack([a, b]))\n",
    "print('method 2: concatenate')\n",
    "print(np.concatenate([a, b], axis=1))\n",
    "print('method 3: c_')\n",
    "print(np.c_[a, b])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10. 如何创建非硬编码的自定义序列？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "如array: ```[1, 2, 3]```, 期望输出：```[1, 1, 1, 2, 2, 2, 3, 3, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3]```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 2 3]\n"
     ]
    }
   ],
   "source": [
    "arr = np.arange(1, 4)\n",
    "print(arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 1 2 2 2 3 3 3 1 2 3 1 2 3 1 2 3]\n"
     ]
    }
   ],
   "source": [
    "print(np.hstack([np.repeat(arr, 3), np.tile(arr, 3)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 1 2 2 2 3 3 3 1 2 3 1 2 3 1 2 3]\n"
     ]
    }
   ],
   "source": [
    "print(np.r_[np.repeat(arr, 3), np.tile(arr, 3)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 11. 如何从两个array获得共同items？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.array([1, 2, 3, 2, 3, 4, 3, 4, 5, 6])\n",
    "b = np.array([7, 2, 10, 2, 7, 4, 9, 4, 9, 8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2 4]\n"
     ]
    }
   ],
   "source": [
    "print(np.intersect1d(a, b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 12. 如何从array a移除与array b的共同项？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.array([1,2,3,4,5])\n",
    "b = np.array([4, 5,6,7,8,9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 2 3]\n"
     ]
    }
   ],
   "source": [
    "print(np.setdiff1d(a, b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 13. 如何获得两个array共有项的位置索引？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.array([1,2,3,2,3,4,3,4,5,6])\n",
    "b = np.array([7,2,10,2,7,4,9,4,9,8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([1, 3, 5, 7], dtype=int64),)\n"
     ]
    }
   ],
   "source": [
    "print(np.where(a == b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14. 如何从一个array获得给定范围数字的项？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.arange(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 5  6  7  8  9 10]\n",
      "[ 5  6  7  8  9 10]\n",
      "[ 5  6  7  8  9 10]\n"
     ]
    }
   ],
   "source": [
    "# 期望获得5 <= items <= 10\n",
    "# 方法1\n",
    "print(a[(a >=5) & (a <= 10)])\n",
    "# 方法2\n",
    "index = np.where((a >=5) & (a <= 10))\n",
    "print(a[index])\n",
    "# 方法3\n",
    "index = np.logical_and(a >= 5, a <= 10)\n",
    "print(a[index])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15. 如何使得作用于标量的函数，可以用于numpy的array对象？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def maxx(x, y):\n",
    "    \"\"\"Get the maximum of two items\"\"\"\n",
    "    if x >= y:\n",
    "        return x\n",
    "    else:\n",
    "        return y\n",
    "\n",
    "maxx(1, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([6., 7., 9., 8., 9., 7., 5.])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pair_max = np.vectorize(maxx, otypes=[float])\n",
    "\n",
    "a = np.array([5, 7, 9, 8, 6, 4, 5])\n",
    "b = np.array([6, 3, 4, 8, 9, 7, 1])\n",
    "\n",
    "pair_max(a, b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "   ## 16. 如何对换矩阵中的两列？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 1, 2],\n",
       "       [3, 4, 5],\n",
       "       [6, 7, 8]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr = np.arange(9).reshape(3, 3)\n",
    "arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 2],\n",
       "       [4, 3, 5],\n",
       "       [7, 6, 8]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr[:, [1, 0, 2]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 17. 如何对换矩阵中的两行？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3, 4, 5],\n",
       "       [0, 1, 2],\n",
       "       [6, 7, 8]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr[[1, 0 ,2], :]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 18. 如何翻转矩阵中的行？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[6, 7, 8],\n",
       "       [3, 4, 5],\n",
       "       [0, 1, 2]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr[::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[6, 7, 8],\n",
       "       [3, 4, 5],\n",
       "       [0, 1, 2]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 或者\n",
    "arr[::-1, :]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 19. 如何翻转矩阵中的列？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 1, 2],\n",
       "       [3, 4, 5],\n",
       "       [6, 7, 8]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2, 1, 0],\n",
       "       [5, 4, 3],\n",
       "       [8, 7, 6]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr[:, ::-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 20. 如何生成5行3列的大小5~10的浮点数矩阵？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[9.18071427, 7.954727  , 7.76443704],\n",
       "       [9.75508086, 7.28040417, 9.9247645 ],\n",
       "       [5.55713147, 5.51826994, 5.16917056],\n",
       "       [9.29136381, 7.37346593, 9.39478482],\n",
       "       [9.05385751, 6.37971229, 6.93677034]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 方法1\n",
    "randarr = np.random.randint(low=5, high=10, size=(5, 3)) + np.random.random(size=(5, 3))\n",
    "randarr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[9.70747544, 8.21069972, 8.80880415],\n",
       "       [5.47818983, 9.60285796, 6.00053812],\n",
       "       [6.56049546, 6.2308864 , 9.11688023],\n",
       "       [5.80379381, 7.34002745, 8.27758297],\n",
       "       [6.56211226, 7.6440549 , 5.96907913]])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 方法2\n",
    "randarr = np.random.uniform(low=5, high=10, size=(5, 3))\n",
    "randarr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 21. 如何设置浮点型矩阵小数精度为3位？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[9.707, 8.211, 8.809],\n",
       "       [5.478, 9.603, 6.001],\n",
       "       [6.56 , 6.231, 9.117],\n",
       "       [5.804, 7.34 , 8.278],\n",
       "       [6.562, 7.644, 5.969]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.set_printoptions(precision=3)\n",
    "randarr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 22. 如何使用小数形式替代科学计数法？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5.434e-04, 2.784e-04, 4.245e-04],\n",
       "       [8.448e-04, 4.719e-06, 1.216e-04],\n",
       "       [6.707e-04, 8.259e-04, 1.367e-04]])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create the random array\n",
    "np.random.seed(100)\n",
    "rand_arr = np.random.random([3,3])/1e3\n",
    "rand_arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5.434e-04, 2.784e-04, 4.245e-04],\n",
       "       [8.448e-04, 4.719e-06, 1.216e-04],\n",
       "       [6.707e-04, 8.259e-04, 1.367e-04]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reset printoptions to default\n",
    "np.set_printoptions(suppress=False)\n",
    "np.random.seed(100)\n",
    "rand_arr = np.random.random([3,3])/1e3\n",
    "rand_arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.000543, 0.000278, 0.000425],\n",
       "       [0.000845, 0.000005, 0.000122],\n",
       "       [0.000671, 0.000826, 0.000137]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.set_printoptions(suppress=True, precision=6)\n",
    "rand_arr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 23. 如何限制array显示数目？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.arange(15)\n",
    "np.set_printoptions(threshold=np.nan)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  1,  2, ..., 12, 13, 14])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.set_printoptions(threshold=6)\n",
    "a"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 24. 如何完整显示array数目？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  1,  2, ..., 12, 13, 14])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.set_printoptions(threshold=6)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.set_printoptions(threshold=np.nan)\n",
    "a"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 25. 如何导入既含有数字又含有字符串的数据集？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[b'5.1' b'3.5' b'1.4' b'0.2' b'Iris-setosa']\n",
      " [b'4.9' b'3.0' b'1.4' b'0.2' b'Iris-setosa']\n",
      " [b'4.7' b'3.2' b'1.3' b'0.2' b'Iris-setosa']]\n",
      "(150, 5)\n"
     ]
    }
   ],
   "source": [
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris = np.genfromtxt(url, delimiter=',', dtype='object')\n",
    "names = ('sepallength', 'sepalwidth', 'petallength', 'petalwidth', 'species')\n",
    "print(iris[:3])\n",
    "print(iris.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 26. 如何从1维元组向量获得特定列？（此例为鸢尾花名称）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(5.1, 3.5, 1.4, 0.2, 'Iris-setosa') (4.9, 3. , 1.4, 0.2, 'Iris-setosa')]\n",
      "(150,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['Iris-setosa', 'Iris-setosa', 'Iris-setosa', 'Iris-setosa',\n",
       "       'Iris-setosa'], dtype='<U15')"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Input:\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris_1d = np.genfromtxt(url, delimiter=',', dtype=None, encoding='utf-8')\n",
    "print(iris_1d[:2])\n",
    "print(iris_1d.shape)\n",
    "\n",
    "# Solution:\n",
    "species = np.array([row[4] for row in iris_1d])\n",
    "species[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 27. 如何将1维元组向量转为2维矩阵？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "150\n",
      "[[5.1 3.5 1.4 0.2]\n",
      " [4.9 3.  1.4 0.2]\n",
      " [4.7 3.2 1.3 0.2]\n",
      " [4.6 3.1 1.5 0.2]]\n",
      "直接作为矩阵导入\n",
      "[[5.1 3.5 1.4 0.2]\n",
      " [4.9 3.  1.4 0.2]\n",
      " [4.7 3.2 1.3 0.2]\n",
      " [4.6 3.1 1.5 0.2]]\n"
     ]
    }
   ],
   "source": [
    "# Method 1: Convert each row to a list and get the first 4 items\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris_1d = np.genfromtxt(url, delimiter=',', dtype=None, encoding='utf-8')\n",
    "# 将索引为0~3的列转为矩阵\n",
    "iris_2dfrom1dtuple = np.array([row.tolist()[:4] for row in iris_1d])\n",
    "print(len(iris_2dfrom1dtuple))\n",
    "print(iris_2dfrom1dtuple[:4])\n",
    "\n",
    "print('直接作为矩阵导入')\n",
    "# Alt Method 2: Import only the first 4 columns from source url\n",
    "iris_2d = np.genfromtxt(url, delimiter=',', dtype='float', usecols=[0,1,2,3], encoding='utf-8')\n",
    "print(iris_2d[:4])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 28. 如何计算均值，中位数以及标准差？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[b'5.1' b'3.5' b'1.4' b'0.2' b'Iris-setosa']\n",
      " [b'4.9' b'3.0' b'1.4' b'0.2' b'Iris-setosa']\n",
      " [b'4.7' b'3.2' b'1.3' b'0.2' b'Iris-setosa']\n",
      " [b'4.6' b'3.1' b'1.5' b'0.2' b'Iris-setosa']]\n"
     ]
    }
   ],
   "source": [
    "print(iris[:4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5.1 4.9 4.7 4.6 5.  5.4 4.6 5.  4.4 4.9]\n",
      "[5.1 4.9 4.7 4.6 5.  5.4 4.6 5.  4.4 4.9]\n"
     ]
    }
   ],
   "source": [
    "# 直接通过已经初始化的矩阵获取\n",
    "sepallength = iris[:, 0].astype(float)\n",
    "print(sepallength[:10])\n",
    "\n",
    "# 重新获取数据，且只获取需要的列\n",
    "sepallength = np.genfromtxt(url, delimiter=',', dtype='float', usecols=[0])\n",
    "print(sepallength[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.843333333333334 5.8 0.8253012917851409\n"
     ]
    }
   ],
   "source": [
    "mean, med, sd = np.mean(sepallength), np.median(sepallength), np.std(sepallength)\n",
    "print(mean, med, sd)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 29. 如何归一化矩阵中的数据？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "归一化: (X - min) / (max - min)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "最大值: 7.9\n",
      "最小值: 4.3\n",
      "原数据: \n",
      "[5.1 4.9 4.7 ... 6.5 6.2 5.9]\n",
      "归一化后数据: \n",
      "[0.222222 0.166667 0.111111 ... 0.611111 0.527778 0.444444]\n"
     ]
    }
   ],
   "source": [
    "np.set_printoptions(threshold=10)\n",
    "Smax, Smin = sepallength.max(), sepallength.min()\n",
    "print('最大值: {0}'.format(Smax))\n",
    "print('最小值: {0}'.format(Smin))\n",
    "print('原数据: ')\n",
    "print(sepallength)\n",
    "# 方法1\n",
    "S = (sepallength - Smin)/(Smax - Smin)\n",
    "print('归一化后数据: ')\n",
    "print(S)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.222 0.167 0.111 ... 0.611 0.528 0.444]\n"
     ]
    }
   ],
   "source": [
    "# 方法2 \n",
    "S = (sepallength - Smin)/sepallength.ptp()\n",
    "print(S)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.set_printoptions(threshold=np.nan)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 30. 如何计算softmax的值？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Softmax 在机器学习和深度学习中有着非常广泛的应用。尤其在处理多分类（C > 2）问题，分类器最后的输出单元需要Softmax 函数进行数值处理。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='./image/softmaxf.jpg' />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "下图通过softmax进行3分类："
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='./image/softmax.jpg' />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "实际应用中，使用 Softmax 需要注意数值溢出的问题。因为有指数运算，如果 V 数值很大，经过指数运算后的数值往往可能有溢出的可能。所以，需要对 V 进行一些数值处理：即 V 中的每个元素减去 V 中的最大值。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(x):\n",
    "    \"\"\"Compute softmax values for each sets of scores in x.\n",
    "    https://stackoverflow.com/questions/34968722/how-to-implement-the-softmax-function-in-python\"\"\"\n",
    "    e_x = np.exp(x - np.max(x))\n",
    "#     print(e_x)\n",
    "#     print('e_x.sum(axis=0)')\n",
    "#     print(e_x.sum(axis=0))\n",
    "#     print('e_x.sum()')\n",
    "#     print(e_x.sum())\n",
    "    return e_x / e_x.sum(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "原数据: \n",
      "[5.1 4.9 4.7 4.6 5.  5.4 4.6 5.  4.4 4.9 5.4 4.8 4.8 4.3 5.8 5.7 5.4 5.1\n",
      " 5.7 5.1 5.4 5.1 4.6 5.1 4.8 5.  5.  5.2 5.2 4.7 4.8 5.4 5.2 5.5 4.9 5.\n",
      " 5.5 4.9 4.4 5.1 5.  4.5 4.4 5.  5.1 4.8 5.1 4.6 5.3 5.  7.  6.4 6.9 5.5\n",
      " 6.5 5.7 6.3 4.9 6.6 5.2 5.  5.9 6.  6.1 5.6 6.7 5.6 5.8 6.2 5.6 5.9 6.1\n",
      " 6.3 6.1 6.4 6.6 6.8 6.7 6.  5.7 5.5 5.5 5.8 6.  5.4 6.  6.7 6.3 5.6 5.5\n",
      " 5.5 6.1 5.8 5.  5.6 5.7 5.7 6.2 5.1 5.7 6.3 5.8 7.1 6.3 6.5 7.6 4.9 7.3\n",
      " 6.7 7.2 6.5 6.4 6.8 5.7 5.8 6.4 6.5 7.7 7.7 6.  6.9 5.6 7.7 6.3 6.7 7.2\n",
      " 6.2 6.1 6.4 7.2 7.4 7.9 6.4 6.3 6.1 7.7 6.3 6.4 6.  6.9 6.7 6.9 5.8 6.8\n",
      " 6.7 6.7 6.3 6.5 6.2 5.9]\n",
      "softmax数据: \n",
      "[0.00222  0.001817 0.001488 0.001346 0.002008 0.002996 0.001346 0.002008\n",
      " 0.001102 0.001817 0.002996 0.001644 0.001644 0.000997 0.00447  0.004044\n",
      " 0.002996 0.00222  0.004044 0.00222  0.002996 0.00222  0.001346 0.00222\n",
      " 0.001644 0.002008 0.002008 0.002453 0.002453 0.001488 0.001644 0.002996\n",
      " 0.002453 0.003311 0.001817 0.002008 0.003311 0.001817 0.001102 0.00222\n",
      " 0.002008 0.001218 0.001102 0.002008 0.00222  0.001644 0.00222  0.001346\n",
      " 0.002711 0.002008 0.01484  0.008144 0.013428 0.003311 0.009001 0.004044\n",
      " 0.007369 0.001817 0.009947 0.002453 0.002008 0.00494  0.005459 0.006033\n",
      " 0.003659 0.010994 0.003659 0.00447  0.006668 0.003659 0.00494  0.006033\n",
      " 0.007369 0.006033 0.008144 0.009947 0.01215  0.010994 0.005459 0.004044\n",
      " 0.003311 0.003311 0.00447  0.005459 0.002996 0.005459 0.010994 0.007369\n",
      " 0.003659 0.003311 0.003311 0.006033 0.00447  0.002008 0.003659 0.004044\n",
      " 0.004044 0.006668 0.00222  0.004044 0.007369 0.00447  0.016401 0.007369\n",
      " 0.009001 0.02704  0.001817 0.020032 0.010994 0.018126 0.009001 0.008144\n",
      " 0.01215  0.004044 0.00447  0.008144 0.009001 0.029884 0.029884 0.005459\n",
      " 0.013428 0.003659 0.029884 0.007369 0.010994 0.018126 0.006668 0.006033\n",
      " 0.008144 0.018126 0.022139 0.0365   0.008144 0.007369 0.006033 0.029884\n",
      " 0.007369 0.008144 0.005459 0.013428 0.010994 0.013428 0.00447  0.01215\n",
      " 0.010994 0.010994 0.007369 0.009001 0.006668 0.00494 ]\n",
      "131\n"
     ]
    }
   ],
   "source": [
    "print('原数据: ')\n",
    "print(sepallength)\n",
    "print('softmax数据: ')\n",
    "print(softmax(sepallength))\n",
    "print(np.argmax(softmax(sepallength)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 31. 如何计算array中的百分位数(Percentile)的值？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Percentile：百分位数是统计中使用的度量，表示小于这个值的观察值占总数q的百分比"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5.1 4.9 4.7 4.6 5.  5.4 4.6 5.  4.4 4.9 5.4 4.8 4.8 4.3 5.8 5.7 5.4 5.1\n",
      " 5.7 5.1 5.4 5.1 4.6 5.1 4.8 5.  5.  5.2 5.2 4.7 4.8 5.4 5.2 5.5 4.9 5.\n",
      " 5.5 4.9 4.4 5.1 5.  4.5 4.4 5.  5.1 4.8 5.1 4.6 5.3 5.  7.  6.4 6.9 5.5\n",
      " 6.5 5.7 6.3 4.9 6.6 5.2 5.  5.9 6.  6.1 5.6 6.7 5.6 5.8 6.2 5.6 5.9 6.1\n",
      " 6.3 6.1 6.4 6.6 6.8 6.7 6.  5.7 5.5 5.5 5.8 6.  5.4 6.  6.7 6.3 5.6 5.5\n",
      " 5.5 6.1 5.8 5.  5.6 5.7 5.7 6.2 5.1 5.7 6.3 5.8 7.1 6.3 6.5 7.6 4.9 7.3\n",
      " 6.7 7.2 6.5 6.4 6.8 5.7 5.8 6.4 6.5 7.7 7.7 6.  6.9 5.6 7.7 6.3 6.7 7.2\n",
      " 6.2 6.1 6.4 7.2 7.4 7.9 6.4 6.3 6.1 7.7 6.3 6.4 6.  6.9 6.7 6.9 5.8 6.8\n",
      " 6.7 6.7 6.3 6.5 6.2 5.9]\n"
     ]
    }
   ],
   "source": [
    "print(sepallength)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4.6   7.408]\n"
     ]
    }
   ],
   "source": [
    "print(np.percentile(sepallength, q=[5, 96]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "#help(np.percentile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 32. 如何向array随机位置插入值？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "如向Iris数据中，插入20个空值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "150 5\n",
      "i是行的索引，每个索引重复列的数量构成\n",
      "[0 0 0 0 0 1 1 1 1 1]\n",
      "(750,)\n",
      "j是列的索引: 01234重复150次\n",
      "[0 1 2 3 4 0 1 2 3 4]\n",
      "(750,)\n",
      "method1\n",
      "[[b'5.1' b'3.5' b'1.4' b'0.2' b'Iris-setosa']\n",
      " [b'4.9' b'3.0' b'1.4' b'0.2' b'Iris-setosa']\n",
      " [b'4.7' b'3.2' b'1.3' b'0.2' b'Iris-setosa']\n",
      " [b'4.6' b'3.1' b'1.5' b'0.2' b'Iris-setosa']\n",
      " [b'5.0' b'3.6' b'1.4' b'0.2' b'Iris-setosa']\n",
      " [b'5.4' b'3.9' b'1.7' b'0.4' b'Iris-setosa']\n",
      " [b'4.6' b'3.4' b'1.4' b'0.3' b'Iris-setosa']\n",
      " [b'5.0' b'3.4' b'1.5' b'0.2' b'Iris-setosa']\n",
      " [b'4.4' b'2.9' b'1.4' b'0.2' b'Iris-setosa']\n",
      " [b'4.9' b'3.1' b'1.5' b'0.1' b'Iris-setosa']\n",
      " [b'5.4' b'3.7' b'1.5' nan b'Iris-setosa']\n",
      " [b'4.8' b'3.4' b'1.6' b'0.2' b'Iris-setosa']\n",
      " [b'4.8' b'3.0' b'1.4' b'0.1' b'Iris-setosa']\n",
      " [b'4.3' b'3.0' b'1.1' b'0.1' b'Iris-setosa']\n",
      " [b'5.8' b'4.0' b'1.2' b'0.2' b'Iris-setosa']]\n",
      "method2，刚才的那个方法太繁琐了，\n",
      "[[b'5.1' b'3.5' b'1.4' b'0.2' b'Iris-setosa']\n",
      " [b'4.9' b'3.0' b'1.4' b'0.2' b'Iris-setosa']\n",
      " [b'4.7' b'3.2' b'1.3' b'0.2' b'Iris-setosa']\n",
      " [b'4.6' b'3.1' b'1.5' b'0.2' b'Iris-setosa']\n",
      " [b'5.0' b'3.6' b'1.4' b'0.2' b'Iris-setosa']\n",
      " [b'5.4' b'3.9' b'1.7' b'0.4' b'Iris-setosa']\n",
      " [b'4.6' b'3.4' b'1.4' b'0.3' b'Iris-setosa']\n",
      " [b'5.0' b'3.4' b'1.5' b'0.2' b'Iris-setosa']\n",
      " [b'4.4' b'2.9' nan b'0.2' b'Iris-setosa']\n",
      " [b'4.9' b'3.1' b'1.5' b'0.1' b'Iris-setosa']\n",
      " [b'5.4' b'3.7' b'1.5' b'0.2' b'Iris-setosa']\n",
      " [b'4.8' b'3.4' b'1.6' b'0.2' b'Iris-setosa']\n",
      " [b'4.8' b'3.0' b'1.4' b'0.1' b'Iris-setosa']\n",
      " [b'4.3' b'3.0' b'1.1' b'0.1' b'Iris-setosa']\n",
      " [b'5.8' nan b'1.2' b'0.2' b'Iris-setosa']]\n"
     ]
    }
   ],
   "source": [
    "# Input\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris_2d = np.genfromtxt(url, delimiter=',', dtype='object')\n",
    "print(iris_2d.shape[0], iris_2d.shape[1])\n",
    "# Method 1\n",
    "iris_2d_1 = iris_2d.copy()\n",
    "i, j = np.where(iris_2d_1)\n",
    "print('i是行的索引，每个索引重复列的数量构成')\n",
    "print(i[:10])\n",
    "print(i.shape)\n",
    "print('j是列的索引: 01234重复150次')\n",
    "print(j[:10])\n",
    "print(j.shape)\n",
    "# i, j contain the row numbers and column numbers of 600 elements of iris_x\n",
    "print('method1')\n",
    "np.random.seed(100)\n",
    "iris_2d_1[np.random.choice((i), 20), np.random.choice((j), 20)] = np.nan\n",
    "print(iris_2d_1[:15])\n",
    "\n",
    "# Method 2\n",
    "print('method2，刚才的那个方法太繁琐了，')\n",
    "np.random.seed(100)\n",
    "iris_2d_2 = iris_2d.copy()\n",
    "iris_2d_2[np.random.randint(iris_2d_2.shape[0], size=20), np.random.randint(iris_2d_2.shape[1], size=20)] = np.nan\n",
    "print(iris_2d_2[:15])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "150\n",
      "5\n",
      "[0 1 4 2 3 3 4 2 4 3 1 0 4 3 4 2 2 4 2 4]\n"
     ]
    }
   ],
   "source": [
    "print(iris_2d_2.shape[0])\n",
    "print(iris_2d_2.shape[1])\n",
    "print(np.random.randint(iris_2d_2.shape[1], size=20))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 33. 如何在array中找到空值的位置？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5.1 4.9 4.7 4.6 5.  5.4 4.6 5.  4.4 4.9 5.4 4.8 4.8 4.3 5.8 5.7 5.4 5.1\n",
      " 5.7 5.1 5.4 5.1 4.6 5.1 4.8 5.  5.  5.2 5.2 4.7 4.8 5.4 5.2 5.5 4.9 5.\n",
      " 5.5 4.9 4.4 5.1 5.  4.5 4.4 5.  5.1 4.8 5.1 4.6 5.3 5.  7.  6.4 6.9 5.5\n",
      " 6.5 5.7 6.3 4.9 6.6 5.2 5.  5.9 6.  6.1 5.6 6.7 5.6 5.8 6.2 5.6 5.9 nan\n",
      " 6.3 6.1 6.4 6.6 6.8 6.7 6.  5.7 5.5 5.5 5.8 6.  5.4 6.  6.7 6.3 5.6 5.5\n",
      " 5.5 6.1 5.8 5.  5.6 5.7 5.7 6.2 5.1 5.7 6.3 5.8 nan 6.3 6.5 7.6 4.9 7.3\n",
      " 6.7 7.2 6.5 6.4 6.8 5.7 5.8 6.4 6.5 7.7 7.7 6.  6.9 5.6 nan 6.3 6.7 7.2\n",
      " 6.2 6.1 6.4 7.2 7.4 7.9 6.4 6.3 6.1 7.7 6.3 6.4 6.  6.9 6.7 6.9 5.8 6.8\n",
      " 6.7 6.7 6.3 6.5 6.2 5.9]\n",
      "Number of missing values: \n",
      " 3\n",
      "Position of missing values: \n",
      " (array([ 71, 102, 122], dtype=int64),)\n"
     ]
    }
   ],
   "source": [
    "# Input\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris_2d = np.genfromtxt(url, delimiter=',', dtype='float', usecols=[0,1,2,3])\n",
    "iris_2d[np.random.randint(150, size=20), np.random.randint(4, size=20)] = np.nan\n",
    "print(iris_2d[:, 0])\n",
    "# Solution\n",
    "print(\"Number of missing values: \\n\", np.isnan(iris_2d[:, 0]).sum())\n",
    "print(\"Position of missing values: \\n\", np.where(np.isnan(iris_2d[:, 0])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 34. 如何基于2个或更多的条件查找数据？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[4.8, 3.4, 1.6, 0.2],\n",
       "       [4.8, 3.4, 1.9, 0.2],\n",
       "       [4.7, 3.2, 1.6, 0.2],\n",
       "       [4.8, 3.1, 1.6, 0.2],\n",
       "       [4.9, 2.4, 3.3, 1. ],\n",
       "       [4.9, 2.5, 4.5, 1.7]])"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Input\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris_2d = np.genfromtxt(url, delimiter=',', dtype='float', usecols=[0,1,2,3])\n",
    "\n",
    "# Filter the rows of iris_2d that has petallength (3rd column) > 1.5 and sepallength (1st column) < 5.0\n",
    "# Solution\n",
    "condition = (iris_2d[:, 2] > 1.5) & (iris_2d[:, 0] < 5.0)\n",
    "iris_2d[condition]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 35. 如何移除含有空值的行？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[5.1 3.5 1.4 0.2]\n",
      " [4.9 3.  1.4 0.2]\n",
      " [4.7 3.2 1.3 0.2]\n",
      " [4.6 3.1 1.5 0.2]\n",
      " [5.  3.6 1.4 0.2]]\n",
      "\n",
      "(130, 4)\n",
      "[[5.1 3.5 1.4 0.2]\n",
      " [4.9 3.  1.4 0.2]\n",
      " [4.7 3.2 1.3 0.2]\n",
      " [4.6 3.1 1.5 0.2]\n",
      " [5.  3.6 1.4 0.2]]\n",
      "(20, 4)\n",
      "[[4.4 nan 1.4 0.2]\n",
      " [nan 3.7 1.5 0.2]\n",
      " [5.  3.  1.6 nan]\n",
      " [nan 3.4 1.6 0.4]\n",
      " [nan 3.2 1.2 0.2]]\n"
     ]
    }
   ],
   "source": [
    "# Input\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris_2d = np.genfromtxt(url, delimiter=',', dtype='float', usecols=[0,1,2,3])\n",
    "iris_2d[np.random.randint(150, size=20), np.random.randint(4, size=20)] = np.nan\n",
    "\n",
    "# Solution\n",
    "# No direct numpy function for this.\n",
    "# Method 1:\n",
    "any_nan_in_row = np.array([~np.any(np.isnan(row)) for row in iris_2d])\n",
    "print(iris_2d[any_nan_in_row][:5])\n",
    "print()\n",
    "# Method 2: \n",
    "withnonan = iris_2d[np.sum(np.isnan(iris_2d), axis = 1) == 0]\n",
    "print(withnonan.shape)\n",
    "print(withnonan[:5])\n",
    "\n",
    "withnan = iris_2d[np.sum(np.isnan(iris_2d), axis = 1) > 0]\n",
    "print(withnan.shape)\n",
    "print(withnan[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 36. 如何得到两列的相关系数？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Correlation coef indicates the degree of linear relationship between two numeric variables. <br>\n",
    "It can range between -1 to +1. <br>\n",
    "The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a correlation at least as extreme as the one computed.<br>\n",
    "The lower the p-value (<0.01), stronger is the significance of the relationship.<br>\n",
    "It is not an indicator of the strength."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "相关系数是最早由统计学家卡尔·皮尔逊设计的统计指标，是研究变量之间线性相关程度的量，一般用字母 r 表示。由于研究对象的不同，相关系数有多种定义方式，较为常用的是皮尔逊相关系数。\n",
    "\n",
    "相关表和相关图可反映两个变量之间的相互关系及其相关方向，但无法确切地表明两个变量之间相关的程度。相关系数是用以反映变量之间相关关系密切程度的统计指标。相关系数是按积差方法计算，同样以两变量与各自平均值的离差为基础，通过两个离差相乘来反映两变量之间相关程度；着重研究线性的单相关系数。\n",
    "需要说明的是，皮尔逊相关系数并不是唯一的相关系数，但是最常见的相关系数，以下解释都是针对皮尔逊相关系数。\n",
    "\n",
    "依据相关现象之间的不同特征，其统计指标的名称有所不同。如将反映两变量间线性相关关系的统计指标称为相关系数（相关系数的平方称为判定系数）；将反映两变量间曲线相关关系的统计指标称为非线性相关系数、非线性判定系数；将反映多元线性相关关系的统计指标称为复相关系数、复判定系数等。\n",
    "\n",
    "相关关系是一种非确定性的关系，相关系数是研究变量之间线性相关程度的量。由于研究对象的不同，相关系数有如下几种定义方式。<br>\n",
    "简单相关系数：又叫相关系数或线性相关系数，一般用字母r 表示，用来度量两个变量间的线性关系。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='./image/corelation.jpg' />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "其中，Cov(X,Y)为X与Y的协方差，Var[X]为X的方差，Var[Y]为Y的方差"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8717541573048718\n",
      "0.8717541573048712\n"
     ]
    }
   ],
   "source": [
    "# Input\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris = np.genfromtxt(url, delimiter=',', dtype='float', usecols=[0,1,2,3])\n",
    "\n",
    "# Find the correlation between 花萼长度：SepalLength(1st column) and 花瓣长度：PetalLength(3rd column) in iris_2d\n",
    "# Solution 1\n",
    "print(np.corrcoef(iris[:, 0], iris[:, 2])[0, 1])\n",
    "\n",
    "# Solution 2\n",
    "from scipy.stats.stats import pearsonr  \n",
    "corr, p_value = pearsonr(iris[:, 0], iris[:, 2])\n",
    "print(corr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这个结果说明花萼与花瓣的长度还是强相关的。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 37. 如何判定array含有nan值？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Input\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris_2d = np.genfromtxt(url, delimiter=',', dtype='float', usecols=[0,1,2,3])\n",
    "# iris_2d[np.random.randint(150, size=20), np.random.randint(4, size=20)] = np.nan\n",
    "np.isnan(iris_2d).any()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 38. 如何将array中所有缺失值替换为0？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5.1, 3.5, 1.4, 0.2],\n",
       "       [4.9, 3. , 1.4, 0.2],\n",
       "       [4.7, 3.2, 0. , 0.2],\n",
       "       [4.6, 3.1, 1.5, 0.2],\n",
       "       [5. , 3.6, 1.4, 0.2],\n",
       "       [5.4, 3.9, 1.7, 0.4],\n",
       "       [4.6, 3.4, 1.4, 0.3],\n",
       "       [5. , 3.4, 1.5, 0.2],\n",
       "       [4.4, 2.9, 1.4, 0.2],\n",
       "       [4.9, 3.1, 1.5, 0.1],\n",
       "       [5.4, 3.7, 1.5, 0.2],\n",
       "       [4.8, 3.4, 1.6, 0.2],\n",
       "       [4.8, 3. , 1.4, 0.1],\n",
       "       [4.3, 3. , 1.1, 0.1],\n",
       "       [5.8, 4. , 1.2, 0.2],\n",
       "       [5.7, 4.4, 1.5, 0.4],\n",
       "       [5.4, 3.9, 1.3, 0.4],\n",
       "       [5.1, 0. , 1.4, 0.3],\n",
       "       [5.7, 3.8, 1.7, 0.3],\n",
       "       [5.1, 3.8, 1.5, 0.3],\n",
       "       [5.4, 3.4, 1.7, 0.2],\n",
       "       [5.1, 3.7, 1.5, 0.4],\n",
       "       [4.6, 3.6, 1. , 0.2],\n",
       "       [5.1, 3.3, 1.7, 0.5],\n",
       "       [4.8, 3.4, 1.9, 0.2],\n",
       "       [5. , 3. , 1.6, 0.2],\n",
       "       [5. , 3.4, 1.6, 0.4],\n",
       "       [5.2, 3.5, 1.5, 0.2],\n",
       "       [5.2, 3.4, 1.4, 0.2],\n",
       "       [4.7, 3.2, 1.6, 0.2]])"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Input\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris_2d = np.genfromtxt(url, delimiter=',', dtype='float', usecols=[0,1,2,3])\n",
    "iris_2d[np.random.randint(150, size=20), np.random.randint(4, size=20)] = np.nan\n",
    "\n",
    "# print(np.isnan(iris_2d))\n",
    "# Solution\n",
    "iris_2d[np.isnan(iris_2d)] = 0\n",
    "iris_2d[:30]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 39. 如何从array找出唯一值的数目？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([b'Iris-setosa', b'Iris-versicolor', b'Iris-virginica'],\n",
      "      dtype='|S15'), array([50, 50, 50], dtype=int64))\n"
     ]
    }
   ],
   "source": [
    "# Import iris keeping the text column intact\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris = np.genfromtxt(url, delimiter=',', dtype='object')\n",
    "names = ('sepallength', 'sepalwidth', 'petallength', 'petalwidth', 'species')\n",
    "\n",
    "# Solution\n",
    "# Extract the species column as an array\n",
    "species = np.array([row.tolist()[4] for row in iris])\n",
    "\n",
    "# Get the unique values and the counts\n",
    "print(np.unique(species, return_counts=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 40. 如何将作为枚举的整型数字转为类别文字？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "问题：\n",
    "将花瓣长度（第三列）转为文本分类，比如花瓣长度:\n",
    "<3 : small\n",
    "3~5 : medium\n",
    "\\>=5 : large"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[b'1.4' b'1.4' b'1.3' b'1.5']\n",
      "[1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n",
      " 2 2 2 3 2 2 2 2 2 3 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 3 3 3 3 3 3 2 3 3 3 3\n",
      " 3 3 3 3 3 3 3 3 3 3 2 3 2 3 3 2 2 3 3 3 3 3 3 3 3 3 3 2 3 3 3 3 3 3 3 3 3\n",
      " 3 3]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['small', 'small', 'small', 'small']"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Input\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris = np.genfromtxt(url, delimiter=',', dtype='object')\n",
    "print(iris[:, 2][:4])\n",
    "\n",
    "# Bin petallength \n",
    "petal_length_bin = np.digitize(iris[:, 2].astype('float'), [0, 3, 5, 10])\n",
    "print(petal_length_bin)\n",
    "# Map it to respective category\n",
    "label_map = {1: 'small', 2: 'medium', 3: 'large', 4: np.nan}\n",
    "petal_length_cat = [label_map[x] for x in petal_length_bin]\n",
    "\n",
    "# View\n",
    "petal_length_cat[:4]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 41. 如何基于已经存在的列创建一个新列？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "问题：根据下述公式 (pi x petallength x sepal_length^2)/3创建一个新列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(150,)\n",
      "[38.13265163 35.20049849 30.07237208 33.23805027]\n",
      "(150, 1)\n",
      "[[38.13265163]\n",
      " [35.20049849]\n",
      " [30.07237208]\n",
      " [33.23805027]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[b'5.1', b'3.5', b'1.4', b'0.2', b'Iris-setosa',\n",
       "        38.13265162927291],\n",
       "       [b'4.9', b'3.0', b'1.4', b'0.2', b'Iris-setosa',\n",
       "        35.200498485922445],\n",
       "       [b'4.7', b'3.2', b'1.3', b'0.2', b'Iris-setosa', 30.0723720777127],\n",
       "       [b'4.6', b'3.1', b'1.5', b'0.2', b'Iris-setosa',\n",
       "        33.238050274980004]], dtype=object)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Input\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris_2d = np.genfromtxt(url, delimiter=',', dtype='object')\n",
    "\n",
    "# Solution\n",
    "# Compute volume\n",
    "sepallength = iris_2d[:, 0].astype('float')\n",
    "petallength = iris_2d[:, 2].astype('float')\n",
    "volume = (np.pi * petallength * (sepallength**2))/3\n",
    "print(volume.shape)\n",
    "print(volume[:4])\n",
    "# Introduce new dimension to match iris_2d's\n",
    "volume = volume[:, np.newaxis]\n",
    "print(volume.shape)\n",
    "print(volume[:4])\n",
    "# Add the new column\n",
    "out = np.hstack([iris_2d, volume])\n",
    "\n",
    "# View\n",
    "out[:4]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 42. 如何根据概率随机取样？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array(['Iris-setosa', 'Iris-versicolor', 'Iris-virginica'], dtype='<U15'), array([77, 37, 36], dtype=int64))\n",
      "\n",
      "[0.         0.01020408 0.02040816 0.03061224 0.04081633 0.05102041\n",
      " 0.06122449 0.07142857 0.08163265 0.09183673]\n",
      "\n",
      "[0.54340494 0.27836939 0.42451759 0.84477613 0.00471886 0.12156912\n",
      " 0.67074908 0.82585276 0.13670659 0.57509333]\n",
      "\n",
      "[ 59  28  42 119   1  12  84 115  14  65]\n",
      "(array([b'Iris-setosa', b'Iris-versicolor', b'Iris-virginica'],\n",
      "      dtype=object), array([77, 37, 36], dtype=int64))\n"
     ]
    }
   ],
   "source": [
    "# Import iris keeping the text column intact\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris = np.genfromtxt(url, delimiter=',', dtype='object')\n",
    "\n",
    "# Solution\n",
    "# Get the species column\n",
    "species = iris[:, 4]\n",
    "\n",
    "# 方法 1: Generate Probablistically\n",
    "np.random.seed(100)\n",
    "a = np.array(['Iris-setosa', 'Iris-versicolor', 'Iris-virginica'])\n",
    "species_out = np.random.choice(a, 150, p=[0.5, 0.25, 0.25])\n",
    "print(np.unique(species_out, return_counts=True))\n",
    "print()\n",
    "\n",
    "# 方法 2: Probablistic Sampling (preferred)\n",
    "np.random.seed(100)\n",
    "randomarray = np.random.random(150)\n",
    "probs = np.r_[np.linspace(0, 0.500, num=50), np.linspace(0.501, .750, num=50), np.linspace(.751, 1.0, num=50)]\n",
    "print(probs[:10])\n",
    "# 将probs中的数，逐个与np.random.random(150)的成员对比，找到最适合的值的索引，从而构成排序索引矩阵\n",
    "index = np.searchsorted(probs, randomarray)\n",
    "print()\n",
    "print(randomarray[:10])\n",
    "print()\n",
    "print(index[:10])\n",
    "species_out = species[index]\n",
    "print(np.unique(species_out, return_counts=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 43. 以某一列分组之后，如何找到第二大的数？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.4 1.4 1.3 1.5 1.4 1.7 1.4 1.5 1.4 1.5 1.5 1.6 1.4 1.1 1.2 1.5 1.3 1.4\n",
      " 1.7 1.5 1.7 1.5 1.  1.7 1.9 1.6 1.6 1.5 1.4 1.6 1.6 1.5 1.5 1.4 1.5 1.2\n",
      " 1.3 1.5 1.3 1.5 1.3 1.3 1.3 1.6 1.9 1.4 1.6 1.4 1.5 1.4]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.7"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import iris keeping the text column intact\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris = np.genfromtxt(url, delimiter=',', dtype='object')\n",
    "\n",
    "# Solution\n",
    "# 获得Itis-setosa品种的\n",
    "petal_len_setosa = iris[iris[:, 4] == b'Iris-setosa', [2]].astype('float')\n",
    "print(petal_len_setosa)\n",
    "# Get the second last value\n",
    "np.unique(np.sort(petal_len_setosa))[-2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 44. 如何根据某一列对矩阵排序？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[b'4.3' b'3.0' b'1.1' b'0.1' b'Iris-setosa']\n",
      " [b'4.4' b'3.2' b'1.3' b'0.2' b'Iris-setosa']\n",
      " [b'4.4' b'3.0' b'1.3' b'0.2' b'Iris-setosa']\n",
      " [b'4.4' b'2.9' b'1.4' b'0.2' b'Iris-setosa']\n",
      " [b'4.5' b'2.3' b'1.3' b'0.3' b'Iris-setosa']\n",
      " [b'4.6' b'3.6' b'1.0' b'0.2' b'Iris-setosa']\n",
      " [b'4.6' b'3.1' b'1.5' b'0.2' b'Iris-setosa']\n",
      " [b'4.6' b'3.4' b'1.4' b'0.3' b'Iris-setosa']\n",
      " [b'4.6' b'3.2' b'1.4' b'0.2' b'Iris-setosa']\n",
      " [b'4.7' b'3.2' b'1.3' b'0.2' b'Iris-setosa']\n",
      " [b'4.7' b'3.2' b'1.6' b'0.2' b'Iris-setosa']\n",
      " [b'4.8' b'3.0' b'1.4' b'0.1' b'Iris-setosa']\n",
      " [b'4.8' b'3.0' b'1.4' b'0.3' b'Iris-setosa']\n",
      " [b'4.8' b'3.4' b'1.9' b'0.2' b'Iris-setosa']\n",
      " [b'4.8' b'3.4' b'1.6' b'0.2' b'Iris-setosa']\n",
      " [b'4.8' b'3.1' b'1.6' b'0.2' b'Iris-setosa']\n",
      " [b'4.9' b'2.4' b'3.3' b'1.0' b'Iris-versicolor']\n",
      " [b'4.9' b'2.5' b'4.5' b'1.7' b'Iris-virginica']\n",
      " [b'4.9' b'3.1' b'1.5' b'0.1' b'Iris-setosa']\n",
      " [b'4.9' b'3.1' b'1.5' b'0.1' b'Iris-setosa']]\n"
     ]
    }
   ],
   "source": [
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris = np.genfromtxt(url, delimiter=',', dtype='object')\n",
    "# Sort by column position 0: SepalLength\n",
    "print(iris[iris[:,0].argsort()][:20])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 45. 如何找到矩阵中某列出现频率最多的数值？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[b'1.0' b'1.1' b'1.2' b'1.3' b'1.4' b'1.5' b'1.6' b'1.7' b'1.9' b'3.0'\n",
      " b'3.3' b'3.5' b'3.6' b'3.7' b'3.8' b'3.9' b'4.0' b'4.1' b'4.2' b'4.3'\n",
      " b'4.4' b'4.5' b'4.6' b'4.7' b'4.8' b'4.9' b'5.0' b'5.1' b'5.2' b'5.3'\n",
      " b'5.4' b'5.5' b'5.6' b'5.7' b'5.8' b'5.9' b'6.0' b'6.1' b'6.3' b'6.4'\n",
      " b'6.6' b'6.7' b'6.9'] [ 1  1  2  7 12 14  7  4  2  1  2  2  1  1  1  3  5  3  4  2  4  8  3  5\n",
      "  4  5  4  8  2  2  2  3  6  3  3  2  2  3  1  1  1  2  1]\n",
      "b'1.5'\n"
     ]
    }
   ],
   "source": [
    "# Input:\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris = np.genfromtxt(url, delimiter=',', dtype='object')\n",
    "\n",
    "# Solution:\n",
    "vals, counts = np.unique(iris[:, 2], return_counts=True)\n",
    "print(vals, counts)\n",
    "print(vals[np.argmax(counts)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 46. 如何找到比给定的值大的首次出现的位置?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([50], dtype=int64)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Input:\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris = np.genfromtxt(url, delimiter=',', dtype='object')\n",
    "\n",
    "# Solution: 第四列花瓣宽度 >1.0的第一个位置\n",
    "np.argwhere(iris[:, 3].astype(float) > 1.0)[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 47. 如何将矩阵中大于或小于某些数值的值，替换为固定值？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[27.63 14.64 21.8  42.39  1.23  6.96 33.87 41.47  7.7  29.18 44.67 11.25\n",
      " 10.08  6.31 11.77 48.95 40.77  9.43 41.   14.43]\n",
      "\n",
      "[27.63 14.64 21.8  30.   10.   10.   30.   30.   10.   29.18 30.   11.25\n",
      " 10.08 10.   11.77 30.   30.   10.   30.   14.43]\n",
      "\n",
      "[27.63 14.64 21.8  30.    1.23  6.96 30.   30.    7.7  29.18 30.   11.25\n",
      " 10.08  6.31 11.77 30.   30.    9.43 30.   14.43]\n",
      "\n",
      "[27.63 14.64 21.8  30.   10.   10.   30.   30.   10.   29.18 30.   11.25\n",
      " 10.08 10.   11.77 30.   30.   10.   30.   14.43]\n",
      "\n",
      "[27.63 14.64 21.8  30.   10.   10.   30.   30.   10.   29.18 30.   11.25\n",
      " 10.08 10.   11.77 30.   30.   10.   30.   14.43]\n"
     ]
    }
   ],
   "source": [
    "# Input\n",
    "np.set_printoptions(precision=2)\n",
    "np.random.seed(100)\n",
    "# 1~50，随机生成20个数\n",
    "a = np.random.uniform(1,50, 20)\n",
    "print(a)\n",
    "print()\n",
    "# Solution 1: Using np.clip\n",
    "print(np.clip(a, a_min=10, a_max=30))\n",
    "print()\n",
    "# Solution 2: Using np.where\n",
    "handle30 = np.where(a > 30, 30, a)\n",
    "print(handle30)\n",
    "print()\n",
    "print(np.where(a < 10, 10,handle30))\n",
    "# 简洁写法\n",
    "print()\n",
    "print(np.where(a < 10, 10, np.where(a > 30, 30, a)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 48. 如何从给定的array中获取最大的前5个数？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[27.63 14.64 21.8  42.39  1.23  6.96 33.87 41.47  7.7  29.18 44.67 11.25\n",
      " 10.08  6.31 11.77 48.95 40.77  9.43 41.   14.43]\n",
      "\n",
      "[15 10  3  7 18]\n",
      "[15 10  3  7 18]\n",
      "[48.95 44.67 42.39 41.47 41.  ]\n",
      "\n",
      "升序排序最大的5个数\n",
      "[41.   41.47 42.39 44.67 48.95]\n",
      "[41.   41.47 42.39 44.67 48.95]\n",
      "\n",
      "降序排序最大的5个数\n",
      "[48.95 44.67 42.39 41.47 41.  ]\n"
     ]
    }
   ],
   "source": [
    "# Input\n",
    "np.random.seed(100)\n",
    "a = np.random.uniform(1,50, 20)\n",
    "print(a)\n",
    "print()\n",
    "# Solution:\n",
    "print(a.argsort()[::-1][:5])\n",
    "\n",
    "# Solution 2:\n",
    "print(np.argpartition(-a, 5)[:5])\n",
    "\n",
    "# Below methods will get you the values.\n",
    "# Method 1:\n",
    "print(a[a.argsort()[::-1]][:5])\n",
    "print()\n",
    "print('升序排序最大的5个数')\n",
    "# Method 2:\n",
    "print(np.sort(a)[-5:])\n",
    "\n",
    "# Method 3:\n",
    "print(np.partition(a, kth=-5)[-5:])\n",
    "print()\n",
    "print('降序排序最大的5个数')\n",
    "# Method 4:\n",
    "print(a[np.argpartition(-a, 5)][:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 49. 如何以行级别，获取1至10个数出现频次的计数？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 9  9  4  8  8  1  5  3  6  3]\n",
      " [ 3  3  2  1  9  5  1 10  7  3]\n",
      " [ 5  2  6  4  5  5  4  8  2  2]\n",
      " [ 8  8  1  3 10 10  4  3  6  9]\n",
      " [ 2  1  8  7  3  1  9  3  6  2]\n",
      " [ 9  2  6  5  3  9  4  6  1 10]]\n",
      "\n",
      "[ 1  2  3  4  5  6  7  8  9 10]\n",
      "[(array([1, 3, 4, 5, 6, 8, 9]), array([1, 2, 1, 1, 1, 2, 2], dtype=int64)), (array([ 1,  2,  3,  5,  7,  9, 10]), array([2, 1, 3, 1, 1, 1, 1], dtype=int64)), (array([2, 4, 5, 6, 8]), array([3, 2, 3, 1, 1], dtype=int64)), (array([ 1,  3,  4,  6,  8,  9, 10]), array([1, 2, 1, 1, 2, 1, 2], dtype=int64)), (array([1, 2, 3, 6, 7, 8, 9]), array([2, 2, 2, 1, 1, 1, 1], dtype=int64)), (array([ 1,  2,  3,  4,  5,  6,  9, 10]), array([1, 1, 1, 1, 1, 2, 2, 1], dtype=int64))]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[[1, 0, 2, 1, 1, 1, 0, 2, 2, 0],\n",
       " [2, 1, 3, 0, 1, 0, 1, 0, 1, 1],\n",
       " [0, 3, 0, 2, 3, 1, 0, 1, 0, 0],\n",
       " [1, 0, 2, 1, 0, 1, 0, 2, 1, 2],\n",
       " [2, 2, 2, 0, 0, 1, 1, 1, 1, 0],\n",
       " [1, 1, 1, 1, 1, 2, 0, 0, 2, 1]]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Input:\n",
    "np.random.seed(100)\n",
    "arr = np.random.randint(1,11,size=(6, 10))\n",
    "print(arr)\n",
    "print()\n",
    "# Solution\n",
    "def counts_of_all_values_rowwise(arr2d):\n",
    "    # Unique values and its counts row wise\n",
    "    num_counts_array = [np.unique(row, return_counts=True) for row in arr2d]\n",
    "    print(num_counts_array)\n",
    "    # Counts of all values row wise\n",
    "    return([[int(b[a==i]) if i in a else 0 for i in np.unique(arr2d)] for a, b in num_counts_array])\n",
    "\n",
    "# Print\n",
    "print(np.arange(1,11))\n",
    "counts_of_all_values_rowwise(arr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 50. 如何将多维array转换为1维？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array_of_arrays:  [array([0, 1, 2]) array([3, 4, 5, 6]) array([7, 8, 9])]\n",
      "\n",
      "[0 1 2 3 4 5 6 7 8 9]\n",
      "\n",
      "[0 1 2 3 4 5 6 7 8 9]\n"
     ]
    }
   ],
   "source": [
    "# Input:\n",
    "arr1 = np.arange(3)\n",
    "arr2 = np.arange(3,7)\n",
    "arr3 = np.arange(7,10)\n",
    "\n",
    "array_of_arrays = np.array([arr1, arr2, arr3])\n",
    "print('array_of_arrays: ', array_of_arrays)\n",
    "print()\n",
    "# Solution 1\n",
    "arr_2d = np.array([a for arr in array_of_arrays for a in arr])\n",
    "print(arr_2d)\n",
    "print()\n",
    "# Solution 2:\n",
    "arr_2d = np.concatenate(array_of_arrays)\n",
    "print(arr_2d)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 51. 如何将数据转换为one-hot编码？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "one-hot编码\n",
    "分类变量（定量特征）与连续变量（定性特征）。我们训练模型的变量，一般分为两种形式。以年收入增长率为例，如果取值为0-1之间任意数，则此时变量为连续变量。如果把增长率进行分段处理，表示成如下形式：[0,0.3],(0.3,0.6],(0.6,1]，那么此时变量为分类变量。\n",
    "\n",
    "特征转换。对于分类变量，建模时要进行转换，通常直接转换为数字。比如将[0,0.3],(0.3,0.6],(0.6,1]表示为0,1,2。原因主要有两点：\n",
    "\n",
    "1,转换后可以提高模型运算效率。\n",
    "\n",
    "2,对于一些模型，比如逻辑回归或计算距离时，无法对分类值直接进行计算。\n",
    "\n",
    "直接转换为数字，也会带来一些问题：\n",
    "\n",
    "1,转换为数字后，默认为连续变量，违背最初设计，影响效率。\n",
    "\n",
    "2,转换后的值会影响同一特征在样本中的权重。比如转换为1000和转换为1对模型影响明显不同。\n",
    "\n",
    "因此，需要更好的编码方式对特征进行转换。\n",
    "\n",
    "one-hot编码。one-hot编码的定义是用N位状态寄存器来对N个状态进行编码。比如上面的例子[0,0.3],(0.3,0.6],(0.6,1]，有3个分类值，因此N为3，对应的one-hot编码可以表示为100,010,001。\n",
    "\n",
    "使用步骤。比如用LR算法做模型，在数据处理过程中，可以先对连续变量进行离散化处理，然后对离散化后数据进行one-hot编码，最后放入LR模型中。这样可以增强模型的非线性能力。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2 3 2 2 2 1]\n",
      "\n",
      "[[0. 1. 0.]\n",
      " [0. 0. 1.]\n",
      " [0. 1. 0.]\n",
      " [0. 1. 0.]\n",
      " [0. 1. 0.]\n",
      " [1. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "# Input:\n",
    "np.random.seed(101) \n",
    "arr = np.random.randint(1,4, size=6)\n",
    "print(arr)\n",
    "print()\n",
    "\n",
    "# Solution:\n",
    "# 如果需要one-hot的成员有m个，而这些成员的unique的值有n个\n",
    "# 那么one-hot的矩阵就是m x n的\n",
    "# 矩阵中，每一行的1是这样计算的，比如第一行对应数字2，唯一值的向量为[1 2 3]，\n",
    "# 则1的索引位于1，即构成了[0 1 0]的行向量\n",
    "# 以此类推\n",
    "def one_hot_encodings(arr):\n",
    "    uniqs = np.unique(arr)\n",
    "    out = np.zeros((arr.shape[0], uniqs.shape[0]))\n",
    "    for i, k in enumerate(arr):\n",
    "        out[i, k-1] = 1\n",
    "    return out\n",
    "\n",
    "print(one_hot_encodings(arr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 52. 如何创建基于分类的分组行号？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "如基于鸢尾花分类名称，作分组行号"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Iris-setosa' 'Iris-setosa' 'Iris-setosa' 'Iris-setosa' 'Iris-setosa'\n",
      " 'Iris-versicolor' 'Iris-versicolor' 'Iris-versicolor' 'Iris-versicolor'\n",
      " 'Iris-versicolor' 'Iris-versicolor' 'Iris-versicolor' 'Iris-versicolor'\n",
      " 'Iris-versicolor' 'Iris-virginica' 'Iris-virginica' 'Iris-virginica'\n",
      " 'Iris-virginica' 'Iris-virginica' 'Iris-virginica']\n"
     ]
    }
   ],
   "source": [
    "# Input:\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "species = np.genfromtxt(url, delimiter=',', dtype='str', usecols=4)\n",
    "np.random.seed(100)\n",
    "species_small = np.sort(np.random.choice(species, size=20))\n",
    "print(species_small)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Iris-setosa', 'Iris-versicolor', 'Iris-virginica']\n",
      "[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6]\n"
     ]
    }
   ],
   "source": [
    "print([val for val in np.unique(species_small)])\n",
    "print([i+1 for val in np.unique(species_small) for i, grp in enumerate(species_small[species_small==val])])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 53. 如何基于给定的分类创建分组编号？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Iris-setosa' 'Iris-setosa' 'Iris-setosa' 'Iris-setosa' 'Iris-setosa'\n",
      " 'Iris-versicolor' 'Iris-versicolor' 'Iris-versicolor' 'Iris-versicolor'\n",
      " 'Iris-versicolor' 'Iris-versicolor' 'Iris-versicolor' 'Iris-versicolor'\n",
      " 'Iris-versicolor' 'Iris-virginica' 'Iris-virginica' 'Iris-virginica'\n",
      " 'Iris-virginica' 'Iris-virginica' 'Iris-virginica']\n"
     ]
    }
   ],
   "source": [
    "# Input:\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "species = np.genfromtxt(url, delimiter=',', dtype='str', usecols=4)\n",
    "np.random.seed(100)\n",
    "species_small = np.sort(np.random.choice(species, size=20))\n",
    "print(species_small)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2]\n",
      "\n",
      "['Iris-setosa' 'Iris-versicolor' 'Iris-virginica']\n",
      "\n",
      "[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2]\n"
     ]
    }
   ],
   "source": [
    "# Solution:\n",
    "output = [np.argwhere(np.unique(species_small) == s).tolist()[0][0] \n",
    "          for val in np.unique(species_small) \n",
    "          for s in species_small[species_small==val]]\n",
    "print(output)\n",
    "print()\n",
    "# Solution: For Loop version\n",
    "output = []\n",
    "uniqs = np.unique(species_small)\n",
    "print(uniqs)\n",
    "print()\n",
    "for val in uniqs:  # uniq values in group\n",
    "    for s in species_small[species_small==val]:  # each element in group\n",
    "        groupid = np.argwhere(uniqs == s)[0][0]  # groupid\n",
    "        output.append(groupid)\n",
    "\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 54. 如何对向量作从小到大顺序排名？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Array:  [ 9  4 15  0 17 16 17  8  9  0]\n",
      "[4 2 6 0 8 7 9 3 5 1]\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(10)\n",
    "a = np.random.randint(20, size=10)\n",
    "print('Array: ', a)\n",
    "\n",
    "# Solution\n",
    "print(a.argsort().argsort())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 55. 如何对多维度的array中的成员作从小到大的顺序排名？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 9  4 15  0 17]\n",
      " [16 17  8  9  0]]\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(10)\n",
    "a = np.random.randint(20, size=[2,5])\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "原矩阵\n",
      "[[ 9  4 15  0 17]\n",
      " [16 17  8  9  0]]\n",
      "矩阵降为1维向量\n",
      "[ 9  4 15  0 17 16 17  8  9  0]\n",
      "给出从小到大数据成员的索引向量\n",
      "[3 9 1 7 0 8 2 5 4 6]\n",
      "对索引向量再从小到大排序，并给出对应的向量，这个就是排名的动作\n",
      "[4 2 6 0 8 7 9 3 5 1]\n",
      "reshape为原矩阵的行与列数目\n",
      "[[4 2 6 0 8]\n",
      " [7 9 3 5 1]]\n"
     ]
    }
   ],
   "source": [
    "# Solution\n",
    "print('原矩阵')\n",
    "print(a)\n",
    "print('矩阵降为1维向量')\n",
    "print(a.ravel())\n",
    "print('给出从小到大数据成员的索引向量')\n",
    "print(a.ravel().argsort())\n",
    "print('对索引向量再从小到大排序，并给出对应的向量，这个就是排名的动作')\n",
    "print(a.ravel().argsort().argsort())\n",
    "print('reshape为原矩阵的行与列数目')\n",
    "print(a.ravel().argsort().argsort().reshape(a.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 56. 如何在一个二维矩阵找到每行的最大值？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[9 9 4]\n",
      " [8 8 1]\n",
      " [5 3 6]\n",
      " [3 3 3]\n",
      " [2 1 9]]\n",
      "[9 8 6 3 9]\n",
      "[9 8 6 3 9]\n"
     ]
    }
   ],
   "source": [
    "# Input\n",
    "np.random.seed(100)\n",
    "a = np.random.randint(1,10, [5,3])\n",
    "print(a)\n",
    "\n",
    "# Solution 1\n",
    "print(np.amax(a, axis=1))\n",
    "\n",
    "# Solution 2\n",
    "print(np.apply_along_axis(np.max, arr=a, axis=1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 57. 如何在一个二维矩阵对每行做到min-by-max？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "即对每行求min/max的值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[9 9 4]\n",
      " [8 8 1]\n",
      " [5 3 6]\n",
      " [3 3 3]\n",
      " [2 1 9]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.44444444, 0.125     , 0.5       , 1.        , 0.11111111])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Input\n",
    "np.random.seed(100)\n",
    "a = np.random.randint(1,10, [5,3])\n",
    "print(a)\n",
    "\n",
    "# Solution\n",
    "np.apply_along_axis(lambda x: np.min(x)/np.max(x), arr=a, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 58. 如何在array发现重复的记录？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 3 0 2 4 2 2 2 2]\n",
      "\n",
      "[ True  True  True  True  True  True  True  True  True  True]\n",
      "(array([0, 2, 3, 4]), array([0, 4, 2, 5], dtype=int64))\n",
      "\n",
      "[0 4 2 5]\n",
      "\n",
      "[False  True False  True False False  True  True  True  True]\n"
     ]
    }
   ],
   "source": [
    "# Input\n",
    "np.random.seed(100)\n",
    "a = np.random.randint(0, 5, 10)\n",
    "print(a)\n",
    "print()\n",
    "## Solution\n",
    "# Create an all True array\n",
    "out = np.full(a.shape[0], True)\n",
    "print(out)\n",
    "# Find the index positions of unique elements\n",
    "unique_positions = np.unique(a, return_index=True)[1]\n",
    "\n",
    "print()\n",
    "print(unique_positions)\n",
    "print()\n",
    "# Mark those positions as False\n",
    "out[unique_positions] = False\n",
    "\n",
    "print(out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 59. 如何求分组后的均值？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[b'Iris-setosa' b'3.5']\n",
      " [b'Iris-setosa' b'3.0']\n",
      " [b'Iris-setosa' b'3.2']\n",
      " [b'Iris-setosa' b'3.1']\n",
      " [b'Iris-setosa' b'3.6']\n",
      " [b'Iris-setosa' b'3.9']\n",
      " [b'Iris-setosa' b'3.4']\n",
      " [b'Iris-setosa' b'3.4']\n",
      " [b'Iris-setosa' b'2.9']\n",
      " [b'Iris-setosa' b'3.1']]\n",
      "[[b'Iris-setosa', 3.418], [b'Iris-versicolor', 2.7700000000000005], [b'Iris-virginica', 2.974]]\n",
      "[[b'Iris-setosa', 3.418], [b'Iris-versicolor', 2.7700000000000005], [b'Iris-virginica', 2.974]]\n"
     ]
    }
   ],
   "source": [
    "# Input\n",
    "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'\n",
    "iris = np.genfromtxt(url, delimiter=',', dtype='object')\n",
    "names = ('sepallength', 'sepalwidth', 'petallength', 'petalwidth', 'species')\n",
    "\n",
    "\n",
    "# Solution\n",
    "# No direct way to implement this. Just a version of a workaround.\n",
    "# 获得根据分组求均值的列\n",
    "numeric_column = iris[:, 1].astype('float')  # sepalwidth\n",
    "# 获得分组依据列，前提在于无空值，且与求均值的列的行数一致\n",
    "grouping_column = iris[:, 4]  # species\n",
    "\n",
    "# 输入的示例：\n",
    "pairvalue = []\n",
    "for specie, value in zip(iris[:, 4], iris[:, 1]):\n",
    "    pairvalue.append([specie, value])\n",
    "print(np.array(pairvalue)[:10])\n",
    "\n",
    "# List comprehension version\n",
    "print([[group_val, numeric_column[grouping_column==group_val].mean()] for group_val in np.unique(grouping_column)])\n",
    "\n",
    "# For Loop version\n",
    "output = []\n",
    "for group_val in np.unique(grouping_column):\n",
    "    output.append([group_val, numeric_column[grouping_column==group_val].mean()])\n",
    "\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 60. 如何将图片转为numpy array?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='https://upload.wikimedia.org/wikipedia/commons/8/8b/Denali_Mt_McKinley.jpg' />"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from io import BytesIO\n",
    "from PIL import Image\n",
    "import PIL, requests\n",
    "\n",
    "# Import image from URL\n",
    "URL = 'https://upload.wikimedia.org/wikipedia/commons/8/8b/Denali_Mt_McKinley.jpg'\n",
    "response = requests.get(URL)\n",
    "\n",
    "# Read it as Image\n",
    "I = Image.open(BytesIO(response.content))\n",
    "\n",
    "# Optionally resize\n",
    "I = I.resize([150,150])\n",
    "\n",
    "# Convert to numpy array\n",
    "arr = np.asarray(I)\n",
    "# print(arr[:2])\n",
    "# Optionaly Convert it back to an image and show\n",
    "im = PIL.Image.fromarray(np.uint8(arr))\n",
    "Image.Image.show(im)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 61. 如何移除所有array中的空值？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 2., 3., 5., 6., 7.])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.array([1,2,3,np.nan,5,6,7,np.nan])\n",
    "a[~np.isnan(a)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 62. 如何计算两个array的欧式距离？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "欧几里得度量（euclidean metric）（也称欧氏距离）是一个通常采用的距离定义，指在m维空间中两个点之间的真实距离，或者向量的自然长度（即该点到原点的距离）。在二维和三维空间中的欧氏距离就是两点之间的实际距离。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='./image/eudistance.jpg' />"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.708203932499369"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Input\n",
    "a = np.array([1,2,3,4,5])\n",
    "b = np.array([4,5,6,7,8])\n",
    "\n",
    "# Solution\n",
    "dist = np.linalg.norm(a-b)\n",
    "dist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 63. 如何求得一维向量中的局部极值的位置？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 2  4 -6  1  4 -6  1]\n",
      "[ 0 -2  2  0 -2  2]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([2, 5], dtype=int64)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.array([1, 3, 7, 1, 2, 6, 0, 1])\n",
    "print(np.diff(a))\n",
    "doublediff = np.diff(np.sign(np.diff(a)))\n",
    "print(doublediff)\n",
    "peak_locations = np.where(doublediff == -2)[0] + 1\n",
    "peak_locations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 64. 已知一维向量，如何用二维矩阵的每一行分别减去一维向量对应的各个数值？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1]\n",
      " [2]\n",
      " [3]]\n",
      "[[2 2 2]\n",
      " [2 2 2]\n",
      " [2 2 2]]\n"
     ]
    }
   ],
   "source": [
    "# Input\n",
    "a_2d = np.array([[3,3,3],[4,4,4],[5,5,5]])\n",
    "# 如a_2d，第一行减1，第二行减2，第三行减3\n",
    "b_1d = np.array([1,2,3])\n",
    "\n",
    "print(b_1d[:, None])\n",
    "# Solution\n",
    "print(a_2d - b_1d[:,None])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 65. 如何从一个array找到某数第N个重复数的索引？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "# 找到数字1，第五次重复位置的索引\n",
    "x = np.array([1, 2, 1, 1, 3, 4, 3, 1, 1, 2, 1, 1, 2])\n",
    "n = 5\n",
    "\n",
    "# Solution 1: List comprehension\n",
    "print([i for i, v in enumerate(x) if v == 1][n-1])\n",
    "\n",
    "# Solution 2: Numpy version\n",
    "print(np.where(x == 1)[0][n-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 66. 如何将numpy的日期时间对象转换为python的日期时间对象？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-02-25 22:10:10\n",
      "2018-02-25 22:10:10\n"
     ]
    }
   ],
   "source": [
    "# Input: a numpy datetime64 object\n",
    "dt64 = np.datetime64('2018-02-25 22:10:10')\n",
    "\n",
    "# Solution\n",
    "from datetime import datetime\n",
    "print(dt64.tolist())\n",
    "\n",
    "# or\n",
    "\n",
    "print(dt64.astype(datetime))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 67. 如何计算array的移动平均值？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='./image/moveaverage.jpg' />"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array:  [8 8 3 7 7 0 4 2 5 2]\n",
      "cumsum is\n",
      "[ 8. 16. 19. 26. 33. 33. 37. 39. 44. 46.]\n",
      "ret[3:] is ret[3:]: [26. 33. 33. 37. 39. 44. 46.] - ret[:-n]:[ 8. 16. 19. 26. 33. 33. 37.]\n",
      "ret is [ 8. 16. 19. 18. 17. 14. 11.  6. 11.  9.]\n",
      "ret[n:] is [18. 17. 14. 11.  6. 11.  9.]\n",
      "ret[n-1:] is [19. 18. 17. 14. 11.  6. 11.  9.]\n",
      "[6.33 6.   5.67 4.67 3.67 2.   3.67 3.  ]\n",
      "[6.33 6.   5.67 4.67 3.67 2.   3.67 3.  ]\n"
     ]
    }
   ],
   "source": [
    "# Solution\n",
    "# Source: https://stackoverflow.com/questions/14313510/how-to-calculate-moving-average-using-numpy\n",
    "def moving_average(a, n=3) :\n",
    "    ret = np.cumsum(a, dtype=float)\n",
    "    print('cumsum is')\n",
    "    print(ret)\n",
    "    print('ret[3:] is ret[3:]: {0} - ret[:-n]:{1}'.format(ret[n:], ret[:-n]))\n",
    "    ret[n:] = ret[n:] - ret[:-n]\n",
    "    print('ret is {0}'.format(ret))\n",
    "    print('ret[n:] is {0}'.format(ret[n:]))\n",
    "    print('ret[n-1:] is {0}'.format(ret[n-1:]))\n",
    "    return ret[n - 1:] / n\n",
    "\n",
    "np.random.seed(100)\n",
    "Z = np.random.randint(10, size=10)\n",
    "print('array: ', Z)\n",
    "# Method 1\n",
    "print(moving_average(Z, n=3).round(2))\n",
    "\n",
    "np.set_printoptions(precision=2)\n",
    "# Method 2: \n",
    "# np.ones(3)/3 gives equal weights. Use np.ones(4)/4 for window size 4.\n",
    "print(np.convolve(Z, np.ones(3)/3, mode='valid'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 68. 如何创建一个给定开始值，给定等差，给定数量的array？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 5,  8, 11, 14, 17, 20, 23, 26, 29, 32])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "length = 10\n",
    "start = 5\n",
    "step = 3\n",
    "\n",
    "def seq(start, length, step):\n",
    "    end = start + (step*length)\n",
    "    return np.arange(start, end, step)\n",
    "\n",
    "seq(start, length, step)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 69. 如何填充不规则序列的numpy的日期对象？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['2018-02-01' '2018-02-03' '2018-02-05' '2018-02-07' '2018-02-09'\n",
      " '2018-02-11' '2018-02-13' '2018-02-15' '2018-02-17' '2018-02-19'\n",
      " '2018-02-21' '2018-02-23']\n",
      "['2018-02-01' '2018-02-02' '2018-02-03' '2018-02-04' '2018-02-05'\n",
      " '2018-02-06' '2018-02-07' '2018-02-08' '2018-02-09' '2018-02-10'\n",
      " '2018-02-11' '2018-02-12' '2018-02-13' '2018-02-14' '2018-02-15'\n",
      " '2018-02-16' '2018-02-17' '2018-02-18' '2018-02-19' '2018-02-20'\n",
      " '2018-02-21' '2018-02-22' '2018-02-23']\n",
      "['2018-02-01' '2018-02-02' '2018-02-03' '2018-02-04' '2018-02-05'\n",
      " '2018-02-06' '2018-02-07' '2018-02-08' '2018-02-09' '2018-02-10'\n",
      " '2018-02-11' '2018-02-12' '2018-02-13' '2018-02-14' '2018-02-15'\n",
      " '2018-02-16' '2018-02-17' '2018-02-18' '2018-02-19' '2018-02-20'\n",
      " '2018-02-21' '2018-02-22' '2018-02-23']\n"
     ]
    }
   ],
   "source": [
    "# Input\n",
    "dates = np.arange(np.datetime64('2018-02-01'), np.datetime64('2018-02-25'), 2)\n",
    "print(dates)\n",
    "\n",
    "# Solution ---------------\n",
    "filled_in = np.array([np.arange(date, (date+d)) for date, d in zip(dates, np.diff(dates))]).reshape(-1)\n",
    "\n",
    "# add the last day\n",
    "output = np.hstack([filled_in, dates[-1]])\n",
    "print(output)\n",
    "\n",
    "# For loop version -------\n",
    "out = []\n",
    "for date, d in zip(dates, np.diff(dates)):\n",
    "    out.append(np.arange(date, (date+d)))\n",
    "\n",
    "filled_in = np.array(out).reshape(-1)\n",
    "\n",
    "# add the last day\n",
    "output = np.hstack([filled_in, dates[-1]])\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 70. 如何创建列步长都为1，行步长不定长(strides)的矩阵？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0  1  2  3]\n",
      " [ 2  3  4  5]\n",
      " [ 4  5  6  7]\n",
      " [ 6  7  8  9]\n",
      " [ 8  9 10 11]\n",
      " [10 11 12 13]]\n",
      "\n",
      "[[ 0  1  2  3  4]\n",
      " [ 4  5  6  7  8]\n",
      " [ 8  9 10 11 12]]\n",
      "\n",
      "[[ 0  1  2  3  4]\n",
      " [ 5  6  7  8  9]\n",
      " [10 11 12 13 14]]\n"
     ]
    }
   ],
   "source": [
    "def gen_strides(a, stride_len=5, window_len=5):\n",
    "    n_strides = ((a.size-window_len)//stride_len) + 1\n",
    "    # return np.array([a[s:(s+window_len)] for s in np.arange(0, a.size, stride_len)[:n_strides]])\n",
    "    return np.array([a[s:(s+window_len)] for s in np.arange(0, n_strides*stride_len, stride_len)])\n",
    "\n",
    "print(gen_strides(np.arange(15), stride_len=2, window_len=4))\n",
    "print()\n",
    "print(gen_strides(np.arange(15), stride_len=4))\n",
    "print()\n",
    "print(gen_strides(np.arange(15)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}