examples/use_case_ipfs.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1: configuration"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Input your walletPrivateKey here:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "check env.walletPrivateKey: 2a776\n",
      "check env.mindLakeAppKey: Kyunq\n",
      "check env.MINDLAKE_GATEWAY: https://sdk.mindnetwork.xyz/node\n"
     ]
    }
   ],
   "source": [
    "# make sure you configure mindlakesdk, please check env.py in the same folder\n",
    "# please familar these configure, and you can read https://github.com/mind-network/mind-lake-sdk-python \n",
    "import env\n",
    "\n",
    "print(\"check env.walletPrivateKey:\", env.walletPrivateKey[0:5])\n",
    "print(\"check env.mindLakeAppKey:\", env.mindLakeAppKey[0:5])\n",
    "print(\"check env.MINDLAKE_GATEWAY:\", env.MINDLAKE_GATEWAY)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Install depedancy and source code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# please \"pip install mindlakesdk\" if not installed\n",
    "import mindlakesdk\n",
    "\n",
    "# please \"pip install minddatapack\" if not installed, or git clone the source code\n",
    "from minddatapack import DataPack\n",
    "\n",
    "# please \"pip install pandas\" if not installed, it is used to display structure content in the notebook\n",
    "import pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check if example_data.csv file exists in the same folder, this demo will start to load this csv file.\n",
    "\n",
    "# check IPFS service is running, if not, please start it by \"ipfs daemon\" in command line or start it in the IPFS desktop app"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2. Local CSV -> MindLake\n",
    "- Load data from a local CSV file without the metadata file, by defining each column manually.\n",
    "- You can also load from a local backup into MindLake once you are familar, then no need to define each column. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>wallet_address</th>\n",
       "      <th>register_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>123</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>124</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>125</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid                              wallet_address               register_date\n",
       "0  123  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7  2023-07-15 02:25:32.392441\n",
       "1  124  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7  2023-07-15 02:25:32.392441\n",
       "2  125  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7  2023-07-15 02:25:32.392441"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn_local_csv = './example_data.csv'\n",
    "\n",
    "# preview local csv\n",
    "df = pandas.read_csv(fn_local_csv)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>123</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>124</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>125</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0                                           1                          2\n",
       "0  123  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441\n",
       "1  124  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441\n",
       "2  125  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataPack1 = DataPack(env.walletPrivateKey)\n",
    "localFile1 = DataPack.LocalFileConnector(columns=[\n",
    "        DataPack.Column('uid', DataPack.DataType.int4, False),\n",
    "        DataPack.Column('wallet_address', DataPack.DataType.text, True),\n",
    "        DataPack.Column('register_date', DataPack.DataType.timestamp, True)\n",
    "    ])\n",
    "result = dataPack1.loadFrom(localFile1, fn_local_csv)\n",
    "assert result, result.message\n",
    "df = pandas.DataFrame(dataPack1.data)\n",
    "df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Save the data into a table in MindLake"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dropTable result: Success\n",
      "a_new_table has been saved to MindLake.\n"
     ]
    }
   ],
   "source": [
    "mindlake = mindlakesdk.connect(env.walletPrivateKey, env.mindLakeAppKey, env.MINDLAKE_GATEWAY)\n",
    "assert mindlake, mindlake.message\n",
    "\n",
    "# drop the table if exists\n",
    "result = mindlake.datalake.dropTable('a_new_table')\n",
    "print(\"dropTable result:\", result.message)\n",
    "\n",
    "# upload from local csv into mindlake\n",
    "mindLakeConnector = DataPack.MindLakeConnector(mindlake)\n",
    "result = dataPack1.saveTo(mindLakeConnector, 'a_new_table')\n",
    "assert result, result.message\n",
    "print(\"a_new_table has been saved to MindLake.\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "you can also check this new downloaded table in https://scan.mindnetwork.xyz/account/myData"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3. MindLake -> IPFS\n",
    "- Load data from a table in MindLake and save to IPFS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>123</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>124</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>125</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0                                           1                          2\n",
       "0  123  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441\n",
       "1  124  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441\n",
       "2  125  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataPack2 = DataPack(env.walletPrivateKey)\n",
    "# make sure if you complete Step 1, so \"a_new_table\" exists in mindlake\n",
    "result = dataPack2.loadFrom(mindLakeConnector, 'select * from \"a_new_table\"')\n",
    "assert result, result.message\n",
    "df = pandas.DataFrame(dataPack2.data)\n",
    "df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Save the data into IPFS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_table_encrypted.csv has been saved to IPFS.\n",
      "\n",
      "The IPFS ID is: QmU1nri3draq7SAHVAjSx91dEVCw1e1LGdv1NaiepGvTBm \n",
      "\n",
      "You can check on local IPFS http service: http://127.0.0.1:8080/ipfs/QmU1nri3draq7SAHVAjSx91dEVCw1e1LGdv1NaiepGvTBm\n",
      "\n",
      "And the content on IPFS: http://127.0.0.1:8080/ipfs/QmU1nri3draq7SAHVAjSx91dEVCw1e1LGdv1NaiepGvTBm/test_table_encrypted.csv\n",
      "\n",
      "The content is:\n",
      "uid,wallet_address,register_date\n",
      "123,\\x82148bcab7b5c8fc8f81ce8cb79b307d81bfde84e51fa87ae99d56b9985b8b9018b26eff1b96a38cf148ffe1b923b99be3908a7e6d27d57e78d769af6be82dc92d,\\x0776d01ca9ffa39ea95916108147db031025ce9e3ce7f6cba619041b9fe9c247ee\n",
      "124,\\x43ad786aca3e01879ee6b7753032842658e6c24a302bf2986b70c247a36bf54724ec914002f930b13fa9de88b072502f887a930b655590090a21d0f6253ab9b5f4,\\xcf0cc270db2108338f9440a0fb810706e31a73b20284226d9bddc40649d66e43f7\n",
      "125,\\xe49d212633890b86855d663d90ee71ed323b68f67a53fed837c88f037d5fdb4ef5456e596a88a10a6b3d9c347033e4c0c7e1f566017fed5c49d7546fbf76348c8d,\\xb8ec6581139a9c29ce58c85f094ec53e26a23047d6ea3c73edbdcf57500e9d5de7\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# make sure you have started IPFS service on your local machine\n",
    "ipfs1 = DataPack.IPFSConnector()\n",
    "result = dataPack2.saveTo(ipfs1, 'test_table_encrypted.csv')\n",
    "assert result, result.message\n",
    "print(\"test_table_encrypted.csv has been saved to IPFS.\\n\")\n",
    "ipfsHash = result.data\n",
    "print(\"The IPFS ID is:\", ipfsHash, '\\n')\n",
    "ipfs_local_url = \"http://127.0.0.1:8080/ipfs/%s\"%ipfsHash\n",
    "ipfs_local_data_url = ipfs_local_url + '/test_table_encrypted.csv'\n",
    "print(f\"You can check on local IPFS http service: {ipfs_local_url}\\n\")\n",
    "print(f\"And the content on IPFS: {ipfs_local_data_url}\\n\")\n",
    "import requests\n",
    "content = requests.get(ipfs_local_data_url).text\n",
    "print(\"The content is:\")\n",
    "print(content)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4. IPFS -> Local File\n",
    "- Load data from IPFS and save to local file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You can see file on IPFS are encrypted, but datapack can decrypted it locally\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>123</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>124</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>125</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0                                           1                          2\n",
       "0  123  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441\n",
       "1  124  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441\n",
       "2  125  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataPack3 = DataPack(env.walletPrivateKey)\n",
    "ipfs2 = DataPack.IPFSConnector()\n",
    "result = dataPack3.loadFrom(ipfs2, ipfsHash)\n",
    "assert result, result.message\n",
    "print(\"You can see file on IPFS are encrypted, but datapack can decrypted it locally\")\n",
    "df = pandas.DataFrame(dataPack3.data)\n",
    "df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Save the data into a local file with encryption"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_localtable_encrypted.csv has been saved to local file.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>wallet_address</th>\n",
       "      <th>register_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>123</td>\n",
       "      <td>\\x14bb21b7b3105cac85bc84383b3b0b84bf9d0503940d...</td>\n",
       "      <td>\\x6aa4984ca3bd3682679308b2ef22cf6f12d42698fde0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>124</td>\n",
       "      <td>\\x3f87e2905e69ee36419ac126dee7eb23b1b7d3b5ba16...</td>\n",
       "      <td>\\x9b8355a538a4b2f84824f55d75a098d984145de1371f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>125</td>\n",
       "      <td>\\x7901ba445c75ceb0016914f307707c84a26b655fe7fd...</td>\n",
       "      <td>\\x7e91fca841863ca2ae32b2b1098b5191fdba7e0e0e6a...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid                                     wallet_address  \\\n",
       "0  123  \\x14bb21b7b3105cac85bc84383b3b0b84bf9d0503940d...   \n",
       "1  124  \\x3f87e2905e69ee36419ac126dee7eb23b1b7d3b5ba16...   \n",
       "2  125  \\x7901ba445c75ceb0016914f307707c84a26b655fe7fd...   \n",
       "\n",
       "                                       register_date  \n",
       "0  \\x6aa4984ca3bd3682679308b2ef22cf6f12d42698fde0...  \n",
       "1  \\x9b8355a538a4b2f84824f55d75a098d984145de1371f...  \n",
       "2  \\x7e91fca841863ca2ae32b2b1098b5191fdba7e0e0e6a...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "localFile2 = DataPack.LocalFileConnector()\n",
    "result = dataPack3.saveTo(localFile2, 'test_localtable_encrypted.csv')\n",
    "assert result, result.message\n",
    "print(\"test_localtable_encrypted.csv has been saved to local file.\")\n",
    "df = pandas.read_csv('test_localtable_encrypted.csv')\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "datapack-dev",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}