examples/use_case_arweave.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1: configuration"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Input your walletPrivateKey here:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "check env.walletPrivateKey: 2a776\n",
      "check env.mindLakeAppKey: Kyunq\n",
      "check env.MINDLAKE_GATEWAY: https://sdk.mindnetwork.xyz/node\n"
     ]
    }
   ],
   "source": [
    "# make sure you configure mindlakesdk, please check env.py in the same folder\n",
    "# please familar these configure, and you can read https://github.com/mind-network/mind-lake-sdk-python \n",
    "import env\n",
    "\n",
    "print(\"check env.walletPrivateKey:\", env.walletPrivateKey[0:5])\n",
    "print(\"check env.mindLakeAppKey:\", env.mindLakeAppKey[0:5])\n",
    "print(\"check env.MINDLAKE_GATEWAY:\", env.MINDLAKE_GATEWAY)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Install depedancy and source code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# please \"pip install mindlakesdk\" if not installed\n",
    "import mindlakesdk\n",
    "\n",
    "# please \"pip install minddatapack\" if not installed, or git clone the source code\n",
    "from minddatapack import DataPack\n",
    "\n",
    "# please \"pip install pandas\" if not installed, it is used to display structure content in the notebook\n",
    "import pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check if example_data.csv file exists in the same folder, this demo will start to load this csv file.\n",
    "\n",
    "# We use everpay to pay for the transaction fee. You can connect to everpay either by your eth wallet or by arweave wallet.\n",
    "# If you choose arweave wallet, check if arweave_wallet.json file exists in the same folder.\n",
    "# Make sure you have some balance in your everpay account."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2. Local CSV -> MindLake\n",
    "- Load data from a local CSV file without the metadata file, by defining each column manually.\n",
    "- You can also load from a local backup into MindLake without the need to define each column. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>wallet_address</th>\n",
       "      <th>register_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>123</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>124</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>125</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid                              wallet_address               register_date\n",
       "0  123  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7  2023-07-15 02:25:32.392441\n",
       "1  124  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7  2023-07-15 02:25:32.392441\n",
       "2  125  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7  2023-07-15 02:25:32.392441"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn_local_csv = './example_data.csv'\n",
    "\n",
    "# preview local csv\n",
    "df = pandas.read_csv(fn_local_csv)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>123</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>124</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>125</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0                                           1                          2\n",
       "0  123  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441\n",
       "1  124  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441\n",
       "2  125  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataPack1 = DataPack(env.walletPrivateKey)\n",
    "localFile1 = DataPack.LocalFileConnector(columns=[\n",
    "        DataPack.Column('uid', DataPack.DataType.int4, False),\n",
    "        DataPack.Column('wallet_address', DataPack.DataType.text, True),\n",
    "        DataPack.Column('register_date', DataPack.DataType.timestamp, True)\n",
    "    ])\n",
    "result = dataPack1.loadFrom(localFile1, fn_local_csv)\n",
    "assert result, result.message\n",
    "df = pandas.DataFrame(dataPack1.data)\n",
    "df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Save the data into a table in MindLake"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dropTable result: Success\n",
      "a_new_table has been saved to MindLake.\n"
     ]
    }
   ],
   "source": [
    "mindlake = mindlakesdk.connect(env.walletPrivateKey, env.mindLakeAppKey, env.MINDLAKE_GATEWAY)\n",
    "assert mindlake, mindlake.message\n",
    "\n",
    "# drop the table if exists\n",
    "result = mindlake.datalake.dropTable('a_new_table')\n",
    "print(\"dropTable result:\", result.message)\n",
    "\n",
    "# upload from local csv into mindlake\n",
    "mindLakeConnector = DataPack.MindLakeConnector(mindlake)\n",
    "result = dataPack1.saveTo(mindLakeConnector, 'a_new_table')\n",
    "assert result, result.message\n",
    "print(\"a_new_table has been saved to MindLake.\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "you can also check this new downloaded table in https://scan.mindnetwork.xyz/account/myData"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3. MindLake -> Arweave\n",
    "- Load data from a table in MindLake"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>123</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>124</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>125</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0                                           1                          2\n",
       "0  123  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441\n",
       "1  124  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441\n",
       "2  125  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataPack2 = DataPack(env.walletPrivateKey)\n",
    "# make sure if you complete Step 1, so \"a_new_table\" exists in mindlake\n",
    "result = dataPack2.loadFrom(mindLakeConnector, 'select * from \"a_new_table\"')\n",
    "assert result, result.message\n",
    "df = pandas.DataFrame(dataPack2.data)\n",
    "df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Save the data into Arweave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_table_encrypted.csv has been saved to Arweave.\n",
      "\n",
      "The Arweave ID is: DwyQeo8B7RdoJlqWNGm1WHiKVh-ua5BmclKu9pdX29w \n",
      "\n",
      "You can check on Arweave: https://viewblock.io/arweave/tx/DwyQeo8B7RdoJlqWNGm1WHiKVh-ua5BmclKu9pdX29w\n",
      "\n",
      "And the content on Arweave: https://arseed.web3infra.dev/DwyQeo8B7RdoJlqWNGm1WHiKVh-ua5BmclKu9pdX29w\n",
      "\n",
      "The content is:\n",
      "uid,wallet_address,register_date\n",
      "123,\\x645039a18f2d0ecfda0049619267f6e4f658d3c8dea202cf2953eaee63894b0b3af0c159a58036cd81d29d4f07d63f75236243d1bbdfd447a90a68e901a62f4153,\\xc9bd351c80b4784cdb1e26b10a8064946eb62b1d53b2b1f42da3092bf6e2cd054f\n",
      "124,\\x6e6190fd99e05a04ea63768f435b8913a350876a758869804ae6a8ca824091af8b89b821cb2f9f1a7b4bdf51e90e94d77e41dac7637bf1babcfa7f1ba94f103ff5,\\x69239ddf275ef971a8bd7051988b1f92dd784377df7b4989719322aab31eeb3cb6\n",
      "125,\\x54c1c6f1bc35bba9a1903e7774ca3ba6646b2aaf0f4a4b92262f8932b92bfb8d3a1699ebbc6b391e3b19983aa7832be12d75d5bdfe128c094e61b7f0a1dbab2e2c,\\x045d590d0ef821fe691ee42e04d066338392adcaf50627e3335250e9886d1a7c02\n",
      "\n"
     ]
    }
   ],
   "source": [
    "arweave1 = DataPack.ArweaveConnector('ACNH', ethWalletPrivateKey=env.walletPrivateKey)\n",
    "# arweave = DataPack.ArweaveConnector('AR', arweaveWalletFile='./arweave_wallet.json')\n",
    "result = dataPack2.saveTo(arweave1, 'test_table_encrypted.csv')\n",
    "assert result, result.message\n",
    "print(\"test_table_encrypted.csv has been saved to Arweave.\\n\")\n",
    "itemID = result.data\n",
    "print(\"The Arweave ID is:\", itemID, '\\n')\n",
    "arweave_explore_url = \"https://viewblock.io/arweave/tx/%s\"%(itemID)\n",
    "arweave_file_url = \"https://arseed.web3infra.dev/%s\"%(itemID)\n",
    "print(f\"You can check on Arweave: {arweave_explore_url}\\n\")\n",
    "print(f\"And the content on Arweave: {arweave_file_url}\\n\")\n",
    "import requests\n",
    "content = requests.get(arweave_file_url).text\n",
    "print(\"The content is:\")\n",
    "print(content)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4. Arweave -> Local File\n",
    "- Load data from Arweave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You can see file on Arweave are encrypted, but datapack can decrypted it locally\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>123</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>124</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>125</td>\n",
       "      <td>0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7</td>\n",
       "      <td>2023-07-15 02:25:32.392441</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0                                           1                          2\n",
       "0  123  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441\n",
       "1  124  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441\n",
       "2  125  0x79Be957bf7e3003aFd0e78f04Bacbc93D3ef2fB7 2023-07-15 02:25:32.392441"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataPack3 = DataPack(env.walletPrivateKey)\n",
    "arweave2 = DataPack.ArweaveConnector()\n",
    "result = dataPack3.loadFrom(arweave2, itemID)\n",
    "assert result, result.message\n",
    "print(\"You can see file on Arweave are encrypted, but datapack can decrypted it locally\")\n",
    "df = pandas.DataFrame(dataPack3.data)\n",
    "df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Save the data into a local file with encryption"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_localtable_encrypted.csv has been saved to local file.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>wallet_address</th>\n",
       "      <th>register_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>123</td>\n",
       "      <td>\\x3c364d7aed8d63b68701fd00feac41435ff456eb307d...</td>\n",
       "      <td>\\x35bafd055143484471c52153385b7ee5eedc58b09021...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>124</td>\n",
       "      <td>\\xbe1d0f00778d9e15e08672f9494a22c11ef870758572...</td>\n",
       "      <td>\\x25ceb7256dd9ab8ef563ae220c62df39cf13bf3a7d08...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>125</td>\n",
       "      <td>\\xa5805d615740ead33874d7036246cc2aee7665901d3c...</td>\n",
       "      <td>\\xb541751a94f6b5128af3a5939867531ed5736b0fd7c4...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uid                                     wallet_address  \\\n",
       "0  123  \\x3c364d7aed8d63b68701fd00feac41435ff456eb307d...   \n",
       "1  124  \\xbe1d0f00778d9e15e08672f9494a22c11ef870758572...   \n",
       "2  125  \\xa5805d615740ead33874d7036246cc2aee7665901d3c...   \n",
       "\n",
       "                                       register_date  \n",
       "0  \\x35bafd055143484471c52153385b7ee5eedc58b09021...  \n",
       "1  \\x25ceb7256dd9ab8ef563ae220c62df39cf13bf3a7d08...  \n",
       "2  \\xb541751a94f6b5128af3a5939867531ed5736b0fd7c4...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "localFile2 = DataPack.LocalFileConnector()\n",
    "result = dataPack3.saveTo(localFile2, 'test_localtable_encrypted.csv')\n",
    "assert result, result.message\n",
    "print(\"test_localtable_encrypted.csv has been saved to local file.\")\n",
    "df = pandas.read_csv('test_localtable_encrypted.csv')\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "datapack-dev",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}