04_training-data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#default_exp data.training_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training Data Preparation\n",
    "\n",
    "> This module reads features computed on customer_id and email, merges both datasets, obtain `is_fraud` label for orders and creates two dataset that are suitable to train the model:\n",
    "\n",
    ">1) Sampled: a dataset that contains all transactions marked as fraud, plus a subsample of the non fraud orders, in order to obtain a less-imbalanced dataset and train faster\n",
    "\n",
    ">2) Validation: a dataset containing only most recent orders, containing all fraud and non fraud transactions to be use as a final validation for the trained model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from typing import Dict\n",
    "from datetime import datetime, timezone, timedelta\n",
    "import random\n",
    "import math\n",
    "import dask.dataframe as dd\n",
    "import numpy as np\n",
    "\n",
    "from hopeit.app.context import EventContext\n",
    "from hopeit.app.events import Spawn, SHUFFLE\n",
    "from hopeit.app.api import event_api\n",
    "from hopeit.app.logger import app_logger\n",
    "\n",
    "from fraud_poc.jobs import get_client, FeatureCalcJob, TrainingDataJob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "__steps__ = ['run']\n",
    "\n",
    "logger = app_logger()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def _merge_feature_datasets(datasets: Dict[str, str]):\n",
    "    df = None\n",
    "    for key, path in datasets.items():\n",
    "        df_key = dd.read_parquet(path, engine='fastparquet')\n",
    "        if df is not None:\n",
    "            df = df.merge(df_key, left_on='order_id', right_on='order_id', suffixes=('', '_DROP'))\n",
    "            keep_cols = [c for c in df.columns if c[-5:] != '_DROP']\n",
    "            df = df[keep_cols]\n",
    "        else:\n",
    "            df = df_key\n",
    "    return df\n",
    "\n",
    "\n",
    "def _add_labels(df):\n",
    "    df['is_fraud'] = (df['known_ip_addr_by_customer_id'] == 0) & (df['num_ip_addr_by_customer_id'] > 3)\n",
    "    df['is_fraud'] = df['is_fraud'] | ((df['known_email_by_customer_id'] == 0) & (df['num_email_by_customer_id'] > 3))\n",
    "    df['is_fraud'] = df['is_fraud'] | (df['order_amount'] > (1. + 0.5 * random.random() * df['order_amount_mean_by_customer_id']))\n",
    "    df['is_fraud'] = df['is_fraud'].apply(lambda x: int(x & (random.random() > 0.1)), meta=('is_fraud', int))\n",
    "    return df\n",
    "\n",
    "def _add_sample_flag(df, subsample_not_fraud: float):\n",
    "    df['sample'] = df['is_fraud'].apply(lambda x: int((x > 0) | (random.random() > (1.-subsample_not_fraud))), meta=('sample', int))\n",
    "    return df\n",
    "\n",
    "def _add_validation_flag(df):\n",
    "    now = datetime.now(tz=timezone.utc)\n",
    "    now_epoch = now.timestamp()\n",
    "    df['now'] = now\n",
    "    df['elapsed_wgt'] = df['order_date'].apply(lambda x: math.log(max(0.001, 1. - (now_epoch - x.timestamp())/now_epoch)) + 1., meta=('elapsed_wgt', float))\n",
    "    df['validation'] = df['elapsed_wgt'].apply(lambda x: int((max(0, x)  * random.random()) > 0.8), meta=('validation', int))\n",
    "    return df\n",
    "\n",
    "def _add_fold_number(df, num_folds):\n",
    "    df['fold'] = df['is_fraud'].apply(lambda x: random.randint(0, num_folds), meta=('fold', int)) \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def run(job: FeatureCalcJob, context: EventContext) -> TrainingDataJob:\n",
    "    base_path = context.env['data']['training']\n",
    "    num_folds = context.env['training_data']['num_folds']\n",
    "    subsample_not_fraud = context.env['training_data']['subsample_not_fraud']\n",
    "    \n",
    "    client = get_client(context)\n",
    "    try:\n",
    "        df = _merge_feature_datasets(job.features)\n",
    "        df = _add_labels(df)\n",
    "        df = _add_sample_flag(df, subsample_not_fraud)\n",
    "        df = _add_validation_flag(df)\n",
    "        df = _add_fold_number(df, num_folds)\n",
    "        \n",
    "        sampled_save_path = f\"{base_path}/sampled/\"\n",
    "        logger.info(context, f\"Saving sampled training dataset to {sampled_save_path}...\")\n",
    "        df_sample = df[df['sample'] > 0]\n",
    "        df_sample = df_sample.set_index('fold')\n",
    "        df_sample.to_parquet(sampled_save_path)\n",
    "        \n",
    "        valid_save_path = f\"{base_path}/validation/\"\n",
    "        logger.info(context, f\"Saving weighted validation dataset to {valid_save_path}...\")\n",
    "        df_validation = df[df['validation'] >0 ]\n",
    "        df_validation.to_parquet(valid_save_path)\n",
    "\n",
    "        return TrainingDataJob(\n",
    "            sources=job.features,\n",
    "            sampled=sampled_save_path,\n",
    "            validation=valid_save_path\n",
    "        )\n",
    "    except Exception as e:\n",
    "        logger.error(context, e)\n",
    "        return None\n",
    "    finally:\n",
    "        client.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test from notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-05-18 20:28:12,444 | INFO | fraud-poc training data.training-data ALT00617 75610 | Saving sampled training dataset to ./data/training/sampled/... | track.operation_id=test_operation_id | track.request_id=test_request_id | track.request_ts=2021-05-18T20:28:11.484283+00:00 | stream.name= | stream.msg_id= | stream.consumer_group=\n",
      "2021-05-18 20:28:14,091 | INFO | fraud-poc training data.training-data ALT00617 75610 | Saving weighted validation dataset to ./data/training/validation/... | track.operation_id=test_operation_id | track.request_id=test_request_id | track.request_ts=2021-05-18T20:28:11.484283+00:00 | stream.name= | stream.msg_id= | stream.consumer_group=\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainingDataJob(sources={'customer_id': './data/features/customer_id/', 'email': './data/features/email/'}, sampled='./data/training/sampled/', validation='./data/training/validation/')"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from hopeit.testing.apps import config, execute_event\n",
    "\n",
    "app_config = config('config/training-pipeline.json')\n",
    "job = FeatureCalcJob(sources={'customer_id': './data/partitioned/customer_id/', 'email': './data/partitioned/email'}, \n",
    "                     features={'customer_id': './data/features/customer_id/', 'email': './data/features/email/'})\n",
    "result = await execute_event(app_config, 'data.training-data', job)\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_amount</th>\n",
       "      <th>num_email_by_customer_id</th>\n",
       "      <th>num_ip_addr_by_customer_id</th>\n",
       "      <th>same_email_by_customer_id</th>\n",
       "      <th>same_ip_addr_by_customer_id</th>\n",
       "      <th>known_email_by_customer_id</th>\n",
       "      <th>known_ip_addr_by_customer_id</th>\n",
       "      <th>order_amount_mean_by_customer_id</th>\n",
       "      <th>order_amount_std_by_customer_id</th>\n",
       "      <th>order_amount_min_by_customer_id</th>\n",
       "      <th>...</th>\n",
       "      <th>known_customer_id_by_email</th>\n",
       "      <th>order_amount_mean_by_email</th>\n",
       "      <th>order_amount_std_by_email</th>\n",
       "      <th>order_amount_min_by_email</th>\n",
       "      <th>order_amount_max_by_email</th>\n",
       "      <th>order_amount_sum_by_email</th>\n",
       "      <th>is_fraud</th>\n",
       "      <th>sample</th>\n",
       "      <th>elapsed_wgt</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.0</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>188.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>524.606828</td>\n",
       "      <td>1.755319</td>\n",
       "      <td>2.138298</td>\n",
       "      <td>0.361702</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.569149</td>\n",
       "      <td>0.446809</td>\n",
       "      <td>511.480117</td>\n",
       "      <td>159.316375</td>\n",
       "      <td>319.896259</td>\n",
       "      <td>...</td>\n",
       "      <td>0.569149</td>\n",
       "      <td>524.794925</td>\n",
       "      <td>113.340613</td>\n",
       "      <td>396.771221</td>\n",
       "      <td>657.255252</td>\n",
       "      <td>1047.595948</td>\n",
       "      <td>0.941489</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999470</td>\n",
       "      <td>0.234043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>281.567721</td>\n",
       "      <td>0.955549</td>\n",
       "      <td>1.193575</td>\n",
       "      <td>0.481776</td>\n",
       "      <td>0.434169</td>\n",
       "      <td>0.496518</td>\n",
       "      <td>0.498490</td>\n",
       "      <td>207.981474</td>\n",
       "      <td>123.908393</td>\n",
       "      <td>245.276464</td>\n",
       "      <td>...</td>\n",
       "      <td>0.496518</td>\n",
       "      <td>231.650609</td>\n",
       "      <td>123.329541</td>\n",
       "      <td>276.938429</td>\n",
       "      <td>272.226871</td>\n",
       "      <td>692.482507</td>\n",
       "      <td>0.235333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000314</td>\n",
       "      <td>0.424529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.402291</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.402291</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.402291</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.402291</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.402291</td>\n",
       "      <td>0.402291</td>\n",
       "      <td>0.402291</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.998941</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>346.147602</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>398.643346</td>\n",
       "      <td>36.087890</td>\n",
       "      <td>134.578503</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>353.471786</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>167.206097</td>\n",
       "      <td>438.680258</td>\n",
       "      <td>520.862094</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999246</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>491.724897</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>522.281127</td>\n",
       "      <td>190.593451</td>\n",
       "      <td>281.717170</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>511.262158</td>\n",
       "      <td>136.818371</td>\n",
       "      <td>350.677562</td>\n",
       "      <td>728.436786</td>\n",
       "      <td>938.823950</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999501</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>793.608032</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>653.541265</td>\n",
       "      <td>281.174204</td>\n",
       "      <td>462.674445</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>701.530790</td>\n",
       "      <td>249.284494</td>\n",
       "      <td>624.516786</td>\n",
       "      <td>925.829980</td>\n",
       "      <td>1454.151908</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999753</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>997.119448</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>989.953833</td>\n",
       "      <td>413.698024</td>\n",
       "      <td>989.953833</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>995.144205</td>\n",
       "      <td>413.698024</td>\n",
       "      <td>995.144205</td>\n",
       "      <td>999.276104</td>\n",
       "      <td>3007.250422</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999994</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       order_amount  num_email_by_customer_id  num_ip_addr_by_customer_id  \\\n",
       "count    188.000000                188.000000                  188.000000   \n",
       "mean     524.606828                  1.755319                    2.138298   \n",
       "std      281.567721                  0.955549                    1.193575   \n",
       "min        0.402291                  1.000000                    1.000000   \n",
       "25%      346.147602                  1.000000                    1.000000   \n",
       "50%      491.724897                  2.000000                    2.000000   \n",
       "75%      793.608032                  2.000000                    3.000000   \n",
       "max      997.119448                  5.000000                    6.000000   \n",
       "\n",
       "       same_email_by_customer_id  same_ip_addr_by_customer_id  \\\n",
       "count                 188.000000                   188.000000   \n",
       "mean                    0.361702                     0.250000   \n",
       "std                     0.481776                     0.434169   \n",
       "min                     0.000000                     0.000000   \n",
       "25%                     0.000000                     0.000000   \n",
       "50%                     0.000000                     0.000000   \n",
       "75%                     1.000000                     1.000000   \n",
       "max                     1.000000                     1.000000   \n",
       "\n",
       "       known_email_by_customer_id  known_ip_addr_by_customer_id  \\\n",
       "count                  188.000000                    188.000000   \n",
       "mean                     0.569149                      0.446809   \n",
       "std                      0.496518                      0.498490   \n",
       "min                      0.000000                      0.000000   \n",
       "25%                      0.000000                      0.000000   \n",
       "50%                      1.000000                      0.000000   \n",
       "75%                      1.000000                      1.000000   \n",
       "max                      1.000000                      1.000000   \n",
       "\n",
       "       order_amount_mean_by_customer_id  order_amount_std_by_customer_id  \\\n",
       "count                        188.000000                       188.000000   \n",
       "mean                         511.480117                       159.316375   \n",
       "std                          207.981474                       123.908393   \n",
       "min                            0.402291                         0.000000   \n",
       "25%                          398.643346                        36.087890   \n",
       "50%                          522.281127                       190.593451   \n",
       "75%                          653.541265                       281.174204   \n",
       "max                          989.953833                       413.698024   \n",
       "\n",
       "       order_amount_min_by_customer_id  ...  known_customer_id_by_email  \\\n",
       "count                       188.000000  ...                  188.000000   \n",
       "mean                        319.896259  ...                    0.569149   \n",
       "std                         245.276464  ...                    0.496518   \n",
       "min                           0.402291  ...                    0.000000   \n",
       "25%                         134.578503  ...                    0.000000   \n",
       "50%                         281.717170  ...                    1.000000   \n",
       "75%                         462.674445  ...                    1.000000   \n",
       "max                         989.953833  ...                    1.000000   \n",
       "\n",
       "       order_amount_mean_by_email  order_amount_std_by_email  \\\n",
       "count                  188.000000                 188.000000   \n",
       "mean                   524.794925                 113.340613   \n",
       "std                    231.650609                 123.329541   \n",
       "min                      0.402291                   0.000000   \n",
       "25%                    353.471786                   0.000000   \n",
       "50%                    511.262158                 136.818371   \n",
       "75%                    701.530790                 249.284494   \n",
       "max                    995.144205                 413.698024   \n",
       "\n",
       "       order_amount_min_by_email  order_amount_max_by_email  \\\n",
       "count                 188.000000                 188.000000   \n",
       "mean                  396.771221                 657.255252   \n",
       "std                   276.938429                 272.226871   \n",
       "min                     0.402291                   0.402291   \n",
       "25%                   167.206097                 438.680258   \n",
       "50%                   350.677562                 728.436786   \n",
       "75%                   624.516786                 925.829980   \n",
       "max                   995.144205                 999.276104   \n",
       "\n",
       "       order_amount_sum_by_email    is_fraud  sample  elapsed_wgt  validation  \n",
       "count                 188.000000  188.000000   188.0   188.000000  188.000000  \n",
       "mean                 1047.595948    0.941489     1.0     0.999470    0.234043  \n",
       "std                   692.482507    0.235333     0.0     0.000314    0.424529  \n",
       "min                     0.402291    0.000000     1.0     0.998941    0.000000  \n",
       "25%                   520.862094    1.000000     1.0     0.999246    0.000000  \n",
       "50%                   938.823950    1.000000     1.0     0.999501    0.000000  \n",
       "75%                  1454.151908    1.000000     1.0     0.999753    1.000000  \n",
       "max                  3007.250422    1.000000     1.0     0.999994    1.000000  \n",
       "\n",
       "[8 rows x 24 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd.read_parquet(result.sampled).describe().compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_amount</th>\n",
       "      <th>num_email_by_customer_id</th>\n",
       "      <th>num_ip_addr_by_customer_id</th>\n",
       "      <th>same_email_by_customer_id</th>\n",
       "      <th>same_ip_addr_by_customer_id</th>\n",
       "      <th>known_email_by_customer_id</th>\n",
       "      <th>known_ip_addr_by_customer_id</th>\n",
       "      <th>order_amount_mean_by_customer_id</th>\n",
       "      <th>order_amount_std_by_customer_id</th>\n",
       "      <th>order_amount_min_by_customer_id</th>\n",
       "      <th>...</th>\n",
       "      <th>order_amount_mean_by_email</th>\n",
       "      <th>order_amount_std_by_email</th>\n",
       "      <th>order_amount_min_by_email</th>\n",
       "      <th>order_amount_max_by_email</th>\n",
       "      <th>order_amount_sum_by_email</th>\n",
       "      <th>is_fraud</th>\n",
       "      <th>sample</th>\n",
       "      <th>elapsed_wgt</th>\n",
       "      <th>validation</th>\n",
       "      <th>fold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>42.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>42.00000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>42.0</td>\n",
       "      <td>42.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>531.340590</td>\n",
       "      <td>1.690476</td>\n",
       "      <td>2.02381</td>\n",
       "      <td>0.357143</td>\n",
       "      <td>0.238095</td>\n",
       "      <td>0.547619</td>\n",
       "      <td>0.357143</td>\n",
       "      <td>515.845594</td>\n",
       "      <td>185.447372</td>\n",
       "      <td>311.285556</td>\n",
       "      <td>...</td>\n",
       "      <td>560.438991</td>\n",
       "      <td>124.731174</td>\n",
       "      <td>430.621161</td>\n",
       "      <td>702.370350</td>\n",
       "      <td>997.584484</td>\n",
       "      <td>0.880952</td>\n",
       "      <td>0.976190</td>\n",
       "      <td>0.999446</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.785714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>264.341622</td>\n",
       "      <td>0.840676</td>\n",
       "      <td>0.99971</td>\n",
       "      <td>0.484966</td>\n",
       "      <td>0.431081</td>\n",
       "      <td>0.503761</td>\n",
       "      <td>0.484966</td>\n",
       "      <td>171.878141</td>\n",
       "      <td>129.856514</td>\n",
       "      <td>215.511969</td>\n",
       "      <td>...</td>\n",
       "      <td>217.599570</td>\n",
       "      <td>139.248552</td>\n",
       "      <td>264.615976</td>\n",
       "      <td>263.050340</td>\n",
       "      <td>558.305630</td>\n",
       "      <td>0.327770</td>\n",
       "      <td>0.154303</td>\n",
       "      <td>0.000333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.841639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>80.334187</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>80.334187</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>57.086638</td>\n",
       "      <td>...</td>\n",
       "      <td>80.334187</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>80.334187</td>\n",
       "      <td>80.334187</td>\n",
       "      <td>80.334187</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.998948</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>420.356323</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>466.393298</td>\n",
       "      <td>102.186373</td>\n",
       "      <td>164.200450</td>\n",
       "      <td>...</td>\n",
       "      <td>433.396839</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>246.722547</td>\n",
       "      <td>462.674445</td>\n",
       "      <td>570.021621</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999238</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>607.866511</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>581.472604</td>\n",
       "      <td>226.378341</td>\n",
       "      <td>281.717170</td>\n",
       "      <td>...</td>\n",
       "      <td>581.472604</td>\n",
       "      <td>119.807098</td>\n",
       "      <td>413.874710</td>\n",
       "      <td>813.828513</td>\n",
       "      <td>1030.247967</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999419</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>662.987751</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>596.497147</td>\n",
       "      <td>306.184266</td>\n",
       "      <td>468.373990</td>\n",
       "      <td>...</td>\n",
       "      <td>766.321292</td>\n",
       "      <td>257.718198</td>\n",
       "      <td>717.698285</td>\n",
       "      <td>935.718323</td>\n",
       "      <td>1546.594138</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999798</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>958.691116</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>891.351667</td>\n",
       "      <td>413.698024</td>\n",
       "      <td>891.351667</td>\n",
       "      <td>...</td>\n",
       "      <td>958.691116</td>\n",
       "      <td>413.698024</td>\n",
       "      <td>958.691116</td>\n",
       "      <td>999.276104</td>\n",
       "      <td>2582.030538</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999994</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       order_amount  num_email_by_customer_id  num_ip_addr_by_customer_id  \\\n",
       "count     42.000000                 42.000000                    42.00000   \n",
       "mean     531.340590                  1.690476                     2.02381   \n",
       "std      264.341622                  0.840676                     0.99971   \n",
       "min       80.334187                  1.000000                     1.00000   \n",
       "25%      420.356323                  1.000000                     1.00000   \n",
       "50%      607.866511                  2.000000                     2.00000   \n",
       "75%      662.987751                  2.000000                     3.00000   \n",
       "max      958.691116                  4.000000                     4.00000   \n",
       "\n",
       "       same_email_by_customer_id  same_ip_addr_by_customer_id  \\\n",
       "count                  42.000000                    42.000000   \n",
       "mean                    0.357143                     0.238095   \n",
       "std                     0.484966                     0.431081   \n",
       "min                     0.000000                     0.000000   \n",
       "25%                     0.000000                     0.000000   \n",
       "50%                     0.000000                     0.000000   \n",
       "75%                     1.000000                     0.500000   \n",
       "max                     1.000000                     1.000000   \n",
       "\n",
       "       known_email_by_customer_id  known_ip_addr_by_customer_id  \\\n",
       "count                   42.000000                     42.000000   \n",
       "mean                     0.547619                      0.357143   \n",
       "std                      0.503761                      0.484966   \n",
       "min                      0.000000                      0.000000   \n",
       "25%                      0.000000                      0.000000   \n",
       "50%                      1.000000                      0.000000   \n",
       "75%                      1.000000                      1.000000   \n",
       "max                      1.000000                      1.000000   \n",
       "\n",
       "       order_amount_mean_by_customer_id  order_amount_std_by_customer_id  \\\n",
       "count                         42.000000                        42.000000   \n",
       "mean                         515.845594                       185.447372   \n",
       "std                          171.878141                       129.856514   \n",
       "min                           80.334187                         0.000000   \n",
       "25%                          466.393298                       102.186373   \n",
       "50%                          581.472604                       226.378341   \n",
       "75%                          596.497147                       306.184266   \n",
       "max                          891.351667                       413.698024   \n",
       "\n",
       "       order_amount_min_by_customer_id  ...  order_amount_mean_by_email  \\\n",
       "count                        42.000000  ...                   42.000000   \n",
       "mean                        311.285556  ...                  560.438991   \n",
       "std                         215.511969  ...                  217.599570   \n",
       "min                          57.086638  ...                   80.334187   \n",
       "25%                         164.200450  ...                  433.396839   \n",
       "50%                         281.717170  ...                  581.472604   \n",
       "75%                         468.373990  ...                  766.321292   \n",
       "max                         891.351667  ...                  958.691116   \n",
       "\n",
       "       order_amount_std_by_email  order_amount_min_by_email  \\\n",
       "count                  42.000000                  42.000000   \n",
       "mean                  124.731174                 430.621161   \n",
       "std                   139.248552                 264.615976   \n",
       "min                     0.000000                  80.334187   \n",
       "25%                     0.000000                 246.722547   \n",
       "50%                   119.807098                 413.874710   \n",
       "75%                   257.718198                 717.698285   \n",
       "max                   413.698024                 958.691116   \n",
       "\n",
       "       order_amount_max_by_email  order_amount_sum_by_email   is_fraud  \\\n",
       "count                  42.000000                  42.000000  42.000000   \n",
       "mean                  702.370350                 997.584484   0.880952   \n",
       "std                   263.050340                 558.305630   0.327770   \n",
       "min                    80.334187                  80.334187   0.000000   \n",
       "25%                   462.674445                 570.021621   1.000000   \n",
       "50%                   813.828513                1030.247967   1.000000   \n",
       "75%                   935.718323                1546.594138   1.000000   \n",
       "max                   999.276104                2582.030538   1.000000   \n",
       "\n",
       "          sample  elapsed_wgt  validation       fold  \n",
       "count  42.000000    42.000000        42.0  42.000000  \n",
       "mean    0.976190     0.999446         1.0   4.785714  \n",
       "std     0.154303     0.000333         0.0   2.841639  \n",
       "min     0.000000     0.998948         1.0   0.000000  \n",
       "25%     1.000000     0.999238         1.0   3.500000  \n",
       "50%     1.000000     0.999419         1.0   6.000000  \n",
       "75%     1.000000     0.999798         1.0   7.000000  \n",
       "max     1.000000     0.999994         1.0  10.000000  \n",
       "\n",
       "[8 rows x 25 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd.read_parquet(result.validation).describe().compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>order_date</th>\n",
       "      <th>email</th>\n",
       "      <th>ip_addr</th>\n",
       "      <th>order_amount</th>\n",
       "      <th>customer_id</th>\n",
       "      <th>email_by_customer_id</th>\n",
       "      <th>ip_addr_by_customer_id</th>\n",
       "      <th>num_email_by_customer_id</th>\n",
       "      <th>num_ip_addr_by_customer_id</th>\n",
       "      <th>...</th>\n",
       "      <th>order_amount_min_by_email</th>\n",
       "      <th>order_amount_max_by_email</th>\n",
       "      <th>order_amount_sum_by_email</th>\n",
       "      <th>order_amount_by_email</th>\n",
       "      <th>is_fraud</th>\n",
       "      <th>sample</th>\n",
       "      <th>now</th>\n",
       "      <th>elapsed_wgt</th>\n",
       "      <th>validation</th>\n",
       "      <th>fold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>9c2b6d93-32a7-45fa-ac40-a47059e4b7c4</td>\n",
       "      <td>2021-05-04 12:29:59+00:00</td>\n",
       "      <td>20d3a0d5ee5fce50d1cc660ea94e8fd6dc3e0d5d</td>\n",
       "      <td>804833f0387c5c3a0aed1218687be20d19e4aa15</td>\n",
       "      <td>222.106098</td>\n",
       "      <td>c6eb3cd2-93bc-413f-a0e9-61c71ab0d224</td>\n",
       "      <td>[\"5402e6915b106c264bd2bd7d7194e45d6de83091\", \"...</td>\n",
       "      <td>[\"29854c437e114323911562c081daa967fc2418a4\", \"...</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>222.106098</td>\n",
       "      <td>222.106098</td>\n",
       "      <td>222.106098</td>\n",
       "      <td>[222.1060979147914]</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2021-05-18 20:28:12.411255+00:00</td>\n",
       "      <td>0.999236</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>a489c647-bd28-456e-ab10-f4dde1151a20</td>\n",
       "      <td>2021-05-18 14:09:46+00:00</td>\n",
       "      <td>7d95a0edb3399a16e29f353350793a55489af201</td>\n",
       "      <td>3de6a5ab79cda5e3a73eff8a2bbb506dab451293</td>\n",
       "      <td>121.437929</td>\n",
       "      <td>2355ad27-ea68-4439-a085-1d294cc772ec</td>\n",
       "      <td>[\"7d95a0edb3399a16e29f353350793a55489af201\", \"...</td>\n",
       "      <td>[\"f898fbab3f673df0b0049098814b2ffabd277471\", \"...</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>101.425959</td>\n",
       "      <td>928.822007</td>\n",
       "      <td>1151.685896</td>\n",
       "      <td>[928.8220074790162, 101.42595902612995, 121.43...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2021-05-18 20:28:12.411255+00:00</td>\n",
       "      <td>0.999986</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>b9df0d87-c37f-4837-a81e-27e23bedc4a8</td>\n",
       "      <td>2021-05-04 18:02:34+00:00</td>\n",
       "      <td>95d1b98298ff8558bebbcd5b68b81426d951dc9d</td>\n",
       "      <td>8d90d3f40f9b4c1974f42ba9cfb3f3513e55018a</td>\n",
       "      <td>801.040739</td>\n",
       "      <td>23925e34-a97b-4a39-9af7-556994f4de50</td>\n",
       "      <td>[\"c5a4b2c53a0f1060909717f27a26b6b611c6abac\", \"...</td>\n",
       "      <td>[\"e09327b09677fd88e72c6007d218ce042cba0f60\", \"...</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>801.040739</td>\n",
       "      <td>801.040739</td>\n",
       "      <td>801.040739</td>\n",
       "      <td>[801.040738517752]</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2021-05-18 20:28:12.411255+00:00</td>\n",
       "      <td>0.999248</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>564739cb-f238-487f-8893-3f8a5c42afe9</td>\n",
       "      <td>2021-05-05 21:21:08+00:00</td>\n",
       "      <td>164e9cd15154ecf244a56d1afa9f7182e12eb038</td>\n",
       "      <td>dadd450850a73d8c87ffef6f6c183a7fd03105e0</td>\n",
       "      <td>80.334187</td>\n",
       "      <td>78378996-89c5-4667-bf77-50a7bcf3899a</td>\n",
       "      <td>[\"164e9cd15154ecf244a56d1afa9f7182e12eb038\"]</td>\n",
       "      <td>[\"dadd450850a73d8c87ffef6f6c183a7fd03105e0\"]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>80.334187</td>\n",
       "      <td>80.334187</td>\n",
       "      <td>80.334187</td>\n",
       "      <td>[80.33418734448439]</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2021-05-18 20:28:12.411255+00:00</td>\n",
       "      <td>0.999309</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>39560977-f66c-4be8-8ec7-00e6364fb253</td>\n",
       "      <td>2021-05-10 09:00:25+00:00</td>\n",
       "      <td>60fd02c90b468b96daf5d3244bdc62631c60e8e2</td>\n",
       "      <td>dcbc5797682d1a4aea5f17ef88627c2ab83501a7</td>\n",
       "      <td>350.677562</td>\n",
       "      <td>85fcce17-2788-481c-83e5-9be65a2219d9</td>\n",
       "      <td>[\"3c87bfd70bf46bfdda0e3324f3a03008a5ba7047\", \"...</td>\n",
       "      <td>[\"ab2348a4cf38fd70b3b39de17b2ae4d4c88d7ed0\", \"...</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>350.677562</td>\n",
       "      <td>999.276104</td>\n",
       "      <td>1349.953666</td>\n",
       "      <td>[999.2761039800785, 350.6775618423433]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2021-05-18 20:28:12.411255+00:00</td>\n",
       "      <td>0.999548</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 39 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                order_id                order_date  \\\n",
       "66  9c2b6d93-32a7-45fa-ac40-a47059e4b7c4 2021-05-04 12:29:59+00:00   \n",
       "18  a489c647-bd28-456e-ab10-f4dde1151a20 2021-05-18 14:09:46+00:00   \n",
       "20  b9df0d87-c37f-4837-a81e-27e23bedc4a8 2021-05-04 18:02:34+00:00   \n",
       "48  564739cb-f238-487f-8893-3f8a5c42afe9 2021-05-05 21:21:08+00:00   \n",
       "55  39560977-f66c-4be8-8ec7-00e6364fb253 2021-05-10 09:00:25+00:00   \n",
       "\n",
       "                                       email  \\\n",
       "66  20d3a0d5ee5fce50d1cc660ea94e8fd6dc3e0d5d   \n",
       "18  7d95a0edb3399a16e29f353350793a55489af201   \n",
       "20  95d1b98298ff8558bebbcd5b68b81426d951dc9d   \n",
       "48  164e9cd15154ecf244a56d1afa9f7182e12eb038   \n",
       "55  60fd02c90b468b96daf5d3244bdc62631c60e8e2   \n",
       "\n",
       "                                     ip_addr  order_amount  \\\n",
       "66  804833f0387c5c3a0aed1218687be20d19e4aa15    222.106098   \n",
       "18  3de6a5ab79cda5e3a73eff8a2bbb506dab451293    121.437929   \n",
       "20  8d90d3f40f9b4c1974f42ba9cfb3f3513e55018a    801.040739   \n",
       "48  dadd450850a73d8c87ffef6f6c183a7fd03105e0     80.334187   \n",
       "55  dcbc5797682d1a4aea5f17ef88627c2ab83501a7    350.677562   \n",
       "\n",
       "                             customer_id  \\\n",
       "66  c6eb3cd2-93bc-413f-a0e9-61c71ab0d224   \n",
       "18  2355ad27-ea68-4439-a085-1d294cc772ec   \n",
       "20  23925e34-a97b-4a39-9af7-556994f4de50   \n",
       "48  78378996-89c5-4667-bf77-50a7bcf3899a   \n",
       "55  85fcce17-2788-481c-83e5-9be65a2219d9   \n",
       "\n",
       "                                 email_by_customer_id  \\\n",
       "66  [\"5402e6915b106c264bd2bd7d7194e45d6de83091\", \"...   \n",
       "18  [\"7d95a0edb3399a16e29f353350793a55489af201\", \"...   \n",
       "20  [\"c5a4b2c53a0f1060909717f27a26b6b611c6abac\", \"...   \n",
       "48       [\"164e9cd15154ecf244a56d1afa9f7182e12eb038\"]   \n",
       "55  [\"3c87bfd70bf46bfdda0e3324f3a03008a5ba7047\", \"...   \n",
       "\n",
       "                               ip_addr_by_customer_id  \\\n",
       "66  [\"29854c437e114323911562c081daa967fc2418a4\", \"...   \n",
       "18  [\"f898fbab3f673df0b0049098814b2ffabd277471\", \"...   \n",
       "20  [\"e09327b09677fd88e72c6007d218ce042cba0f60\", \"...   \n",
       "48       [\"dadd450850a73d8c87ffef6f6c183a7fd03105e0\"]   \n",
       "55  [\"ab2348a4cf38fd70b3b39de17b2ae4d4c88d7ed0\", \"...   \n",
       "\n",
       "    num_email_by_customer_id  num_ip_addr_by_customer_id  ...  \\\n",
       "66                         2                           2  ...   \n",
       "18                         2                           4  ...   \n",
       "20                         3                           3  ...   \n",
       "48                         1                           1  ...   \n",
       "55                         2                           3  ...   \n",
       "\n",
       "   order_amount_min_by_email order_amount_max_by_email  \\\n",
       "66                222.106098                222.106098   \n",
       "18                101.425959                928.822007   \n",
       "20                801.040739                801.040739   \n",
       "48                 80.334187                 80.334187   \n",
       "55                350.677562                999.276104   \n",
       "\n",
       "    order_amount_sum_by_email  \\\n",
       "66                 222.106098   \n",
       "18                1151.685896   \n",
       "20                 801.040739   \n",
       "48                  80.334187   \n",
       "55                1349.953666   \n",
       "\n",
       "                                order_amount_by_email  is_fraud  sample  \\\n",
       "66                                [222.1060979147914]         0       1   \n",
       "18  [928.8220074790162, 101.42595902612995, 121.43...         0       1   \n",
       "20                                 [801.040738517752]         0       1   \n",
       "48                                [80.33418734448439]         0       1   \n",
       "55             [999.2761039800785, 350.6775618423433]         0       0   \n",
       "\n",
       "                                now  elapsed_wgt  validation  fold  \n",
       "66 2021-05-18 20:28:12.411255+00:00     0.999236           1     4  \n",
       "18 2021-05-18 20:28:12.411255+00:00     0.999986           1     6  \n",
       "20 2021-05-18 20:28:12.411255+00:00     0.999248           1     6  \n",
       "48 2021-05-18 20:28:12.411255+00:00     0.999309           1     6  \n",
       "55 2021-05-18 20:28:12.411255+00:00     0.999548           1     8  \n",
       "\n",
       "[5 rows x 39 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = dd.read_parquet('./data/training/validation/')\n",
    "df[df.is_fraud == 0].head(npartitions=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}