00_jobs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp jobs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Jobs Management\n",
    "\n",
    "> This module define helper functions to obtain connection to Dask and a bunch of dataclasses to communicate Job execution between different steps of the training pipeline."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from typing import Dict, Optional, Any\n",
    "\n",
    "from hopeit.app.context import EventContext\n",
    "from dataclasses import dataclass, field\n",
    "from hopeit.dataobjects import dataobject"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "async def get_client_async(context: Optional[EventContext] = None):\n",
    "    from dask.distributed import Client\n",
    "    url = \"local\" if context is None else context.env['dask']['url']\n",
    "    if url == \"local\":\n",
    "        client = await Client(asynchronous=True)\n",
    "    else:\n",
    "        client = await Client(url, asynchronous=True)\n",
    "    return client\n",
    "\n",
    "def get_client(context: Optional[EventContext] = None):\n",
    "    from dask.distributed import Client\n",
    "    url = \"local\" if context is None else context.env['dask']['url']\n",
    "    if url == \"local\":\n",
    "        client = Client()\n",
    "    else:\n",
    "        client = Client(url)\n",
    "    return client\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "@dataobject\n",
    "@dataclass\n",
    "class MakeSampleDataJob:\n",
    "    path: str\n",
    "    num_batches: int = 0\n",
    "    batch_size: int = 0\n",
    "    batch_span_days: int = 0\n",
    "    num_customers: int = 0\n",
    "    num_emails: int = 0\n",
    "    num_ips: int = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "@dataobject\n",
    "@dataclass\n",
    "class PreprocessingJob:\n",
    "    source: str\n",
    "    partitioned: Dict[str, str] = field(default_factory=dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "@dataobject\n",
    "@dataclass\n",
    "class FeatureCalcJob:\n",
    "    sources: Dict[str, str] = field(default_factory=dict)\n",
    "    features: Dict[str, str] = field(default_factory=dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "@dataobject\n",
    "@dataclass\n",
    "class TrainingDataJob:\n",
    "    sources: Dict[str, str]\n",
    "    sampled: str\n",
    "    validation: str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "@dataobject\n",
    "@dataclass\n",
    "class TrainModelJob:\n",
    "    train_data: str\n",
    "    validation_data: str\n",
    "    model_path: str\n",
    "    evals: Dict[str, Any] = field(default_factory=dict)\n",
    "    test_roc_curve: Dict[float, float] = field(default_factory=dict)\n",
    "    validation_score: Dict[str, float] = field(default_factory=dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "@dataobject\n",
    "@dataclass\n",
    "class PrepareDbJob:\n",
    "    features: Dict[str, str]\n",
    "    db: str\n",
    "    saved: Dict[str, Any]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}