01_DataPrep.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating Azure Resources\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from azure.core.exceptions import ResourceExistsError\n",
    "from azure.storage.blob import BlobServiceClient\n",
    "from azureml.core import Workspace\n",
    "from azureml.core.authentication import ServicePrincipalAuthentication, AzureCliAuthentication, \\\n",
    "    InteractiveLoginAuthentication\n",
    "\n",
    "import json\n",
    "from datetime import datetime\n",
    "from azureml.exceptions import AuthenticationException\n",
    "from dotenv import find_dotenv, get_key"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Start with specifying your subscription, resource group, and authentication information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# Subscription & resource group\n",
    "subscription_id = \"\"  # Name of an existing subscription \n",
    "resource_group = \"\"  # Name of the resource group that would be created\n",
    "workspace_region = \"\"  # location where the resources would be created (Ex: \"eastus\")"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "Below, we retrieve the AML workspace and create a compute target."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def get_auth(env_path):\n",
    "    if get_key(env_path, 'password') != \"YOUR_SERVICE_PRINCIPAL_PASSWORD\":\n",
    "        aml_sp_password = get_key(env_path, 'password')\n",
    "        aml_sp_tennant_id = get_key(env_path, 'tenant_id')\n",
    "        aml_sp_username = get_key(env_path, 'username')\n",
    "        auth = ServicePrincipalAuthentication(\n",
    "            tenant_id=aml_sp_tennant_id,\n",
    "            service_principal_id=aml_sp_username,\n",
    "            service_principal_password=aml_sp_password\n",
    "        )\n",
    "    else:\n",
    "        try:\n",
    "            auth = AzureCliAuthentication()\n",
    "            auth.get_authentication_header()\n",
    "        except AuthenticationException:\n",
    "            auth = InteractiveLoginAuthentication()\n",
    "\n",
    "    return auth\n",
    "\n",
    "# Get AML workspace\n",
    "env_path = find_dotenv(raise_error_if_not_found=True)\n",
    "ws = Workspace.from_config(auth=get_auth(env_path))\n",
    "print(ws.name, ws.resource_group, ws.location, sep=\"\\n\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following are parameters needed to create and access the main Azure resources. \n",
    "You can use the default values below as is, or feel free to modify as needed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "now_str = datetime.now().strftime(\"%y%m%d%H%M%S\")\n",
    "# AML workspace and compute target\n",
    "AML_WORKSPACE = \"ws{}\".format(now_str)\n",
    "AML_COMPUTE_NAME = \"cmp{}\".format(now_str) # limit to 16 chars\n",
    "\n",
    "# AML scheduling\n",
    "SCHED_FREQUENCY = \"Hour\"\n",
    "SCHED_INTERVAL = 1\n",
    "\n",
    "# Scoring script\n",
    "PIP_PACKAGES = [\"numpy\", \"scipy\", \"scikit-learn\", \"pandas\"]\n",
    "PYTHON_VERSION = \"3.6.7\"\n",
    "PYTHON_SCRIPT_NAME = \"predict.py\"\n",
    "PYTHON_SCRIPT_DIRECTORY = \"scripts\"\n",
    "\n",
    "# Blob storage\n",
    "BLOB_ACCOUNT = \"ba{}\".format(now_str) # limit to 24 chars\n",
    "MODELS_CONTAINER = \"models\"\n",
    "PREDS_CONTAINER = \"preds\"\n",
    "DATA_CONTAINER = \"data\"\n",
    "DATA_BLOB = \"sensor_data.csv\"  # name of data file to be copied to blob storage\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we retrieve the Blob storage account and create containers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "blob_datastore = ws.get_default_datastore()\n",
    "\n",
    "blob_account = blob_datastore.account_name\n",
    "blob_key = blob_datastore.account_key\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create models, predictions and data containers\n",
    "service = BlobServiceClient(account_url=\"https://\"+blob_account+\".blob.core.windows.net/\", credential=blob_key)\n",
    "\n",
    "for container in [MODELS_CONTAINER, PREDS_CONTAINER, DATA_CONTAINER]:\n",
    "    try:\n",
    "        service.create_container(container)\n",
    "    except ResourceExistsError: \n",
    "        print(str(container) + ' - Exists')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following commands copy the pre-trained models and sample data from this repo to blob storage so that AML can access them during job submission."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azure.storage.blob import BlobClient\n",
    "\n",
    "try:\n",
    "    data_container = service.get_container_client(DATA_CONTAINER)\n",
    "    with open(\"./data/sensor_data.csv\", \"rb\") as data:    \n",
    "        data_container.upload_blob(\"sensor_data.csv\", data)\n",
    "except ResourceExistsError: \n",
    "    print('sensor_data.csv - Exists')\n",
    "\n",
    "try:\n",
    "    models_container = service.get_container_client(MODELS_CONTAINER)\n",
    "    for model in os.listdir(\"models\"):\n",
    "        with open(\"./models/\"+model, \"rb\") as data:    \n",
    "            models_container.upload_blob(model, data)\n",
    "except ResourceExistsError: \n",
    "    print('models dir - Exists')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We also generate a json config file, that we use in the next notebook to create and schedule the AML pipeline. The config file includes AML, Blob storage, and Python parameters.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_config = {\n",
    "  \"resource_group_name\": resource_group,\n",
    "  \"subscription_id\": subscription_id,\n",
    "  \"aml_work_space\": AML_WORKSPACE,\n",
    "  \"experiment_name\": \"mm_score\",\n",
    "  \"cluster_name\": AML_COMPUTE_NAME,\n",
    "  \"workspace_region\": workspace_region,\n",
    "  \"blob_account\": BLOB_ACCOUNT,\n",
    "  \"blob_key\": blob_key,\n",
    "  \"models_blob_container\": MODELS_CONTAINER,\n",
    "  \"data_blob_container\": DATA_CONTAINER,\n",
    "  \"data_blob\" : DATA_BLOB,\n",
    "  \"preds_blob_container\": PREDS_CONTAINER,\n",
    "  \"pip_packages\": PIP_PACKAGES,\n",
    "  \"python_version\": PYTHON_VERSION,\n",
    "  \"python_script_name\": PYTHON_SCRIPT_NAME,\n",
    "  \"python_script_directory\": PYTHON_SCRIPT_DIRECTORY,\n",
    "  \"sched_frequency\": SCHED_FREQUENCY,\n",
    "  \"sched_interval\": SCHED_INTERVAL,\n",
    "  \"device_ids\": [ 1, 2, 3 ],\n",
    "  \"sensors\": [ 1, 2, 3, 4, 5 ]\n",
    "}\n",
    "with open('pipeline_config.json', 'w') as f:\n",
    "    json.dump(pipeline_config, f, indent=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we will create and schedule the scoring pipeline using AML. Follow the instructions in this [notebook](02_create_pipeline.ipynb)."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:amlmm]",
   "language": "python",
   "name": "conda-env-amlmm-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}