hyperparameter_tuning.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Hyperparameter Tuning using HyperDrive\n",
        "\n",
        "TODO: Import Dependencies. In the cell below, import all the dependencies that you will need to complete the project."
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "from azureml.core import Workspace, Experiment\n",
        "from azureml.core.compute import ComputeTarget, AmlCompute\n",
        "from azureml.core.compute_target import ComputeTargetException\n",
        "\n",
        "from matplotlib import pyplot as plt\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn import datasets\n",
        "import pkg_resources\n",
        "\n",
        "from azureml.widgets import RunDetails\n",
        "from azureml.train.sklearn import SKLearn\n",
        "from azureml.train.hyperdrive.run import PrimaryMetricGoal\n",
        "from azureml.train.hyperdrive.policy import BanditPolicy\n",
        "from azureml.train.hyperdrive.sampling import RandomParameterSampling\n",
        "from azureml.train.hyperdrive.runconfig import HyperDriveConfig\n",
        "from azureml.train.hyperdrive.parameter_expressions import choice, uniform\n",
        "import os\n",
        "\n",
        "import joblib"
      ],
      "outputs": [],
      "execution_count": 1,
      "metadata": {
        "gather": {
          "logged": 1612382326642
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "ws = Workspace.from_config()\n",
        "print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep = '\\n')"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Performing interactive authentication. Please follow the instructions on the terminal.\n",
            "To sign in, use a web browser to open the page https://microsoft.com/devicelogin and enter the code EL5B5LVFH to authenticate.\n",
            "You have logged in. Now let us find all the subscriptions to which you have access...\n",
            "Interactive authentication successfully completed.\n",
            "quick-starts-ws-137128\n",
            "aml-quickstarts-137128\n",
            "southcentralus\n",
            "610d6e37-4747-4a20-80eb-3aad70a55f43\n"
          ]
        }
      ],
      "execution_count": 2,
      "metadata": {
        "gather": {
          "logged": 1612382352879
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create compute cluster and choose a name for it\n",
        "cpu_cluster_name = \"compute-cluster\"\n",
        "\n",
        "# Verify that cluster does not exist already\n",
        "try:\n",
        "    compute_target = ComputeTarget(workspace=ws, name=cpu_cluster_name)\n",
        "    print('Found existing cluster, use it.')\n",
        "except ComputeTargetException:\n",
        "    print('Creating a new compute cluster...')\n",
        "    compute_config = AmlCompute.provisioning_configuration(vm_size='Standard_DS3_v2', min_nodes=1, max_nodes=4)\n",
        "    compute_target = ComputeTarget.create(ws, cpu_cluster_name, compute_config)\n",
        "\n",
        "# Can poll for a minimum number of nodes and for a specific timeout. \n",
        "# If no min node count is provided it uses the scale settings for the cluster.\n",
        "compute_target.wait_for_completion(show_output=True)\n",
        "\n",
        "# use get_status() to get a detailed status for the current cluster. \n",
        "print(compute_target.get_status().serialize())\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found existing cluster, use it.\n",
            "\n",
            "Running\n",
            "{'errors': [], 'creationTime': '2021-02-03T19:34:13.360040+00:00', 'createdBy': {'userObjectId': 'd09572f8-030d-4907-b22a-4b802e0f7788', 'userTenantId': '660b3398-b80e-49d2-bc5b-ac1dc93b5254', 'userName': 'ODL_User 137128'}, 'modifiedTime': '2021-02-03T19:37:00.587794+00:00', 'state': 'Running', 'vmSize': 'STANDARD_DS3_V2'}\n"
          ]
        }
      ],
      "execution_count": 3,
      "metadata": {
        "gather": {
          "logged": 1612382362592
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Dataset\n",
        "\n",
        "### Overview\n",
        "\n",
        "The dataset used is taken from [Kaggle](https://www.kaggle.com/andrewmvd/heart-failure-clinical-data) and the data comes from 299 patients with heart failure collected at the Faisalabad Institute of Cardiology and at the Allied Hospital in Faisalabad (Punjab, Pakistan), during April–December 2015. The patients consisted of both women (105) and men (194), and the main task of the project is to classify the patients based on their odds of survival.\n",
        "\n",
        "Dataset features:\n",
        "\n",
        "| Feature | Explanation |\n",
        "| :---: | :---: |\n",
        "| *age* | Age of patient |\n",
        "| *anaemia* | Decrease of red blood cells or hemoglobin |\n",
        "| *creatinine-phosphokinase* | Level of the CPK enzyme in the blood |\n",
        "| *diabetes* | Whether the patient has diabetes or not |\n",
        "| *ejection_fraction* | Percentage of blood leaving the heart at each contraction |\n",
        "| *high_blood_pressure* | Whether the patient has hypertension or not |\n",
        "| *platelets* | Platelets in the blood |\n",
        "| *serum_creatinine* | Level of creatinine in the blood |\n",
        "| *serum_sodium* | Level of sodium in the blood |\n",
        "| *sex* | Female (F) or Male (M) |\n",
        "| *smoking* | Whether the patient smokes or not |\n",
        "| *time* | Follow-up period |\n",
        "| *DEATH_EVENT* | Whether the patient died during the follow-up period |\n"
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "ws = Workspace.from_config()\n",
        "print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep = '\\n')\n",
        "\n",
        "data = pd.read_csv('./heart_failure_clinical_records_dataset.csv')\n",
        "\n",
        "found = False\n",
        "key = \"heart-failure-prediction\"\n",
        "description_text = \"Prediction of patients’ survival with heart failure - Capstone project\"\n",
        "\n",
        "if key in ws.datasets.keys(): \n",
        "        found = True\n",
        "        dataset = ws.datasets[key] \n",
        "\n",
        "if not found:\n",
        "        # Create AML Dataset and register it into Workspace\n",
        "        my_dataset = 'https://raw.githubusercontent.com/dimikara/Survival-Prediction-of-Patients-with-Heart-Failure/master/heart_failure_clinical_records_dataset.csv'\n",
        "        dataset = Dataset.Tabular.from_delimited_files(my_dataset)        \n",
        "        # Register Dataset in Workspace\n",
        "        dataset = dataset.register(workspace=ws,\n",
        "                                   name=key,\n",
        "                                   description=description_text)\n",
        "                                \n",
        "# Preview of the first five rows\n",
        "print(data.head())\n",
        "\n",
        "# Explore data\n",
        "print(data.describe())\n",
        "\n",
        "df = dataset.to_pandas_dataframe()\n",
        "df.describe()\n",
        "\n",
        "# Data columns\n",
        "df.columns = ['age', 'anaemia', 'creatinine_phosphokinase', 'diabetes', 'ejection_fraction', 'high_blood_pressure', 'platelets', 'serum_creatinine', 'serum_sodium', 'sex', 'smoking', 'time', 'DEATH_EVENT']\n",
        "x = df[['age', 'anaemia', 'creatinine_phosphokinase', 'diabetes', 'ejection_fraction', 'high_blood_pressure', 'platelets', 'serum_creatinine', 'serum_sodium', 'sex', 'smoking', 'time']]\n",
        "y = df[['DEATH_EVENT']]\n",
        "\n",
        "experiment_name = 'heart-failure-prediction'\n",
        "project_folder = './capstone-project'\n",
        "\n",
        "experiment = Experiment(ws, experiment_name)\n",
        "experiment\n",
        "\n",
        "run = experiment.start_logging()\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "quick-starts-ws-137128\n",
            "aml-quickstarts-137128\n",
            "southcentralus\n",
            "610d6e37-4747-4a20-80eb-3aad70a55f43\n",
            "    age  anaemia  creatinine_phosphokinase  diabetes  ejection_fraction  \\\n",
            "0  75.0        0                       582         0                 20   \n",
            "1  55.0        0                      7861         0                 38   \n",
            "2  65.0        0                       146         0                 20   \n",
            "3  50.0        1                       111         0                 20   \n",
            "4  65.0        1                       160         1                 20   \n",
            "\n",
            "   high_blood_pressure  platelets  serum_creatinine  serum_sodium  sex  \\\n",
            "0                    1  265000.00               1.9           130    1   \n",
            "1                    0  263358.03               1.1           136    1   \n",
            "2                    0  162000.00               1.3           129    1   \n",
            "3                    0  210000.00               1.9           137    1   \n",
            "4                    0  327000.00               2.7           116    0   \n",
            "\n",
            "   smoking  time  DEATH_EVENT  \n",
            "0        0     4            1  \n",
            "1        0     6            1  \n",
            "2        1     7            1  \n",
            "3        0     7            1  \n",
            "4        0     8            1  \n",
            "              age     anaemia  creatinine_phosphokinase    diabetes  \\\n",
            "count  299.000000  299.000000                299.000000  299.000000   \n",
            "mean    60.833893    0.431438                581.839465    0.418060   \n",
            "std     11.894809    0.496107                970.287881    0.494067   \n",
            "min     40.000000    0.000000                 23.000000    0.000000   \n",
            "25%     51.000000    0.000000                116.500000    0.000000   \n",
            "50%     60.000000    0.000000                250.000000    0.000000   \n",
            "75%     70.000000    1.000000                582.000000    1.000000   \n",
            "max     95.000000    1.000000               7861.000000    1.000000   \n",
            "\n",
            "       ejection_fraction  high_blood_pressure      platelets  \\\n",
            "count         299.000000           299.000000     299.000000   \n",
            "mean           38.083612             0.351171  263358.029264   \n",
            "std            11.834841             0.478136   97804.236869   \n",
            "min            14.000000             0.000000   25100.000000   \n",
            "25%            30.000000             0.000000  212500.000000   \n",
            "50%            38.000000             0.000000  262000.000000   \n",
            "75%            45.000000             1.000000  303500.000000   \n",
            "max            80.000000             1.000000  850000.000000   \n",
            "\n",
            "       serum_creatinine  serum_sodium         sex    smoking        time  \\\n",
            "count         299.00000    299.000000  299.000000  299.00000  299.000000   \n",
            "mean            1.39388    136.625418    0.648829    0.32107  130.260870   \n",
            "std             1.03451      4.412477    0.478136    0.46767   77.614208   \n",
            "min             0.50000    113.000000    0.000000    0.00000    4.000000   \n",
            "25%             0.90000    134.000000    0.000000    0.00000   73.000000   \n",
            "50%             1.10000    137.000000    1.000000    0.00000  115.000000   \n",
            "75%             1.40000    140.000000    1.000000    1.00000  203.000000   \n",
            "max             9.40000    148.000000    1.000000    1.00000  285.000000   \n",
            "\n",
            "       DEATH_EVENT  \n",
            "count    299.00000  \n",
            "mean       0.32107  \n",
            "std        0.46767  \n",
            "min        0.00000  \n",
            "25%        0.00000  \n",
            "50%        0.00000  \n",
            "75%        1.00000  \n",
            "max        1.00000  \n"
          ]
        }
      ],
      "execution_count": 4,
      "metadata": {
        "gather": {
          "logged": 1612382377332
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Hyperdrive Configuration\n",
        "\n",
        "For this experiment I am using a custom Scikit-learn Logistic Regression model, whose hyperparameters I am optimising using HyperDrive.\n",
        "\n",
        "***Early stopping policy***\n",
        "\n",
        "An early stopping policy is used to automatically terminate poorly performing runs thus improving computational efficiency. I chose the ***BanditPolicy*** which I specified as follows:\n",
        "\n",
        "***policy = BanditPolicy(evaluation_interval=2, slack_factor=0.1)***\n",
        "\n",
        "where:\n",
        "\n",
        "***evaluation_interval***: This is optional and represents the frequency for applying the policy. Each time the training script logs the primary metric counts as one interval.\n",
        "\n",
        "***slack_factor***: The amount of slack allowed with respect to the best performing training run. This factor specifies the slack as a ratio.\n",
        "\n",
        "Any run that doesn't fall within the slack factor or slack amount of the evaluation metric with respect to the best performing run will be terminated. This means that with this policy, the best performing runs will execute until they finish and this is the reason I chose it.\n",
        "\n",
        "### Parameter Sampler\n",
        "\n",
        "I specify the parameter sampler using the parameters ***C*** and ***max_iter***. I chose discrete values with choice for both parameters.\n",
        "\n",
        "***C*** is the Regularization while ***max_iter*** is the maximum number of iterations.\n",
        "\n",
        "***RandomParameterSampling*** is one of the choices available for the sampler and I chose it because it is the faster and supports early termination of low-performance runs. If budget is not an issue, it would be better to use GridParameterSampling to exhaustively search over the search space or BayesianParameterSampling to explore the hyperparameter space.\n",
        "\n",
        "### HyperDriveConfig\n",
        "\n",
        "The configuration chosen is as follows:\n",
        "\n",
        "***hyperparameter_sampling***\n",
        "\n",
        "The hyperparameter sampling space as defined above.\n",
        "\n",
        "***primary_metric_name*** \n",
        "\n",
        "The name of the primary metric reported by the experiment runs. In our case, it is `Accuracy`.\n",
        "\n",
        "***primary_metric_goal***\n",
        "\n",
        "I chose `PrimaryMetricGoal.MAXIMIZE`. This parameter determines that the primary metric is to be maximized when evaluating runs.\n",
        "\n",
        "***policy*** \n",
        "\n",
        "It refers to the early termination policy that is specified above.\n",
        "\n",
        "***estimator***\n",
        "\n",
        "An estimator that will be called with sampled hyperparameters. In this case, I choose `estimator` while the other two options are `run_config` and `pipeline`.\n",
        "The estimator will be used with train.py file which does a very basic manipulation of the data.\n",
        "\n",
        "***max_total_runs=16***\n",
        "\n",
        "The maximum total number of runs to create. This is the upper bound; there may be fewer runs when the sample space is smaller than this value. If both max_total_runs and max_duration_minutes are specified, the hyperparameter tuning experiment terminates when the first of these two thresholds is reached.\n",
        "\n",
        "***max_concurrent_runs=4***\n",
        "\n",
        "The maximum number of runs to execute concurrently. If None, all runs are launched in parallel. The number of concurrent runs is gated on the resources available in the specified compute target. Hence, you need to ensure that the compute target has the available resources for the desired concurrency.\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "collapsed": true,
        "gather": {
          "logged": 1598531923519
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create an early termination policy. This is not required if you are using Bayesian sampling.\n",
        "policy = BanditPolicy(evaluation_interval=2, slack_factor=0.1)\n",
        "\n",
        "# Specify parameter sampler \n",
        "# Create the different params that you will be using during training\n",
        "\n",
        "ps = RandomParameterSampling(\n",
        "    {\n",
        "        '--C' : choice(0.001,0.01,0.1,1,10,20,50,100,200,500,1000),\n",
        "        '--max_iter': choice(50,100,200,300)\n",
        "    }\n",
        ")\n",
        "\n",
        "if \"training\" not in os.listdir():\n",
        "    os.mkdir(\"./training\")\n",
        "\n",
        "# Create a SKLearn estimator for use with train.py\n",
        "est = SKLearn(source_directory = \"./\",\n",
        "            compute_target=compute_target,\n",
        "            vm_size='STANDARD_DS3_V2',\n",
        "            entry_script=\"train.py\")\n",
        "\n",
        "# Create a HyperDriveConfig using the estimator, hyperparameter sampler, and policy.\n",
        "hyperdrive_config = HyperDriveConfig(hyperparameter_sampling=ps, \n",
        "                                     primary_metric_name='Accuracy',\n",
        "                                     primary_metric_goal=PrimaryMetricGoal.MAXIMIZE,\n",
        "                                     policy=policy,\n",
        "                                     estimator=est,\n",
        "                                     max_total_runs=16,\n",
        "                                     max_concurrent_runs=4)\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "'SKLearn' estimator is deprecated. Please use 'ScriptRunConfig' from 'azureml.core.script_run_config' with your own defined environment or the AzureML-Tutorial curated environment.\n"
          ]
        }
      ],
      "execution_count": 5,
      "metadata": {
        "gather": {
          "logged": 1612382381642
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Run Details\n",
        "\n",
        "In the cell below, I use the `RunDetails` widget to show the different experiments."
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "# Submit the hyperdrive run to the experiment and show run details with the widget.\n",
        "\n",
        "# Start the HyperDrive run\n",
        "hyperdrive_run = experiment.submit(hyperdrive_config)\n",
        "\n",
        "# Monitor HyperDrive runs \n",
        "# You can monitor the progress of the runs with the following Jupyter widget\n",
        "RunDetails(hyperdrive_run).show()\n",
        "\n",
        "hyperdrive_run.wait_for_completion(show_output=True)\n",
        "\n",
        "assert(hyperdrive_run.get_status() == \"Completed\")\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:root:If 'script' has been provided here and a script file name has been specified in 'run_config', 'script' provided in ScriptRunConfig initialization will take precedence.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "_HyperDriveWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': False, 'log_level': 'INFO'…",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6fd1eeb24e294b3da7179b20f6a0bff4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/aml.mini.widget.v1": "{\"status\": \"Completed\", \"workbench_run_details_uri\": \"https://ml.azure.com/experiments/heart-failure-prediction/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4?wsid=/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourcegroups/aml-quickstarts-137128/workspaces/quick-starts-ws-137128\", \"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4\", \"run_properties\": {\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4\", \"created_utc\": \"2021-02-03T19:59:53.279669Z\", \"properties\": {\"primary_metric_config\": \"{\\\"name\\\": \\\"Accuracy\\\", \\\"goal\\\": \\\"maximize\\\"}\", \"resume_from\": \"null\", \"runTemplate\": \"HyperDrive\", \"azureml.runsource\": \"hyperdrive\", \"platform\": \"AML\", \"ContentSnapshotId\": \"b0fa1234-4c3e-41c9-bf41-eac3d4dc177f\", \"score\": \"0.8333333333333334\", \"best_child_run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_1\", \"best_metric_status\": \"Succeeded\"}, \"tags\": {\"_aml_system_max_concurrent_jobs\": \"4\", \"max_concurrent_jobs\": \"4\", \"_aml_system_max_total_jobs\": \"16\", \"max_total_jobs\": \"16\", \"_aml_system_max_duration_minutes\": \"10080\", \"max_duration_minutes\": \"10080\", \"_aml_system_policy_config\": \"{\\\"name\\\": \\\"BANDIT\\\", \\\"properties\\\": {\\\"evaluation_interval\\\": 2, \\\"delay_evaluation\\\": 0, \\\"slack_factor\\\": 0.1}}\", \"policy_config\": \"{\\\"name\\\": \\\"BANDIT\\\", \\\"properties\\\": {\\\"evaluation_interval\\\": 2, \\\"delay_evaluation\\\": 0, \\\"slack_factor\\\": 0.1}}\", \"_aml_system_generator_config\": \"{\\\"name\\\": \\\"RANDOM\\\", \\\"parameter_space\\\": {\\\"--C\\\": [\\\"choice\\\", [[0.001, 0.01, 0.1, 1, 10, 20, 50, 100, 200, 500, 1000]]], \\\"--max_iter\\\": [\\\"choice\\\", [[50, 100, 200, 300]]]}}\", \"generator_config\": \"{\\\"name\\\": \\\"RANDOM\\\", \\\"parameter_space\\\": {\\\"--C\\\": [\\\"choice\\\", [[0.001, 0.01, 0.1, 1, 10, 20, 50, 100, 200, 500, 1000]]], \\\"--max_iter\\\": [\\\"choice\\\", [[50, 100, 200, 300]]]}}\", \"_aml_system_primary_metric_config\": \"{\\\"name\\\": \\\"Accuracy\\\", \\\"goal\\\": \\\"maximize\\\"}\", \"primary_metric_config\": \"{\\\"name\\\": \\\"Accuracy\\\", \\\"goal\\\": \\\"maximize\\\"}\", \"_aml_system_platform_config\": \"{\\\"ServiceAddress\\\": \\\"https://southcentralus.api.azureml.ms\\\", \\\"ServiceArmScope\\\": \\\"subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/heart-failure-prediction\\\", \\\"SubscriptionId\\\": \\\"610d6e37-4747-4a20-80eb-3aad70a55f43\\\", \\\"ResourceGroupName\\\": \\\"aml-quickstarts-137128\\\", \\\"WorkspaceName\\\": \\\"quick-starts-ws-137128\\\", \\\"ExperimentName\\\": \\\"heart-failure-prediction\\\", \\\"Definition\\\": {\\\"Overrides\\\": {\\\"script\\\": \\\"train.py\\\", \\\"arguments\\\": [], \\\"target\\\": \\\"compute-cluster\\\", \\\"framework\\\": \\\"Python\\\", \\\"communicator\\\": \\\"None\\\", \\\"maxRunDurationSeconds\\\": null, \\\"nodeCount\\\": 1, \\\"environment\\\": {\\\"name\\\": null, \\\"version\\\": null, \\\"environmentVariables\\\": {\\\"EXAMPLE_ENV_VAR\\\": \\\"EXAMPLE_VALUE\\\"}, \\\"python\\\": {\\\"userManagedDependencies\\\": false, \\\"interpreterPath\\\": \\\"python\\\", \\\"condaDependenciesFile\\\": null, \\\"baseCondaEnvironment\\\": null, \\\"condaDependencies\\\": {\\\"name\\\": \\\"project_environment\\\", \\\"dependencies\\\": [\\\"python=3.6.2\\\", {\\\"pip\\\": [\\\"azureml-defaults\\\", \\\"scikit-learn==0.20.3\\\", \\\"scipy==1.2.1\\\", \\\"joblib==0.13.2\\\"]}], \\\"channels\\\": [\\\"anaconda\\\", \\\"conda-forge\\\"]}}, \\\"docker\\\": {\\\"enabled\\\": true, \\\"baseImage\\\": \\\"mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20200423.v1\\\", \\\"baseDockerfile\\\": null, \\\"sharedVolumes\\\": true, \\\"shmSize\\\": \\\"2g\\\", \\\"arguments\\\": [], \\\"baseImageRegistry\\\": {\\\"address\\\": null, \\\"username\\\": null, \\\"password\\\": null, \\\"registryIdentity\\\": null}, \\\"platform\\\": {\\\"os\\\": \\\"Linux\\\", \\\"architecture\\\": \\\"amd64\\\"}}, \\\"spark\\\": {\\\"repositories\\\": [], \\\"packages\\\": [], \\\"precachePackages\\\": false}, \\\"databricks\\\": {\\\"mavenLibraries\\\": [], \\\"pypiLibraries\\\": [], \\\"rcranLibraries\\\": [], \\\"jarLibraries\\\": [], \\\"eggLibraries\\\": []}, \\\"r\\\": null, \\\"inferencingStackVersion\\\": null}, \\\"history\\\": {\\\"outputCollection\\\": true, \\\"snapshotProject\\\": true, \\\"directoriesToWatch\\\": [\\\"logs\\\"]}, \\\"spark\\\": {\\\"configuration\\\": {\\\"spark.app.name\\\": \\\"Azure ML Experiment\\\", \\\"spark.yarn.maxAppAttempts\\\": 1}}, \\\"hdi\\\": {\\\"yarnDeployMode\\\": \\\"cluster\\\"}, \\\"tensorflow\\\": {\\\"workerCount\\\": 1, \\\"parameterServerCount\\\": 1}, \\\"mpi\\\": {\\\"processCountPerNode\\\": 1, \\\"nodeCount\\\": 1}, \\\"paralleltask\\\": {\\\"maxRetriesPerWorker\\\": 0, \\\"workerCountPerNode\\\": 1, \\\"terminalExitCodes\\\": null}, \\\"dataReferences\\\": {}, \\\"data\\\": {}, \\\"outputData\\\": {}, \\\"sourceDirectoryDataStore\\\": null, \\\"amlcompute\\\": {\\\"vmSize\\\": \\\"STANDARD_DS3_V2\\\", \\\"vmPriority\\\": null, \\\"retainCluster\\\": false, \\\"name\\\": null, \\\"clusterMaxNodeCount\\\": 1}, \\\"command\\\": \\\"\\\"}, \\\"TargetDetails\\\": null, \\\"SnapshotId\\\": \\\"b0fa1234-4c3e-41c9-bf41-eac3d4dc177f\\\", \\\"TelemetryValues\\\": {\\\"amlClientType\\\": \\\"azureml-sdk-train\\\", \\\"amlClientModule\\\": \\\"[Scrubbed]\\\", \\\"amlClientFunction\\\": \\\"[Scrubbed]\\\", \\\"tenantId\\\": \\\"660b3398-b80e-49d2-bc5b-ac1dc93b5254\\\", \\\"amlClientRequestId\\\": \\\"efab072d-a5cd-4e1b-82b3-eac16c72abe6\\\", \\\"amlClientSessionId\\\": \\\"545822e6-b5cd-49f7-a117-c5829e07fee3\\\", \\\"subscriptionId\\\": \\\"610d6e37-4747-4a20-80eb-3aad70a55f43\\\", \\\"estimator\\\": \\\"SKLearn\\\", \\\"samplingMethod\\\": \\\"RANDOM\\\", \\\"terminationPolicy\\\": \\\"Bandit\\\", \\\"primaryMetricGoal\\\": \\\"maximize\\\", \\\"maxTotalRuns\\\": 16, \\\"maxConcurrentRuns\\\": 4, \\\"maxDurationMinutes\\\": 10080, \\\"vmSize\\\": \\\"STANDARD_DS3_V2\\\"}}}\", \"platform_config\": \"{\\\"ServiceAddress\\\": \\\"https://southcentralus.api.azureml.ms\\\", \\\"ServiceArmScope\\\": \\\"subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/heart-failure-prediction\\\", \\\"SubscriptionId\\\": \\\"610d6e37-4747-4a20-80eb-3aad70a55f43\\\", \\\"ResourceGroupName\\\": \\\"aml-quickstarts-137128\\\", \\\"WorkspaceName\\\": \\\"quick-starts-ws-137128\\\", \\\"ExperimentName\\\": \\\"heart-failure-prediction\\\", \\\"Definition\\\": {\\\"Overrides\\\": {\\\"script\\\": \\\"train.py\\\", \\\"arguments\\\": [], \\\"target\\\": \\\"compute-cluster\\\", \\\"framework\\\": \\\"Python\\\", \\\"communicator\\\": \\\"None\\\", \\\"maxRunDurationSeconds\\\": null, \\\"nodeCount\\\": 1, \\\"environment\\\": {\\\"name\\\": null, \\\"version\\\": null, \\\"environmentVariables\\\": {\\\"EXAMPLE_ENV_VAR\\\": \\\"EXAMPLE_VALUE\\\"}, \\\"python\\\": {\\\"userManagedDependencies\\\": false, \\\"interpreterPath\\\": \\\"python\\\", \\\"condaDependenciesFile\\\": null, \\\"baseCondaEnvironment\\\": null, \\\"condaDependencies\\\": {\\\"name\\\": \\\"project_environment\\\", \\\"dependencies\\\": [\\\"python=3.6.2\\\", {\\\"pip\\\": [\\\"azureml-defaults\\\", \\\"scikit-learn==0.20.3\\\", \\\"scipy==1.2.1\\\", \\\"joblib==0.13.2\\\"]}], \\\"channels\\\": [\\\"anaconda\\\", \\\"conda-forge\\\"]}}, \\\"docker\\\": {\\\"enabled\\\": true, \\\"baseImage\\\": \\\"mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20200423.v1\\\", \\\"baseDockerfile\\\": null, \\\"sharedVolumes\\\": true, \\\"shmSize\\\": \\\"2g\\\", \\\"arguments\\\": [], \\\"baseImageRegistry\\\": {\\\"address\\\": null, \\\"username\\\": null, \\\"password\\\": null, \\\"registryIdentity\\\": null}, \\\"platform\\\": {\\\"os\\\": \\\"Linux\\\", \\\"architecture\\\": \\\"amd64\\\"}}, \\\"spark\\\": {\\\"repositories\\\": [], \\\"packages\\\": [], \\\"precachePackages\\\": false}, \\\"databricks\\\": {\\\"mavenLibraries\\\": [], \\\"pypiLibraries\\\": [], \\\"rcranLibraries\\\": [], \\\"jarLibraries\\\": [], \\\"eggLibraries\\\": []}, \\\"r\\\": null, \\\"inferencingStackVersion\\\": null}, \\\"history\\\": {\\\"outputCollection\\\": true, \\\"snapshotProject\\\": true, \\\"directoriesToWatch\\\": [\\\"logs\\\"]}, \\\"spark\\\": {\\\"configuration\\\": {\\\"spark.app.name\\\": \\\"Azure ML Experiment\\\", \\\"spark.yarn.maxAppAttempts\\\": 1}}, \\\"hdi\\\": {\\\"yarnDeployMode\\\": \\\"cluster\\\"}, \\\"tensorflow\\\": {\\\"workerCount\\\": 1, \\\"parameterServerCount\\\": 1}, \\\"mpi\\\": {\\\"processCountPerNode\\\": 1, \\\"nodeCount\\\": 1}, \\\"paralleltask\\\": {\\\"maxRetriesPerWorker\\\": 0, \\\"workerCountPerNode\\\": 1, \\\"terminalExitCodes\\\": null}, \\\"dataReferences\\\": {}, \\\"data\\\": {}, \\\"outputData\\\": {}, \\\"sourceDirectoryDataStore\\\": null, \\\"amlcompute\\\": {\\\"vmSize\\\": \\\"STANDARD_DS3_V2\\\", \\\"vmPriority\\\": null, \\\"retainCluster\\\": false, \\\"name\\\": null, \\\"clusterMaxNodeCount\\\": 1}, \\\"command\\\": \\\"\\\"}, \\\"TargetDetails\\\": null, \\\"SnapshotId\\\": \\\"b0fa1234-4c3e-41c9-bf41-eac3d4dc177f\\\", \\\"TelemetryValues\\\": {\\\"amlClientType\\\": \\\"azureml-sdk-train\\\", \\\"amlClientModule\\\": \\\"[Scrubbed]\\\", \\\"amlClientFunction\\\": \\\"[Scrubbed]\\\", \\\"tenantId\\\": \\\"660b3398-b80e-49d2-bc5b-ac1dc93b5254\\\", \\\"amlClientRequestId\\\": \\\"efab072d-a5cd-4e1b-82b3-eac16c72abe6\\\", \\\"amlClientSessionId\\\": \\\"545822e6-b5cd-49f7-a117-c5829e07fee3\\\", \\\"subscriptionId\\\": \\\"610d6e37-4747-4a20-80eb-3aad70a55f43\\\", \\\"estimator\\\": \\\"SKLearn\\\", \\\"samplingMethod\\\": \\\"RANDOM\\\", \\\"terminationPolicy\\\": \\\"Bandit\\\", \\\"primaryMetricGoal\\\": \\\"maximize\\\", \\\"maxTotalRuns\\\": 16, \\\"maxConcurrentRuns\\\": 4, \\\"maxDurationMinutes\\\": 10080, \\\"vmSize\\\": \\\"STANDARD_DS3_V2\\\"}}}\", \"_aml_system_resume_child_runs\": \"null\", \"resume_child_runs\": \"null\", \"_aml_system_all_jobs_generated\": \"true\", \"all_jobs_generated\": \"true\", \"_aml_system_cancellation_requested\": \"false\", \"cancellation_requested\": \"false\", \"_aml_system_progress_metadata_evaluation_timestamp\": \"\\\"2021-02-03T19:59:54.852201\\\"\", \"progress_metadata_evaluation_timestamp\": \"\\\"2021-02-03T19:59:54.852201\\\"\", \"_aml_system_progress_metadata_digest\": \"\\\"6ed90cb4679399a3a4d4b6eabeca29bfc3647bc007351a8a13cb11eee0d2289c\\\"\", \"progress_metadata_digest\": \"\\\"6ed90cb4679399a3a4d4b6eabeca29bfc3647bc007351a8a13cb11eee0d2289c\\\"\", \"_aml_system_progress_metadata_active_timestamp\": \"\\\"2021-02-03T19:59:54.852201\\\"\", \"progress_metadata_active_timestamp\": \"\\\"2021-02-03T19:59:54.852201\\\"\", \"_aml_system_HD_debd4c29-658d-4280-b761-2308b5eff7e4_0\": \"{\\\"--C\\\": 50, \\\"--max_iter\\\": 300}\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_0\": \"{\\\"--C\\\": 50, \\\"--max_iter\\\": 300}\", \"_aml_system_HD_debd4c29-658d-4280-b761-2308b5eff7e4_1\": \"{\\\"--C\\\": 1, \\\"--max_iter\\\": 300}\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_1\": \"{\\\"--C\\\": 1, \\\"--max_iter\\\": 300}\", \"_aml_system_HD_debd4c29-658d-4280-b761-2308b5eff7e4_2\": \"{\\\"--C\\\": 1, \\\"--max_iter\\\": 200}\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_2\": \"{\\\"--C\\\": 1, \\\"--max_iter\\\": 200}\", \"_aml_system_HD_debd4c29-658d-4280-b761-2308b5eff7e4_3\": \"{\\\"--C\\\": 0.1, \\\"--max_iter\\\": 50}\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_3\": \"{\\\"--C\\\": 0.1, \\\"--max_iter\\\": 50}\", \"_aml_system_environment_preparation_status\": \"PREPARED\", \"environment_preparation_status\": \"PREPARED\", \"_aml_system_prepare_run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_preparation\", \"prepare_run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_preparation\", \"_aml_system_HD_debd4c29-658d-4280-b761-2308b5eff7e4_4\": \"{\\\"--C\\\": 0.1, \\\"--max_iter\\\": 100}\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_4\": \"{\\\"--C\\\": 0.1, \\\"--max_iter\\\": 100}\", \"_aml_system_HD_debd4c29-658d-4280-b761-2308b5eff7e4_5\": \"{\\\"--C\\\": 500, \\\"--max_iter\\\": 50}\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_5\": \"{\\\"--C\\\": 500, \\\"--max_iter\\\": 50}\", \"_aml_system_HD_debd4c29-658d-4280-b761-2308b5eff7e4_6\": \"{\\\"--C\\\": 1000, \\\"--max_iter\\\": 50}\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_6\": \"{\\\"--C\\\": 1000, \\\"--max_iter\\\": 50}\", \"_aml_system_HD_debd4c29-658d-4280-b761-2308b5eff7e4_7\": \"{\\\"--C\\\": 0.1, \\\"--max_iter\\\": 200}\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_7\": \"{\\\"--C\\\": 0.1, \\\"--max_iter\\\": 200}\", \"_aml_system_HD_debd4c29-658d-4280-b761-2308b5eff7e4_8\": \"{\\\"--C\\\": 0.001, \\\"--max_iter\\\": 200}\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_8\": \"{\\\"--C\\\": 0.001, \\\"--max_iter\\\": 200}\", \"_aml_system_HD_debd4c29-658d-4280-b761-2308b5eff7e4_9\": \"{\\\"--C\\\": 50, \\\"--max_iter\\\": 50}\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_9\": \"{\\\"--C\\\": 50, \\\"--max_iter\\\": 50}\", \"_aml_system_HD_debd4c29-658d-4280-b761-2308b5eff7e4_10\": \"{\\\"--C\\\": 100, \\\"--max_iter\\\": 50}\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_10\": \"{\\\"--C\\\": 100, \\\"--max_iter\\\": 50}\", \"_aml_system_HD_debd4c29-658d-4280-b761-2308b5eff7e4_11\": \"{\\\"--C\\\": 1, \\\"--max_iter\\\": 100}\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_11\": \"{\\\"--C\\\": 1, \\\"--max_iter\\\": 100}\", \"_aml_system_HD_debd4c29-658d-4280-b761-2308b5eff7e4_12\": \"{\\\"--C\\\": 0.01, \\\"--max_iter\\\": 100}\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_12\": \"{\\\"--C\\\": 0.01, \\\"--max_iter\\\": 100}\", \"_aml_system_HD_debd4c29-658d-4280-b761-2308b5eff7e4_13\": \"{\\\"--C\\\": 50, \\\"--max_iter\\\": 100}\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_13\": \"{\\\"--C\\\": 50, \\\"--max_iter\\\": 100}\", \"_aml_system_HD_debd4c29-658d-4280-b761-2308b5eff7e4_14\": \"{\\\"--C\\\": 0.01, \\\"--max_iter\\\": 300}\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_14\": \"{\\\"--C\\\": 0.01, \\\"--max_iter\\\": 300}\", \"_aml_system_HD_debd4c29-658d-4280-b761-2308b5eff7e4_15\": \"{\\\"--C\\\": 20, \\\"--max_iter\\\": 100}\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_15\": \"{\\\"--C\\\": 20, \\\"--max_iter\\\": 100}\", \"_aml_system_final_best_metric_update_retry_count\": \"3\", \"final_best_metric_update_retry_count\": \"3\"}, \"end_time_utc\": \"2021-02-03T20:12:14.029541Z\", \"status\": \"Completed\", \"log_files\": {\"azureml-logs/hyperdrive.txt\": \"https://mlstrg137128.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_debd4c29-658d-4280-b761-2308b5eff7e4/azureml-logs/hyperdrive.txt?sv=2019-02-02&sr=b&sig=WDuPEgQFt4yDDz2DR%2FJQeLAK%2FUUo5du96AEwvDyewTo%3D&st=2021-02-03T20%3A02%3A23Z&se=2021-02-04T04%3A12%3A23Z&sp=r\"}, \"log_groups\": [[\"azureml-logs/hyperdrive.txt\"]], \"run_duration\": \"0:12:20\", \"hyper_parameters\": {\"--C\": [\"choice\", [[0.001, 0.01, 0.1, 1, 10, 20, 50, 100, 200, 500, 1000]]], \"--max_iter\": [\"choice\", [[50, 100, 200, 300]]]}}, \"child_runs\": [{\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_3\", \"run_number\": 7, \"metric\": 0.81666667, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-02-03T20:05:13.949451Z\", \"end_time\": \"2021-02-03T20:06:30.948068Z\", \"created_time\": \"2021-02-03T20:05:02.923713Z\", \"created_time_dt\": \"2021-02-03T20:05:02.923713Z\", \"duration\": \"0:01:28\", \"hyperdrive_id\": \"debd4c29-658d-4280-b761-2308b5eff7e4\", \"arguments\": null, \"param_--C\": 0.1, \"param_--max_iter\": 50, \"best_metric\": 0.81666667}, {\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_0\", \"run_number\": 8, \"metric\": 0.81666667, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-02-03T20:05:15.327961Z\", \"end_time\": \"2021-02-03T20:06:30.719386Z\", \"created_time\": \"2021-02-03T20:05:03.372219Z\", \"created_time_dt\": \"2021-02-03T20:05:03.372219Z\", \"duration\": \"0:01:27\", \"hyperdrive_id\": \"debd4c29-658d-4280-b761-2308b5eff7e4\", \"arguments\": null, \"param_--C\": 50, \"param_--max_iter\": 300, \"best_metric\": 0.81666667}, {\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_2\", \"run_number\": 9, \"metric\": 0.83333333, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-02-03T20:05:15.024357Z\", \"end_time\": \"2021-02-03T20:06:29.593806Z\", \"created_time\": \"2021-02-03T20:05:03.601305Z\", \"created_time_dt\": \"2021-02-03T20:05:03.601305Z\", \"duration\": \"0:01:25\", \"hyperdrive_id\": \"debd4c29-658d-4280-b761-2308b5eff7e4\", \"arguments\": null, \"param_--C\": 1, \"param_--max_iter\": 200, \"best_metric\": 0.83333333}, {\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_1\", \"run_number\": 10, \"metric\": 0.83333333, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-02-03T20:05:17.623936Z\", \"end_time\": \"2021-02-03T20:06:28.806696Z\", \"created_time\": \"2021-02-03T20:05:06.917581Z\", \"created_time_dt\": \"2021-02-03T20:05:06.917581Z\", \"duration\": \"0:01:21\", \"hyperdrive_id\": \"debd4c29-658d-4280-b761-2308b5eff7e4\", \"arguments\": null, \"param_--C\": 1, \"param_--max_iter\": 300, \"best_metric\": 0.83333333}, {\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_4\", \"run_number\": 11, \"metric\": 0.81666667, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-02-03T20:06:49.957175Z\", \"end_time\": \"2021-02-03T20:07:35.982734Z\", \"created_time\": \"2021-02-03T20:06:39.269899Z\", \"created_time_dt\": \"2021-02-03T20:06:39.269899Z\", \"duration\": \"0:00:56\", \"hyperdrive_id\": \"debd4c29-658d-4280-b761-2308b5eff7e4\", \"arguments\": null, \"param_--C\": 0.1, \"param_--max_iter\": 100, \"best_metric\": 0.83333333}, {\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_7\", \"run_number\": 12, \"metric\": 0.81666667, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-02-03T20:06:50.474606Z\", \"end_time\": \"2021-02-03T20:07:19.612751Z\", \"created_time\": \"2021-02-03T20:06:39.867623Z\", \"created_time_dt\": \"2021-02-03T20:06:39.867623Z\", \"duration\": \"0:00:39\", \"hyperdrive_id\": \"debd4c29-658d-4280-b761-2308b5eff7e4\", \"arguments\": null, \"param_--C\": 0.1, \"param_--max_iter\": 200, \"best_metric\": 0.83333333}, {\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_6\", \"run_number\": 13, \"metric\": 0.81666667, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-02-03T20:06:50.525035Z\", \"end_time\": \"2021-02-03T20:07:22.780006Z\", \"created_time\": \"2021-02-03T20:06:39.904594Z\", \"created_time_dt\": \"2021-02-03T20:06:39.904594Z\", \"duration\": \"0:00:42\", \"hyperdrive_id\": \"debd4c29-658d-4280-b761-2308b5eff7e4\", \"arguments\": null, \"param_--C\": 1000, \"param_--max_iter\": 50, \"best_metric\": 0.83333333}, {\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_5\", \"run_number\": 14, \"metric\": 0.83333333, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-02-03T20:06:52.652197Z\", \"end_time\": \"2021-02-03T20:07:25.087085Z\", \"created_time\": \"2021-02-03T20:06:40.650068Z\", \"created_time_dt\": \"2021-02-03T20:06:40.650068Z\", \"duration\": \"0:00:44\", \"hyperdrive_id\": \"debd4c29-658d-4280-b761-2308b5eff7e4\", \"arguments\": null, \"param_--C\": 500, \"param_--max_iter\": 50, \"best_metric\": 0.83333333}, {\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_11\", \"run_number\": 17, \"metric\": 0.83333333, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-02-03T20:07:58.984236Z\", \"end_time\": \"2021-02-03T20:08:32.607324Z\", \"created_time\": \"2021-02-03T20:07:42.974017Z\", \"created_time_dt\": \"2021-02-03T20:07:42.974017Z\", \"duration\": \"0:00:49\", \"hyperdrive_id\": \"debd4c29-658d-4280-b761-2308b5eff7e4\", \"arguments\": null, \"param_--C\": 1, \"param_--max_iter\": 100, \"best_metric\": 0.83333333}, {\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_9\", \"run_number\": 18, \"metric\": 0.81666667, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-02-03T20:07:57.433414Z\", \"end_time\": \"2021-02-03T20:08:35.865964Z\", \"created_time\": \"2021-02-03T20:07:43.190954Z\", \"created_time_dt\": \"2021-02-03T20:07:43.190954Z\", \"duration\": \"0:00:52\", \"hyperdrive_id\": \"debd4c29-658d-4280-b761-2308b5eff7e4\", \"arguments\": null, \"param_--C\": 50, \"param_--max_iter\": 50, \"best_metric\": 0.83333333}, {\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_8\", \"run_number\": 19, \"metric\": 0.81666667, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-02-03T20:07:58.645517Z\", \"end_time\": \"2021-02-03T20:08:30.785695Z\", \"created_time\": \"2021-02-03T20:07:44.998873Z\", \"created_time_dt\": \"2021-02-03T20:07:44.998873Z\", \"duration\": \"0:00:45\", \"hyperdrive_id\": \"debd4c29-658d-4280-b761-2308b5eff7e4\", \"arguments\": null, \"param_--C\": 0.001, \"param_--max_iter\": 200, \"best_metric\": 0.83333333}, {\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_10\", \"run_number\": 20, \"metric\": 0.83333333, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-02-03T20:08:00.712284Z\", \"end_time\": \"2021-02-03T20:08:39.037105Z\", \"created_time\": \"2021-02-03T20:07:49.056879Z\", \"created_time_dt\": \"2021-02-03T20:07:49.056879Z\", \"duration\": \"0:00:49\", \"hyperdrive_id\": \"debd4c29-658d-4280-b761-2308b5eff7e4\", \"arguments\": null, \"param_--C\": 100, \"param_--max_iter\": 50, \"best_metric\": 0.83333333}, {\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_12\", \"run_number\": 25, \"metric\": 0.83333333, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-02-03T20:09:00.934572Z\", \"end_time\": \"2021-02-03T20:09:32.039158Z\", \"created_time\": \"2021-02-03T20:08:51.699865Z\", \"created_time_dt\": \"2021-02-03T20:08:51.699865Z\", \"duration\": \"0:00:40\", \"hyperdrive_id\": \"debd4c29-658d-4280-b761-2308b5eff7e4\", \"arguments\": null, \"param_--C\": 0.01, \"param_--max_iter\": 100, \"best_metric\": 0.83333333}, {\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_13\", \"run_number\": 26, \"metric\": 0.81666667, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-02-03T20:09:01.411533Z\", \"end_time\": \"2021-02-03T20:09:35.142638Z\", \"created_time\": \"2021-02-03T20:08:51.733245Z\", \"created_time_dt\": \"2021-02-03T20:08:51.733245Z\", \"duration\": \"0:00:43\", \"hyperdrive_id\": \"debd4c29-658d-4280-b761-2308b5eff7e4\", \"arguments\": null, \"param_--C\": 50, \"param_--max_iter\": 100, \"best_metric\": 0.83333333}, {\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_14\", \"run_number\": 27, \"metric\": 0.83333333, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-02-03T20:09:03.299474Z\", \"end_time\": \"2021-02-03T20:09:35.393167Z\", \"created_time\": \"2021-02-03T20:08:52.590184Z\", \"created_time_dt\": \"2021-02-03T20:08:52.590184Z\", \"duration\": \"0:00:42\", \"hyperdrive_id\": \"debd4c29-658d-4280-b761-2308b5eff7e4\", \"arguments\": null, \"param_--C\": 0.01, \"param_--max_iter\": 300, \"best_metric\": 0.83333333}, {\"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_15\", \"run_number\": 28, \"metric\": 0.81666667, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-02-03T20:09:37.649545Z\", \"end_time\": \"2021-02-03T20:10:12.818864Z\", \"created_time\": \"2021-02-03T20:09:24.360509Z\", \"created_time_dt\": \"2021-02-03T20:09:24.360509Z\", \"duration\": \"0:00:48\", \"hyperdrive_id\": \"debd4c29-658d-4280-b761-2308b5eff7e4\", \"arguments\": null, \"param_--C\": 20, \"param_--max_iter\": 100, \"best_metric\": 0.83333333}], \"children_metrics\": {\"categories\": [0], \"series\": {\"Regularization Strength:\": [{\"categories\": [7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 25, 26, 27, 28], \"mode\": \"markers\", \"name\": \"Regularization Strength:\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.1, 50.0, 1.0, 1.0, 0.1, 0.1, 1000.0, 500.0, 1.0, 50.0, 0.001, 100.0, 0.01, 50.0, 0.01, 20.0]}, {\"categories\": [7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 25, 26, 27, 28], \"mode\": \"lines\", \"name\": \"Regularization Strength:_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.1, 50.0, 50.0, 50.0, 50.0, 50.0, 1000.0, 1000.0, 1000.0, 1000.0, 1000.0, 1000.0, 1000.0, 1000.0, 1000.0, 1000.0]}], \"Max iterations:\": [{\"categories\": [7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 25, 26, 27, 28], \"mode\": \"markers\", \"name\": \"Max iterations:\", \"stepped\": false, \"type\": \"scatter\", \"data\": [50, 300, 200, 300, 100, 200, 50, 50, 100, 50, 200, 50, 100, 100, 300, 100]}, {\"categories\": [7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 25, 26, 27, 28], \"mode\": \"lines\", \"name\": \"Max iterations:_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [50, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300]}], \"Accuracy\": [{\"categories\": [7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 25, 26, 27, 28], \"mode\": \"markers\", \"name\": \"Accuracy\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.8166666666666667, 0.8166666666666667, 0.8333333333333334, 0.8333333333333334, 0.8166666666666667, 0.8166666666666667, 0.8166666666666667, 0.8333333333333334, 0.8333333333333334, 0.8166666666666667, 0.8166666666666667, 0.8333333333333334, 0.8333333333333334, 0.8166666666666667, 0.8333333333333334, 0.8166666666666667]}, {\"categories\": [7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 25, 26, 27, 28], \"mode\": \"lines\", \"name\": \"Accuracy_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.8166666666666667, 0.8166666666666667, 0.8333333333333334, 0.8333333333333334, 0.8333333333333334, 0.8333333333333334, 0.8333333333333334, 0.8333333333333334, 0.8333333333333334, 0.8333333333333334, 0.8333333333333334, 0.8333333333333334, 0.8333333333333334, 0.8333333333333334, 0.8333333333333334, 0.8333333333333334]}]}, \"metricName\": null, \"primaryMetricName\": \"Accuracy\", \"showLegend\": false}, \"run_metrics\": [{\"name\": \"best_child_by_primary_metric\", \"run_id\": \"HD_debd4c29-658d-4280-b761-2308b5eff7e4\", \"categories\": [0], \"series\": [{\"data\": [{\"metric_name\": [\"Accuracy\", \"Accuracy\"], \"timestamp\": [\"2021-02-03 20:07:00.873077+00:00\", \"2021-02-03 20:07:00.873077+00:00\"], \"run_id\": [\"HD_debd4c29-658d-4280-b761-2308b5eff7e4_1\", \"HD_debd4c29-658d-4280-b761-2308b5eff7e4_1\"], \"metric_value\": [0.8333333333333334, 0.8333333333333334], \"final\": [false, true]}]}]}], \"run_logs\": \"[2021-02-03T19:59:55.640552][GENERATOR][INFO]Successfully sampled '4' jobs, they will soon be submitted to the execution target.\\r\\n[2021-02-03T19:59:55.9400848Z][SCHEDULER][INFO]The execution environment is being prepared. Please be patient as it can take a few minutes.\\r\\n[2021-02-03T19:59:55.285984][GENERATOR][INFO]Trying to sample '4' jobs from the hyperparameter space\\r\\n[2021-02-03T19:59:54.309673][API][INFO]Experiment created\\r\\n[2021-02-03T20:05:01.8492085Z][SCHEDULER][INFO]Scheduling job, id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_0'\\r\\n[2021-02-03T20:05:01.8486437Z][SCHEDULER][INFO]The execution environment was successfully prepared.\\r\\n[2021-02-03T20:05:01.8722156Z][SCHEDULER][INFO]Scheduling job, id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_3'\\r\\n[2021-02-03T20:05:01.9044415Z][SCHEDULER][INFO]Scheduling job, id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_2'\\r\\n[2021-02-03T20:05:03.0202232Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_3'\\r\\n[2021-02-03T20:05:03.8216458Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_2'\\r\\n[2021-02-03T20:05:03.5862138Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_0'\\r\\n[2021-02-03T20:05:01.9091185Z][SCHEDULER][INFO]Scheduling job, id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_1'\\r\\n[2021-02-03T20:05:07.1399738Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_1'\\r\\n[2021-02-03T20:05:29.626144][ENFORCER][INFO]Jobs [https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_0, https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_1, https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_2, https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_3] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-02-03T20:06:00.272753][ENFORCER][INFO]Jobs [https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_0, https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_1, https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_2, https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_3] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-02-03T20:06:29.912972][ENFORCER][INFO]Jobs [https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_0, https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_3] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-02-03T20:06:36.556836][GENERATOR][INFO]Trying to sample '4' jobs from the hyperparameter space\\r\\n[2021-02-03T20:06:36.698683][GENERATOR][INFO]Successfully sampled '4' jobs, they will soon be submitted to the execution target.\\r\\n[2021-02-03T20:06:38.8320651Z][SCHEDULER][INFO]Scheduling job, id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_6'\\r\\n[2021-02-03T20:06:38.8185986Z][SCHEDULER][INFO]Scheduling job, id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_7'\\r\\n[2021-02-03T20:06:38.8167034Z][SCHEDULER][INFO]Scheduling job, id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_5'\\r\\n[2021-02-03T20:06:38.8038026Z][SCHEDULER][INFO]Scheduling job, id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_4'\\r\\n[2021-02-03T20:06:39.3526951Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_4'\\r\\n[2021-02-03T20:06:40.0004595Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_7'\\r\\n[2021-02-03T20:06:40.1265288Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_6'\\r\\n[2021-02-03T20:06:40.8615494Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_5'\\r\\n[2021-02-03T20:07:00.187483][ENFORCER][INFO]Jobs [https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_4, https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_5, https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_6, https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_7] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-02-03T20:07:36.760883][GENERATOR][INFO]Trying to sample '4' jobs from the hyperparameter space\\r\\n[2021-02-03T20:07:36.909211][GENERATOR][INFO]Successfully sampled '4' jobs, they will soon be submitted to the execution target.\\r\\n[2021-02-03T20:07:41.9656897Z][SCHEDULER][INFO]Scheduling job, id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_8'\\r\\n[2021-02-03T20:07:41.9693894Z][SCHEDULER][INFO]Scheduling job, id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_11'\\r\\n[2021-02-03T20:07:41.9671891Z][SCHEDULER][INFO]Scheduling job, id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_9'\\r\\n[2021-02-03T20:07:41.9686257Z][SCHEDULER][INFO]Scheduling job, id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_10'\\r\\n[2021-02-03T20:07:43.1632431Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_11'\\r\\n[2021-02-03T20:07:43.4363905Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_9'\\r\\n[2021-02-03T20:07:45.0778112Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_8'\\r\\n[2021-02-03T20:07:49.2468302Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_10'\\r\\n[2021-02-03T20:08:01.297351][ENFORCER][INFO]Jobs [https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_8, https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_9, https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_10, https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_11] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-02-03T20:08:35.023515][ENFORCER][INFO]Jobs [https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_9, https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_10] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-02-03T20:08:37.044379][GENERATOR][INFO]Trying to sample '3' jobs from the hyperparameter space\\r\\n[2021-02-03T20:08:37.311989][GENERATOR][INFO]Successfully sampled '3' jobs, they will soon be submitted to the execution target.\\r\\n[2021-02-03T20:08:50.9809376Z][SCHEDULER][INFO]Scheduling job, id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_12'\\r\\n[2021-02-03T20:08:50.9837683Z][SCHEDULER][INFO]Scheduling job, id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_14'\\r\\n[2021-02-03T20:08:50.9825205Z][SCHEDULER][INFO]Scheduling job, id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_13'\\r\\n[2021-02-03T20:08:51.7948058Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_12'\\r\\n[2021-02-03T20:08:51.8554776Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_13'\\r\\n[2021-02-03T20:08:52.9788758Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_14'\\r\\n[2021-02-03T20:09:05.154462][ENFORCER][INFO]Jobs [https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_12, https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_13, https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_14] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-02-03T20:09:08.408506][GENERATOR][INFO]Trying to sample '1' jobs from the hyperparameter space\\r\\n[2021-02-03T20:09:08.684301][GENERATOR][INFO]Successfully sampled '1' jobs, they will soon be submitted to the execution target.\\r\\n[2021-02-03T20:09:23.7815430Z][SCHEDULER][INFO]Scheduling job, id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_15'\\r\\n[2021-02-03T20:09:24.4685892Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_debd4c29-658d-4280-b761-2308b5eff7e4_15'\\r\\n[2021-02-03T20:09:38.978133][GENERATOR][INFO]Max number of jobs '16' reached for experiment.\\r\\n[2021-02-03T20:09:39.108419][GENERATOR][INFO]All jobs generated.\\r\\n[2021-02-03T20:10:06.397447][ENFORCER][INFO]Jobs [https://southcentralus.api.azureml.ms/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourceGroups/aml-quickstarts-137128/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-137128/experiments/**SCRUBBED**/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4_15] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-02-03T20:12:14.207727][CONTROLLER][INFO]Experiment was 'ExperimentStatus.RUNNING', is 'ExperimentStatus.FINISHED'.\\n\\nRun is completed.\", \"graph\": {}, \"widget_settings\": {\"childWidgetDisplay\": \"popup\", \"send_telemetry\": false, \"log_level\": \"INFO\", \"sdk_version\": \"1.20.0\"}, \"loading\": false}"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "RunId: HD_debd4c29-658d-4280-b761-2308b5eff7e4\n",
            "Web View: https://ml.azure.com/experiments/heart-failure-prediction/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4?wsid=/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourcegroups/aml-quickstarts-137128/workspaces/quick-starts-ws-137128\n",
            "\n",
            "Streaming azureml-logs/hyperdrive.txt\n",
            "=====================================\n",
            "\n",
            "\"<START>[2021-02-03T19:59:55.640552][GENERATOR][INFO]Successfully sampled '4' jobs, they will soon be submitted to the execution target.<END>\\n\"<START>[2021-02-03T19:59:55.9400848Z][SCHEDULER][INFO]The execution environment is being prepared. Please be patient as it can take a few minutes.<END>\"<START>[2021-02-03T19:59:55.285984][GENERATOR][INFO]Trying to sample '4' jobs from the hyperparameter space<END>\\n\"\"<START>[2021-02-03T19:59:54.309673][API][INFO]Experiment created<END>\\n\"\n",
            "\n",
            "Execution Summary\n",
            "=================\n",
            "RunId: HD_debd4c29-658d-4280-b761-2308b5eff7e4\n",
            "Web View: https://ml.azure.com/experiments/heart-failure-prediction/runs/HD_debd4c29-658d-4280-b761-2308b5eff7e4?wsid=/subscriptions/610d6e37-4747-4a20-80eb-3aad70a55f43/resourcegroups/aml-quickstarts-137128/workspaces/quick-starts-ws-137128\n",
            "\n"
          ]
        }
      ],
      "execution_count": 6,
      "metadata": {
        "gather": {
          "logged": 1612383177197
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# get_children_sorted_by_primary_metric: Returns a list of children sorted by their best primary metric.\n",
        "# Each child in the result has run id, hyperparameters, best primary metric value and status.\n",
        "\n",
        "print(hyperdrive_run.get_children_sorted_by_primary_metric(top=0, reverse=False, discard_no_metric=False))\n",
        "print('===================================================')\n",
        "\n",
        "# get_hyperparameters\n",
        "# Return the hyperparameters for all the child runs that were launched by this HyperDriveRun.\n",
        "\n",
        "print(hyperdrive_run.get_hyperparameters())\n",
        "print('===================================================')\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_14', 'hyperparameters': '{\"--C\": 0.01, \"--max_iter\": 300}', 'best_primary_metric': 0.8333333333333334, 'status': 'Completed'}, {'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_12', 'hyperparameters': '{\"--C\": 0.01, \"--max_iter\": 100}', 'best_primary_metric': 0.8333333333333334, 'status': 'Completed'}, {'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_10', 'hyperparameters': '{\"--C\": 100, \"--max_iter\": 50}', 'best_primary_metric': 0.8333333333333334, 'status': 'Completed'}, {'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_11', 'hyperparameters': '{\"--C\": 1, \"--max_iter\": 100}', 'best_primary_metric': 0.8333333333333334, 'status': 'Completed'}, {'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_5', 'hyperparameters': '{\"--C\": 500, \"--max_iter\": 50}', 'best_primary_metric': 0.8333333333333334, 'status': 'Completed'}, {'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_1', 'hyperparameters': '{\"--C\": 1, \"--max_iter\": 300}', 'best_primary_metric': 0.8333333333333334, 'status': 'Completed'}, {'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_2', 'hyperparameters': '{\"--C\": 1, \"--max_iter\": 200}', 'best_primary_metric': 0.8333333333333334, 'status': 'Completed'}, {'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_15', 'hyperparameters': '{\"--C\": 20, \"--max_iter\": 100}', 'best_primary_metric': 0.8166666666666667, 'status': 'Completed'}, {'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_13', 'hyperparameters': '{\"--C\": 50, \"--max_iter\": 100}', 'best_primary_metric': 0.8166666666666667, 'status': 'Completed'}, {'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_8', 'hyperparameters': '{\"--C\": 0.001, \"--max_iter\": 200}', 'best_primary_metric': 0.8166666666666667, 'status': 'Completed'}, {'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_9', 'hyperparameters': '{\"--C\": 50, \"--max_iter\": 50}', 'best_primary_metric': 0.8166666666666667, 'status': 'Completed'}, {'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_6', 'hyperparameters': '{\"--C\": 1000, \"--max_iter\": 50}', 'best_primary_metric': 0.8166666666666667, 'status': 'Completed'}, {'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_7', 'hyperparameters': '{\"--C\": 0.1, \"--max_iter\": 200}', 'best_primary_metric': 0.8166666666666667, 'status': 'Completed'}, {'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_4', 'hyperparameters': '{\"--C\": 0.1, \"--max_iter\": 100}', 'best_primary_metric': 0.8166666666666667, 'status': 'Completed'}, {'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_0', 'hyperparameters': '{\"--C\": 50, \"--max_iter\": 300}', 'best_primary_metric': 0.8166666666666667, 'status': 'Completed'}, {'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_3', 'hyperparameters': '{\"--C\": 0.1, \"--max_iter\": 50}', 'best_primary_metric': 0.8166666666666667, 'status': 'Completed'}, {'run_id': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_preparation', 'hyperparameters': None, 'best_primary_metric': None, 'status': 'Completed'}]\n",
            "===================================================\n",
            "{'HD_debd4c29-658d-4280-b761-2308b5eff7e4_0': '{\"--C\": 50, \"--max_iter\": 300}', 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_1': '{\"--C\": 1, \"--max_iter\": 300}', 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_2': '{\"--C\": 1, \"--max_iter\": 200}', 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_3': '{\"--C\": 0.1, \"--max_iter\": 50}', 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_4': '{\"--C\": 0.1, \"--max_iter\": 100}', 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_5': '{\"--C\": 500, \"--max_iter\": 50}', 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_6': '{\"--C\": 1000, \"--max_iter\": 50}', 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_7': '{\"--C\": 0.1, \"--max_iter\": 200}', 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_8': '{\"--C\": 0.001, \"--max_iter\": 200}', 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_9': '{\"--C\": 50, \"--max_iter\": 50}', 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_10': '{\"--C\": 100, \"--max_iter\": 50}', 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_11': '{\"--C\": 1, \"--max_iter\": 100}', 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_12': '{\"--C\": 0.01, \"--max_iter\": 100}', 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_13': '{\"--C\": 50, \"--max_iter\": 100}', 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_14': '{\"--C\": 0.01, \"--max_iter\": 300}', 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_15': '{\"--C\": 20, \"--max_iter\": 100}'}\n",
            "===================================================\n"
          ]
        }
      ],
      "execution_count": 8,
      "metadata": {
        "gather": {
          "logged": 1612383227816
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Best Model\n",
        "\n",
        "In the cell below, I get the best model from the HyperDrive experiment and display all the properties of the model.\n"
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "# Get the best run and save the model from that run.\n",
        "\n",
        "# get_best_run_by_primary_metric()\n",
        "# Returns the best Run, or None if no child has the primary metric.\n",
        "best_run = hyperdrive_run.get_best_run_by_primary_metric()\n",
        "\n",
        "# get_metrics()\n",
        "# Returns the metrics from all the runs that were launched by this HyperDriveRun.\n",
        "print(\"Best run metrics :\",best_run.get_metrics())\n",
        "print('===================================================')\n",
        "\n",
        "# get_details()\n",
        "# Returns a dictionary with the details for the run\n",
        "print(\"Best run details :\",best_run.get_details())\n",
        "print('===================================================')\n",
        "\n",
        "# get_file_names()\n",
        "# Returns a list of the files that are stored in association with the run.\n",
        "\n",
        "print(\"Best run file names :\",best_run.get_file_names())\n",
        "print('===================================================')\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best run metrics : {'Regularization Strength:': 1.0, 'Max iterations:': 300, 'Accuracy': 0.8333333333333334}\n",
            "===================================================\n",
            "Best run details : {'runId': 'HD_debd4c29-658d-4280-b761-2308b5eff7e4_1', 'target': 'compute-cluster', 'status': 'Completed', 'startTimeUtc': '2021-02-03T20:05:17.623936Z', 'endTimeUtc': '2021-02-03T20:06:28.806696Z', 'properties': {'_azureml.ComputeTargetType': 'amlcompute', 'ContentSnapshotId': 'b0fa1234-4c3e-41c9-bf41-eac3d4dc177f', 'ProcessInfoFile': 'azureml-logs/process_info.json', 'ProcessStatusFile': 'azureml-logs/process_status.json'}, 'inputDatasets': [], 'outputDatasets': [], 'runDefinition': {'script': 'train.py', 'command': '', 'useAbsolutePath': False, 'arguments': ['--C', '1', '--max_iter', '300'], 'sourceDirectoryDataStore': None, 'framework': 'Python', 'communicator': 'None', 'target': 'compute-cluster', 'dataReferences': {}, 'data': {}, 'outputData': {}, 'jobName': None, 'maxRunDurationSeconds': None, 'nodeCount': 1, 'priority': None, 'credentialPassthrough': False, 'environment': {'name': 'Experiment heart-failure-prediction Environment', 'version': 'Autosave_2021-02-03T19:59:54Z_10902712', 'python': {'interpreterPath': 'python', 'userManagedDependencies': False, 'condaDependencies': {'channels': ['anaconda', 'conda-forge'], 'dependencies': ['python=3.6.2', {'pip': ['azureml-defaults', 'scikit-learn==0.20.3', 'scipy==1.2.1', 'joblib==0.13.2']}], 'name': 'azureml_ba9520bf386d662001eeb9523395794e'}, 'baseCondaEnvironment': None}, 'environmentVariables': {'EXAMPLE_ENV_VAR': 'EXAMPLE_VALUE'}, 'docker': {'baseImage': 'mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20200423.v1', 'platform': {'os': 'Linux', 'architecture': 'amd64'}, 'baseDockerfile': None, 'baseImageRegistry': {'address': None, 'username': None, 'password': None}, 'enabled': True, 'arguments': []}, 'spark': {'repositories': [], 'packages': [], 'precachePackages': False}, 'inferencingStackVersion': None}, 'history': {'outputCollection': True, 'directoriesToWatch': ['logs'], 'enableMLflowTracking': True, 'snapshotProject': True}, 'spark': {'configuration': {'spark.app.name': 'Azure ML Experiment', 'spark.yarn.maxAppAttempts': '1'}}, 'parallelTask': {'maxRetriesPerWorker': 0, 'workerCountPerNode': 1, 'terminalExitCodes': None, 'configuration': {}}, 'amlCompute': {'name': None, 'vmSize': 'STANDARD_DS3_V2', 'retainCluster': False, 'clusterMaxNodeCount': 1}, 'aiSuperComputer': {'instanceType': None, 'frameworkImage': None, 'imageVersion': None, 'location': None, 'aiSuperComputerStorageData': None, 'interactive': False, 'scalePolicy': None}, 'tensorflow': {'workerCount': 1, 'parameterServerCount': 1}, 'mpi': {'processCountPerNode': 1}, 'pyTorch': {'communicationBackend': None, 'processCount': None}, 'hdi': {'yarnDeployMode': 'Cluster'}, 'containerInstance': {'region': None, 'cpuCores': 2.0, 'memoryGb': 3.5}, 'exposedPorts': None, 'docker': {'useDocker': True, 'sharedVolumes': True, 'shmSize': '2g', 'arguments': []}, 'cmk8sCompute': {'configuration': {}}, 'commandReturnCodeConfig': {'returnCode': 'Zero', 'successfulReturnCodes': []}}, 'logFiles': {'azureml-logs/55_azureml-execution-tvmps_8eab53f119d2aefa01914d49e0bbc867c55899d9b74edd9e2d742a9b20c8c4bc_d.txt': 'https://mlstrg137128.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_debd4c29-658d-4280-b761-2308b5eff7e4_1/azureml-logs/55_azureml-execution-tvmps_8eab53f119d2aefa01914d49e0bbc867c55899d9b74edd9e2d742a9b20c8c4bc_d.txt?sv=2019-02-02&sr=b&sig=rVTp33U%2FWKNaDWxJIgUXr%2B6ZhOXvsuzrLKvFSChTxB8%3D&st=2021-02-03T20%3A04%3A12Z&se=2021-02-04T04%3A14%3A12Z&sp=r', 'azureml-logs/65_job_prep-tvmps_8eab53f119d2aefa01914d49e0bbc867c55899d9b74edd9e2d742a9b20c8c4bc_d.txt': 'https://mlstrg137128.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_debd4c29-658d-4280-b761-2308b5eff7e4_1/azureml-logs/65_job_prep-tvmps_8eab53f119d2aefa01914d49e0bbc867c55899d9b74edd9e2d742a9b20c8c4bc_d.txt?sv=2019-02-02&sr=b&sig=cqjeoBxhr9gegWLOktXaUGLRxsiNoDLBoOI9dwPpex4%3D&st=2021-02-03T20%3A04%3A12Z&se=2021-02-04T04%3A14%3A12Z&sp=r', 'azureml-logs/70_driver_log.txt': 'https://mlstrg137128.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_debd4c29-658d-4280-b761-2308b5eff7e4_1/azureml-logs/70_driver_log.txt?sv=2019-02-02&sr=b&sig=9l33ulqIGIkpbdfMMSlT9KFXIZedh%2B86yw%2BSRPD%2FV00%3D&st=2021-02-03T20%3A04%3A12Z&se=2021-02-04T04%3A14%3A12Z&sp=r', 'azureml-logs/75_job_post-tvmps_8eab53f119d2aefa01914d49e0bbc867c55899d9b74edd9e2d742a9b20c8c4bc_d.txt': 'https://mlstrg137128.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_debd4c29-658d-4280-b761-2308b5eff7e4_1/azureml-logs/75_job_post-tvmps_8eab53f119d2aefa01914d49e0bbc867c55899d9b74edd9e2d742a9b20c8c4bc_d.txt?sv=2019-02-02&sr=b&sig=XW%2FHRszyATj8nF5O8PyHcH94I5M3nhaVzVDDpVOFTHM%3D&st=2021-02-03T20%3A04%3A12Z&se=2021-02-04T04%3A14%3A12Z&sp=r', 'logs/azureml/99_azureml.log': 'https://mlstrg137128.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_debd4c29-658d-4280-b761-2308b5eff7e4_1/logs/azureml/99_azureml.log?sv=2019-02-02&sr=b&sig=YvLqCCviIgfIi%2BA%2Bb2D5a%2BuMGWsvYvjuAoyDIpiumI8%3D&st=2021-02-03T20%3A04%3A12Z&se=2021-02-04T04%3A14%3A12Z&sp=r', 'logs/azureml/job_prep_azureml.log': 'https://mlstrg137128.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_debd4c29-658d-4280-b761-2308b5eff7e4_1/logs/azureml/job_prep_azureml.log?sv=2019-02-02&sr=b&sig=I2lwx5AoaMwCJgaTHbhKa7jp1HuJ6ZzdPKyzGTJqFCQ%3D&st=2021-02-03T20%3A04%3A12Z&se=2021-02-04T04%3A14%3A12Z&sp=r', 'logs/azureml/job_release_azureml.log': 'https://mlstrg137128.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_debd4c29-658d-4280-b761-2308b5eff7e4_1/logs/azureml/job_release_azureml.log?sv=2019-02-02&sr=b&sig=wtzIPHLREQXkGV603jsde073%2FJSZrHn6700W%2FqBFGog%3D&st=2021-02-03T20%3A04%3A12Z&se=2021-02-04T04%3A14%3A12Z&sp=r'}, 'submittedBy': 'ODL_User 137128'}\n",
            "===================================================\n",
            "Best run file names : ['azureml-logs/55_azureml-execution-tvmps_8eab53f119d2aefa01914d49e0bbc867c55899d9b74edd9e2d742a9b20c8c4bc_d.txt', 'azureml-logs/65_job_prep-tvmps_8eab53f119d2aefa01914d49e0bbc867c55899d9b74edd9e2d742a9b20c8c4bc_d.txt', 'azureml-logs/70_driver_log.txt', 'azureml-logs/75_job_post-tvmps_8eab53f119d2aefa01914d49e0bbc867c55899d9b74edd9e2d742a9b20c8c4bc_d.txt', 'logs/azureml/99_azureml.log', 'logs/azureml/job_prep_azureml.log', 'logs/azureml/job_release_azureml.log', 'outputs/model.pkl']\n",
            "===================================================\n"
          ]
        }
      ],
      "execution_count": 9,
      "metadata": {
        "gather": {
          "logged": 1612383251034
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Register the model"
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "# Save the best model\n",
        "\n",
        "best_run.register_model(model_name = \"best_run_hyperdrive.pkl\", model_path = './outputs/')\n",
        "\n",
        "print(best_run)\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Run(Experiment: heart-failure-prediction,\n",
            "Id: HD_debd4c29-658d-4280-b761-2308b5eff7e4_1,\n",
            "Type: azureml.scriptrun,\n",
            "Status: Completed)\n"
          ]
        }
      ],
      "execution_count": 10,
      "metadata": {
        "gather": {
          "logged": 1612383261535
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Download the model file\r\n",
        "\r\n",
        "best_run.download_file('outputs/model.pkl', 'hyperdrive_model.pkl')"
      ],
      "outputs": [],
      "execution_count": 13,
      "metadata": {
        "collapsed": true,
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1612383565273
        }
      }
    }
  ],
  "metadata": {
    "kernel_info": {
      "name": "python3"
    },
    "kernelspec": {
      "name": "python3",
      "language": "python",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python",
      "version": "3.6.9",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "nteract": {
      "version": "nteract-front-end@1.0.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}