06_amls/Docetaxel_ML.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6d207b28",
   "metadata": {},
   "source": [
    "# Docetaxel Sensitivity: Machine Learning using Gene Expression\n",
    "---------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "71dffb44",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from azureml.core import Workspace, Dataset\n",
    "\n",
    "ws = Workspace.from_config()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84f23543",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Get Sensitivity Data\n",
    "doce_data = Dataset.get_by_name(ws, name='docetaxel_sensitivity')\n",
    "doce_data_df = doce_data.to_pandas_dataframe()\n",
    "doce_data_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "156b0d72",
   "metadata": {},
   "outputs": [],
   "source": [
    "doce_data_df.to_csv('docetaxel_sensitivity.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4000f7a9",
   "metadata": {},
   "source": [
    "## Create Training Script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36dfb939",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%writefile train.py\n",
    "## Import libraries\n",
    "import argparse\n",
    "import joblib\n",
    "import os\n",
    "from azureml.core import Dataset, Run\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import r2_score\n",
    "from xgboost import XGBRegressor\n",
    "\n",
    "## Set the input parameters\n",
    "parser = argparse.ArgumentParser()\n",
    "parser.add_argument(\"--n_estimators\", type=int, dest='n_estimators', help='Number of gradient boosted trees.')\n",
    "parser.add_argument(\"--max_depth\", type=float, dest='max_depth', help='Maximum tree depth for base learners.')\n",
    "parser.add_argument(\"--booster\", type=str, dest='booster', help='Boosting method.')\n",
    "args = parser.parse_args()\n",
    "\n",
    "n_estimators = args.n_estimators\n",
    "max_depth = int(args.max_depth)\n",
    "booster = args.booster\n",
    "\n",
    "## Get the experiment run context\n",
    "run = Run.get_context()\n",
    "ws = run.experiment.workspace\n",
    "\n",
    "## Log run options\n",
    "run.log('n_estimators', str(n_estimators))\n",
    "run.log('max_depth', str(max_depth))\n",
    "run.log('booster', str(booster))\n",
    "\n",
    "## load the dataset(s)\n",
    "print(\"Loading Data...\")\n",
    "# dataset = run.input_datasets['doce_data'].to_pandas_dataframe() # Get the training data from the estimator input\n",
    "dataset = pd.read_csv('docetaxel_sensitivity.csv')\n",
    "run.log(\"Input data shape: \", str(dataset.shape))\n",
    "\n",
    "\n",
    "## Separate features and labels\n",
    "X = dataset[dataset.columns[1:]]\n",
    "y = dataset[['L10_IC_50']].values\n",
    "\n",
    "## Get X as numpy array\n",
    "X = X.values\n",
    "\n",
    "## Scale the dataset\n",
    "scaler = StandardScaler()\n",
    "X = scaler.fit_transform(X)\n",
    "\n",
    "\n",
    "## Split data into training set and test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y,\n",
    "    test_size = 0.25, \n",
    "    random_state = 1337\n",
    ")\n",
    "\n",
    "## Initialize Algorithm\n",
    "xgb = XGBRegressor(objective ='reg:linear',\n",
    "                   n_estimators = n_estimators,\n",
    "                   max_depth = max_depth,\n",
    "                   booster = booster,\n",
    "                   seed = 1337)\n",
    "\n",
    "## Train Model w/ Cross-Validation (5-fold)\n",
    "scores = cross_validate(xgb,\n",
    "                        X_train, y_train,\n",
    "                        cv=5, scoring=('r2'),\n",
    "                        return_estimator=True)\n",
    "\n",
    "model = scores['estimator'][0]\n",
    "\n",
    "## Score data\n",
    "y_train_hat = model.predict(X_train)\n",
    "y_test_hat = model.predict(X_test)\n",
    "\n",
    "## Log model performance\n",
    "run.log('r2', r2_score(y_train, y_train_hat))\n",
    "run.log('r2_test', r2_score(y_test, y_test_hat))\n",
    "\n",
    "## Note: Files saved in the outputs folder is automatically uploaded into experiment record\n",
    "os.makedirs('outputs', exist_ok=True)\n",
    "\n",
    "## Save model\n",
    "joblib.dump(value=model, filename = f'outputs/model.pkl')\n",
    "\n",
    "run.complete()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "364f25b4",
   "metadata": {},
   "source": [
    "## Compute Target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36333fca",
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.core.compute import ComputeTarget, AmlCompute\n",
    "from azureml.core.compute_target import ComputeTargetException\n",
    "\n",
    "cluster_name = \"ml-cluster1\"\n",
    "\n",
    "try:\n",
    "    # Check for existing compute target\n",
    "    training_cluster = ComputeTarget(workspace=ws, name=cluster_name)\n",
    "    print('Found existing cluster.')\n",
    "except ComputeTargetException:\n",
    "    # If it doesn't already exist, create it\n",
    "    try:\n",
    "        compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_DS11_V2', max_nodes=4)\n",
    "        training_cluster = ComputeTarget.create(ws, cluster_name, compute_config)\n",
    "        training_cluster.wait_for_completion(show_output=True)\n",
    "    except Exception as ex:\n",
    "        print(ex)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90e47e8d",
   "metadata": {},
   "source": [
    "## Configure Hyperdrive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a484c449",
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.core import Experiment, ScriptRunConfig, Environment\n",
    "from azureml.core.conda_dependencies import CondaDependencies\n",
    "from azureml.train.hyperdrive import GridParameterSampling, RandomParameterSampling, HyperDriveConfig, PrimaryMetricGoal, choice, uniform\n",
    "\n",
    "\n",
    "# Create a Python environment for the experiment\n",
    "sklearn_env = Environment(\"sklearn-env\")\n",
    "\n",
    "# Ensure the required packages are installed (we need scikit-learn, Azure ML defaults, and Azure ML dataprep)\n",
    "packages = CondaDependencies.create(pip_packages=['scikit-learn','azureml-defaults','azureml-dataprep[pandas]', 'xgboost'])\n",
    "sklearn_env.python.conda_dependencies = packages\n",
    "\n",
    "## Create a script config\n",
    "script_config = ScriptRunConfig(source_directory = \".\",\n",
    "                                script='train.py',\n",
    "                                # arguments = ['--doce_data', doce_data.as_named_input('doce_data')], # Reference to dataset\n",
    "                                environment = sklearn_env,\n",
    "                                compute_target = training_cluster)\n",
    "\n",
    "## Sample a range of parameter values\n",
    "# params = GridParameterSampling(\n",
    "params = RandomParameterSampling(\n",
    "    {\n",
    "        '--n_estimators': choice(1, 5, 100),\n",
    "        '--max_depth': uniform(100, 5000),\n",
    "        '--booster': choice('gbtree', 'gblinear', 'dart')\n",
    "    }\n",
    ")\n",
    "\n",
    "## Configure hyperdrive settings\n",
    "hyperdrive = HyperDriveConfig(run_config = script_config, \n",
    "                              hyperparameter_sampling = params, \n",
    "                              policy = None, \n",
    "                              primary_metric_name = 'r2', \n",
    "                              primary_metric_goal = PrimaryMetricGoal.MAXIMIZE, \n",
    "                              max_total_runs = 10,\n",
    "                              max_concurrent_runs = 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a20c2d0",
   "metadata": {},
   "source": [
    "## Run the Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "babfe629",
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment = Experiment(workspace = ws, name = 'docetaxel_sensitivity_training_hyperdrive')\n",
    "run = experiment.submit(config = hyperdrive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "48697cba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "848bfb06df034d24b1eeb02bc7183189",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "_HyperDriveWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': False, 'log_level': 'INFO'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/aml.mini.widget.v1": "{\"status\": \"CancelRequested\", \"workbench_run_details_uri\": \"https://ml.azure.com/runs/HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23?wsid=/subscriptions/0bb59590-d012-407d-a545-7513aae8c4a7/resourcegroups/dsba6190class/workspaces/dsba6190ml&tid=88d59d7d-aecb-41b2-90c5-55595de02536\", \"run_id\": \"HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23\", \"run_properties\": {\"run_id\": \"HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23\", \"created_utc\": \"2022-04-03T01:26:40.594228Z\", \"properties\": {\"primary_metric_config\": \"{\\\"name\\\": \\\"r2\\\", \\\"goal\\\": \\\"maximize\\\"}\", \"resume_from\": \"null\", \"runTemplate\": \"HyperDrive\", \"azureml.runsource\": \"hyperdrive\", \"platform\": \"AML\", \"ContentSnapshotId\": \"adb39e4c-47cd-4db4-952c-a9ce57b00f91\", \"user_agent\": \"python/3.6.9 (Linux-5.4.0-1064-azure-x86_64-with-debian-buster-sid) msrest/0.6.21 Hyperdrive.Service/1.0.0 Hyperdrive.SDK/core.1.37.0\", \"space_size\": \"infinite_space_size\"}, \"tags\": {\"_aml_system_max_concurrent_jobs\": \"4\", \"_aml_system_max_total_jobs\": \"10\", \"_aml_system_max_duration_minutes\": \"10080\", \"_aml_system_policy_config\": \"{\\\"name\\\": \\\"DEFAULT\\\"}\", \"_aml_system_generator_config\": \"{\\\"name\\\": \\\"RANDOM\\\", \\\"parameter_space\\\": {\\\"--n_estimators\\\": [\\\"choice\\\", [[1, 5, 100]]], \\\"--max_depth\\\": [\\\"uniform\\\", [100, 5000]], \\\"--booster\\\": [\\\"choice\\\", [[\\\"gbtree\\\", \\\"gblinear\\\", \\\"dart\\\"]]]}}\", \"_aml_system_primary_metric_config\": \"{\\\"name\\\": \\\"r2\\\", \\\"goal\\\": \\\"maximize\\\"}\", \"_aml_system_platform_config\": \"{\\\"ServiceAddress\\\": \\\"https://eastus.experiments.azureml.net\\\", \\\"ServiceArmScope\\\": \\\"subscriptions/0bb59590-d012-407d-a545-7513aae8c4a7/resourceGroups/dsba6190class/providers/Microsoft.MachineLearningServices/workspaces/dsba6190ml/experiments/docetaxel_sensitivity_training_hyperdrive\\\", \\\"SubscriptionId\\\": \\\"0bb59590-d012-407d-a545-7513aae8c4a7\\\", \\\"ResourceGroupName\\\": \\\"dsba6190class\\\", \\\"WorkspaceName\\\": \\\"dsba6190ml\\\", \\\"ExperimentName\\\": \\\"docetaxel_sensitivity_training_hyperdrive\\\", \\\"Definition\\\": {\\\"Overrides\\\": {\\\"script\\\": \\\"train.py\\\", \\\"arguments\\\": [\\\"--doce_data\\\", \\\"DatasetConsumptionConfig:doce_data\\\"], \\\"target\\\": \\\"ml-cluster1\\\", \\\"framework\\\": \\\"Python\\\", \\\"communicator\\\": \\\"None\\\", \\\"maxRunDurationSeconds\\\": 2592000, \\\"nodeCount\\\": 1, \\\"priority\\\": null, \\\"environment\\\": {\\\"name\\\": \\\"sklearn-env\\\", \\\"version\\\": null, \\\"environmentVariables\\\": {\\\"EXAMPLE_ENV_VAR\\\": \\\"EXAMPLE_VALUE\\\"}, \\\"python\\\": {\\\"userManagedDependencies\\\": false, \\\"interpreterPath\\\": \\\"python\\\", \\\"condaDependenciesFile\\\": null, \\\"baseCondaEnvironment\\\": null, \\\"condaDependencies\\\": {\\\"name\\\": \\\"project_environment\\\", \\\"dependencies\\\": [\\\"python=3.6.2\\\", {\\\"pip\\\": [\\\"scikit-learn\\\", \\\"azureml-defaults~=1.37.0\\\", \\\"azureml-dataprep[pandas]\\\", \\\"xgboost\\\"]}], \\\"channels\\\": [\\\"anaconda\\\", \\\"conda-forge\\\"]}}, \\\"docker\\\": {\\\"enabled\\\": false, \\\"baseImage\\\": \\\"mcr.microsoft.com/azureml/openmpi3.1.2-ubuntu18.04:20211124.v1\\\", \\\"baseDockerfile\\\": null, \\\"sharedVolumes\\\": true, \\\"shmSize\\\": \\\"2g\\\", \\\"arguments\\\": [], \\\"baseImageRegistry\\\": {\\\"address\\\": null, \\\"username\\\": null, \\\"password\\\": null, \\\"registryIdentity\\\": null}, \\\"platform\\\": {\\\"os\\\": \\\"Linux\\\", \\\"architecture\\\": \\\"amd64\\\"}}, \\\"spark\\\": {\\\"repositories\\\": [], \\\"packages\\\": [], \\\"precachePackages\\\": true}, \\\"databricks\\\": {\\\"mavenLibraries\\\": [], \\\"pypiLibraries\\\": [], \\\"rcranLibraries\\\": [], \\\"jarLibraries\\\": [], \\\"eggLibraries\\\": []}, \\\"r\\\": null, \\\"inferencingStackVersion\\\": null}, \\\"history\\\": {\\\"outputCollection\\\": true, \\\"snapshotProject\\\": true, \\\"directoriesToWatch\\\": [\\\"logs\\\"]}, \\\"spark\\\": {\\\"configuration\\\": {\\\"spark.app.name\\\": \\\"Azure ML Experiment\\\", \\\"spark.yarn.maxAppAttempts\\\": 1}}, \\\"docker\\\": {\\\"useDocker\\\": false, \\\"sharedVolumes\\\": true, \\\"arguments\\\": [], \\\"shmSize\\\": \\\"2g\\\"}, \\\"hdi\\\": {\\\"yarnDeployMode\\\": \\\"cluster\\\"}, \\\"tensorflow\\\": {\\\"workerCount\\\": 1, \\\"parameterServerCount\\\": 1}, \\\"mpi\\\": {\\\"processCountPerNode\\\": 1, \\\"nodeCount\\\": 1}, \\\"pytorch\\\": {\\\"communicationBackend\\\": \\\"nccl\\\", \\\"processCount\\\": null, \\\"nodeCount\\\": 1}, \\\"paralleltask\\\": {\\\"maxRetriesPerWorker\\\": 0, \\\"workerCountPerNode\\\": 1, \\\"terminalExitCodes\\\": null}, \\\"dataReferences\\\": {}, \\\"data\\\": {\\\"doce_data\\\": {\\\"dataLocation\\\": {\\\"dataset\\\": {\\\"id\\\": \\\"542e6c51-8026-42f8-863d-8a55b709a88f\\\", \\\"name\\\": \\\"docetaxel_sensitivity\\\", \\\"version\\\": 1}, \\\"dataPath\\\": null, \\\"uri\\\": null}, \\\"createOutputDirectories\\\": false, \\\"mechanism\\\": \\\"direct\\\", \\\"environmentVariableName\\\": \\\"doce_data\\\", \\\"pathOnCompute\\\": null, \\\"overwrite\\\": false, \\\"options\\\": null}}, \\\"datacaches\\\": [], \\\"outputData\\\": {}, \\\"sourceDirectoryDataStore\\\": null, \\\"amlcompute\\\": {\\\"vmSize\\\": null, \\\"vmPriority\\\": null, \\\"retainCluster\\\": false, \\\"name\\\": null, \\\"clusterMaxNodeCount\\\": null}, \\\"kubernetescompute\\\": {\\\"instanceType\\\": null}, \\\"credentialPassthrough\\\": false, \\\"command\\\": \\\"\\\", \\\"environmentVariables\\\": {}, \\\"applicationEndpoints\\\": {}}, \\\"TargetDetails\\\": null, \\\"SnapshotId\\\": \\\"adb39e4c-47cd-4db4-952c-a9ce57b00f91\\\", \\\"TelemetryValues\\\": {\\\"amlClientType\\\": \\\"azureml-sdk-train\\\", \\\"amlClientModule\\\": \\\"[Scrubbed]\\\", \\\"amlClientFunction\\\": \\\"[Scrubbed]\\\", \\\"tenantId\\\": \\\"88d59d7d-aecb-41b2-90c5-55595de02536\\\", \\\"amlClientRequestId\\\": \\\"5cebd497-e2bb-4642-96ee-3a05cb0ee8b3\\\", \\\"amlClientSessionId\\\": \\\"961955fa-8dd5-407a-bf38-e3cbfc02a4cf\\\", \\\"subscriptionId\\\": \\\"0bb59590-d012-407d-a545-7513aae8c4a7\\\", \\\"estimator\\\": \\\"NoneType\\\", \\\"samplingMethod\\\": \\\"RANDOM\\\", \\\"terminationPolicy\\\": \\\"Default\\\", \\\"primaryMetricGoal\\\": \\\"maximize\\\", \\\"maxTotalRuns\\\": 10, \\\"maxConcurrentRuns\\\": 4, \\\"maxDurationMinutes\\\": 10080, \\\"vmSize\\\": null}}}\", \"_aml_system_resume_child_runs\": \"null\", \"_aml_system_all_jobs_generated\": \"true\", \"_aml_system_cancellation_requested\": \"true\", \"_aml_system_progress_metadata_evaluation_timestamp\": \"\\\"2022-04-03T01:26:41.399152\\\"\", \"_aml_system_progress_metadata_digest\": \"\\\"bdbb79ee641c92e41f1674482aef70c353babf625b8030ee55aee08bbe4fa904\\\"\", \"_aml_system_progress_metadata_active_timestamp\": \"\\\"2022-04-03T01:26:41.399152\\\"\", \"_aml_system_optimizer_state_artifact\": \"null\", \"_aml_system_outdated_optimizer_state_artifacts\": \"\\\"[]\\\"\", \"_aml_system_HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_0\": \"{\\\"--booster\\\": \\\"dart\\\", \\\"--max_depth\\\": 2391.5797699400573, \\\"--n_estimators\\\": 1}\", \"_aml_system_HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_1\": \"{\\\"--booster\\\": \\\"gbtree\\\", \\\"--max_depth\\\": 618.7737673456335, \\\"--n_estimators\\\": 100}\", \"_aml_system_HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_2\": \"{\\\"--booster\\\": \\\"gbtree\\\", \\\"--max_depth\\\": 2436.0157218289783, \\\"--n_estimators\\\": 5}\", \"_aml_system_HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_3\": \"{\\\"--booster\\\": \\\"gblinear\\\", \\\"--max_depth\\\": 106.16684340790636, \\\"--n_estimators\\\": 1}\", \"_aml_system_HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_4\": \"{\\\"--booster\\\": \\\"dart\\\", \\\"--max_depth\\\": 3625.4228305368006, \\\"--n_estimators\\\": 1}\", \"_aml_system_HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_5\": \"{\\\"--booster\\\": \\\"gblinear\\\", \\\"--max_depth\\\": 4941.384093608857, \\\"--n_estimators\\\": 1}\", \"_aml_system_HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_6\": \"{\\\"--booster\\\": \\\"dart\\\", \\\"--max_depth\\\": 1107.6435929062777, \\\"--n_estimators\\\": 1}\", \"_aml_system_HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_7\": \"{\\\"--booster\\\": \\\"gbtree\\\", \\\"--max_depth\\\": 2581.685194200145, \\\"--n_estimators\\\": 100}\", \"_aml_system_HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_4_cancelled\": \"true\", \"_aml_system_HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_5_cancelled\": \"true\", \"_aml_system_HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_6_cancelled\": \"true\"}, \"end_time_utc\": null, \"status\": \"CancelRequested\", \"log_files\": {\"azureml-logs/hyperdrive.txt\": \"https://dsba6190ml1753578132.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23/azureml-logs/hyperdrive.txt?sv=2019-07-07&sr=b&sig=wsr%2FvOIx7PFyJxPovE1q%2F%2B4xAzBMpnk8%2B6qy%2FQfCepA%3D&skoid=922cf51e-04be-4662-8597-706c835dfc1e&sktid=88d59d7d-aecb-41b2-90c5-55595de02536&skt=2022-04-03T00%3A56%3A11Z&ske=2022-04-04T09%3A06%3A11Z&sks=b&skv=2019-07-07&st=2022-04-03T01%3A22%3A41Z&se=2022-04-03T09%3A32%3A41Z&sp=r\"}, \"log_groups\": [[\"azureml-logs/hyperdrive.txt\"]], \"run_duration\": \"0:07:16\", \"run_number\": \"1648949200\", \"run_queued_details\": {\"status\": \"CancelRequested\", \"details\": null}, \"hyper_parameters\": {\"--n_estimators\": [\"choice\", [[1, 5, 100]]], \"--max_depth\": [\"uniform\", [100, 5000]], \"--booster\": [\"choice\", [[\"gbtree\", \"gblinear\", \"dart\"]]]}}, \"child_runs\": [{\"run_id\": \"HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_2\", \"run_number\": 1648949263, \"metric\": null, \"status\": \"Failed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2022-04-03T01:29:33.149545Z\", \"end_time\": \"2022-04-03T01:29:46.263285Z\", \"created_time\": \"2022-04-03T01:27:43.076814Z\", \"created_time_dt\": \"2022-04-03T01:27:43.076814Z\", \"duration\": \"0:02:03\", \"hyperdrive_id\": \"500ca449-1ded-492d-a8b8-4f9f6d35ee23\", \"arguments\": null, \"param_--booster\": \"gbtree\", \"param_--max_depth\": 2436.0157218289783, \"param_--n_estimators\": 5}, {\"run_id\": \"HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_1\", \"run_number\": 1648949262, \"metric\": null, \"status\": \"Failed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2022-04-03T01:29:08.717154Z\", \"end_time\": \"2022-04-03T01:29:46.504666Z\", \"created_time\": \"2022-04-03T01:27:42.639613Z\", \"created_time_dt\": \"2022-04-03T01:27:42.639613Z\", \"duration\": \"0:02:03\", \"hyperdrive_id\": \"500ca449-1ded-492d-a8b8-4f9f6d35ee23\", \"arguments\": null, \"param_--booster\": \"gbtree\", \"param_--max_depth\": 618.7737673456335, \"param_--n_estimators\": 100}, {\"run_id\": \"HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_6\", \"run_number\": 1648949445, \"metric\": null, \"status\": \"Canceled\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2022-04-03T01:31:17.447468Z\", \"end_time\": \"2022-04-03T01:32:16.78782Z\", \"created_time\": \"2022-04-03T01:30:45.598486Z\", \"created_time_dt\": \"2022-04-03T01:30:45.598486Z\", \"duration\": \"0:01:31\", \"hyperdrive_id\": \"500ca449-1ded-492d-a8b8-4f9f6d35ee23\", \"arguments\": null, \"param_--booster\": \"dart\", \"param_--max_depth\": 1107.6435929062777, \"param_--n_estimators\": 1}], \"children_metrics\": {\"categories\": null, \"series\": null, \"metricName\": null}, \"run_metrics\": [], \"run_logs\": \"[2022-04-03T01:26:40.829484][API][INFO]Experiment created\\r\\n[2022-04-03T01:26:42.092382][GENERATOR][INFO]Trying to sample '4' jobs from the hyperparameter space\\r\\n[2022-04-03T01:26:42.329047][GENERATOR][INFO]Successfully sampled '4' jobs, they will soon be submitted to the execution target.\\r\\n[2022-04-03T01:27:41.3868369Z][SCHEDULER][INFO]Scheduling job, id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_0'\\r\\n[2022-04-03T01:27:41.3879235Z][SCHEDULER][INFO]Scheduling job, id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_1'\\r\\n[2022-04-03T01:27:41.3886358Z][SCHEDULER][INFO]Scheduling job, id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_2'\\r\\n[2022-04-03T01:27:41.3892755Z][SCHEDULER][INFO]Scheduling job, id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_3'\\r\\n[2022-04-03T01:27:42.7192056Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_1'\\r\\n[2022-04-03T01:27:43.1616767Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_2'\\r\\n[2022-04-03T01:27:42.1953052Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_0'\\r\\n[2022-04-03T01:27:43.6753645Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_3'\\r\\n[2022-04-03T01:30:11.498621][GENERATOR][INFO]Trying to sample '3' jobs from the hyperparameter space\\r\\n[2022-04-03T01:30:11.661277][GENERATOR][INFO]Successfully sampled '3' jobs, they will soon be submitted to the execution target.\\r\\n[2022-04-03T01:30:44.7833634Z][SCHEDULER][INFO]Scheduling job, id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_4'\\r\\n[2022-04-03T01:30:44.7849589Z][SCHEDULER][INFO]Scheduling job, id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_6'\\r\\n[2022-04-03T01:30:44.7841256Z][SCHEDULER][INFO]Scheduling job, id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_5'\\r\\n[2022-04-03T01:30:45.3563368Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_4'\\r\\n[2022-04-03T01:30:45.4720124Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_5'\\r\\n[2022-04-03T01:30:45.6695491Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_6'\\r\\n[2022-04-03T01:31:41.425718][GENERATOR][INFO]Trying to sample '1' jobs from the hyperparameter space\\r\\n[2022-04-03T01:31:41.556799][GENERATOR][INFO]Successfully sampled '1' jobs, they will soon be submitted to the execution target.\\r\\n[2022-04-03T01:31:43.501692][CONTROLLER][WARNING]The first 4 jobs have failed. The system is canceling the experiment. Please resolve the issues before resubmitting the experiment.\\r\\n[2022-04-03T01:31:43.501641][CONTROLLER][INFO]Experiment has been marked for cancellation.\\r\\n[2022-04-03T01:32:16.2169182Z][SCHEDULER][INFO]Cancelling job, id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_4'\\r\\n[2022-04-03T01:32:16.2183944Z][SCHEDULER][INFO]Cancelling job, id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_6'\\r\\n[2022-04-03T01:32:16.2181367Z][SCHEDULER][INFO]Cancelling job, id='HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_5'\\r\\n[2022-04-03T01:32:16.5502477Z][SCHEDULER][INFO]Updating job statuses to cancelled: [(job id = 'HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_4', previous status = 'SCHEDULED'), (job id = 'HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_5', previous status = 'RUNNING'), (job id = 'HD_500ca449-1ded-492d-a8b8-4f9f6d35ee23_6', previous status = 'RUNNING')]\\r\\n[2022-04-03T01:32:31.235169][API][INFO]Processing cancellation request by the user.\\r\\n[2022-04-03T01:32:31.148515][API][INFO]Experiment has been marked as canceled by the user, all active/pending jobs will soon be terminated/canceled.\\n\", \"graph\": {}, \"widget_settings\": {\"childWidgetDisplay\": \"popup\", \"send_telemetry\": false, \"log_level\": \"INFO\", \"sdk_version\": \"1.37.0\"}, \"loading\": false}"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-22a20f620c40>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m## Show the status in the notebook as the experiment runs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mRunDetails\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mrun\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait_for_completion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/core/run.py\u001b[0m in \u001b[0;36mwait_for_completion\u001b[0;34m(self, show_output, wait_post_processing, raise_on_error)\u001b[0m\n\u001b[1;32m    859\u001b[0m             \u001b[0mpoll_start_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    860\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0mcurrent_status\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mcurrent_status\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrunning_states\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 861\u001b[0;31m                 \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mRun\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wait_before_polling\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mpoll_start_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    862\u001b[0m                 \u001b[0mcurrent_status\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    863\u001b[0m                 \u001b[0;31m# add unknown status as running status\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from azureml.widgets import RunDetails\n",
    "\n",
    "## Show the status in the notebook as the experiment runs\n",
    "RunDetails(run).show()\n",
    "run.wait_for_completion()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4159203",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 - AzureML",
   "language": "python",
   "name": "python3-azureml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}