00_ml_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c60007db-23cc-489a-b324-16311c540685",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 02_ml_pipeline\n",
    "\n",
    "This notebook covers the following tasks:\n",
    "\n",
    "1. Build the Pipeline\n",
    "2. Compile the Pipeline\n",
    "3. Execute the Pipeline on Vertex AI"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92723fa7-d935-402b-a190-d9c6da1e89ed",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9eb6592-2cf9-4970-af4c-bfad2ac19aab",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "272b7567-40cb-4144-af94-083ccf303424",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 318 µs (started: 2022-11-27 20:44:21 +00:00)\n"
     ]
    }
   ],
   "source": [
    "%load_ext autotime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "788586d3-dfa5-4651-b4fc-ee7da7b763bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 3 s (started: 2022-11-27 20:44:21 +00:00)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import datetime\n",
    "import json\n",
    "import kfp\n",
    "from kfp.v2.dsl import component\n",
    "from kfp.v2 import dsl, compiler\n",
    "from google.cloud import aiplatform as aip\n",
    "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
    "import gcsfs\n",
    "import importlib"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa9fc0f9-ba1c-48a5-83f5-a42052fbe662",
   "metadata": {},
   "source": [
    "### Set configurations and constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4afe13ba-ae85-4922-b74a-a466e51f1f98",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 1.09 ms (started: 2022-11-27 20:44:24 +00:00)\n"
     ]
    }
   ],
   "source": [
    "NOTEBOOK = '02_ml_pipeline'\n",
    "REGION = \"us-west1\"\n",
    "PROJECT = 'babyweight-prediction'\n",
    "BUCKET = 'b_w_bucket'\n",
    "BQ_DATASET = \"bw_dataset\"\n",
    "APPNAME = \"bw-prediction\"\n",
    "\n",
    "os.environ[\"REGION\"] = REGION\n",
    "os.environ[\"PROJECT\"] = PROJECT\n",
    "os.environ[\"BUCKET\"] = BUCKET"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "612c35d8-b940-442f-950e-4a723b776d1f",
   "metadata": {},
   "source": [
    "### Set service account access for Vertex AI Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "420a4215-48ed-425c-8c21-1904482fa17f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 1.03 ms (started: 2022-11-27 20:44:24 +00:00)\n"
     ]
    }
   ],
   "source": [
    "SERVICE_ACCOUNT = \"69318036822-compute@developer.gserviceaccount.com\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "027c67a8-a025-419c-a2af-e7184f8b2be3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[core]\n",
      "account = 69318036822-compute@developer.gserviceaccount.com\n",
      "disable_usage_reporting = True\n",
      "project = babyweight-prediction\n",
      "\n",
      "Your active configuration is: [default]\n",
      "time: 933 ms (started: 2022-11-27 20:44:24 +00:00)\n"
     ]
    }
   ],
   "source": [
    "!gcloud config list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d037a541-5686-42c7-ae0a-5cc7b3c542dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                  Credentialed Accounts\n",
      "ACTIVE  ACCOUNT\n",
      "*       69318036822-compute@developer.gserviceaccount.com\n",
      "\n",
      "To set the active account, run:\n",
      "    $ gcloud config set account `ACCOUNT`\n",
      "\n",
      "time: 938 ms (started: 2022-11-27 20:44:25 +00:00)\n"
     ]
    }
   ],
   "source": [
    "!gcloud auth list"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee4520c1-a5d9-47ee-bbe9-322914cc072b",
   "metadata": {},
   "source": [
    "### Initialize the Vertex AI SDK & Tensorboard Instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1d3dd206-1544-4b61-b861-73c60bebb389",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 649 µs (started: 2022-11-27 20:44:26 +00:00)\n"
     ]
    }
   ],
   "source": [
    "aip.init(project=PROJECT, location=REGION, staging_bucket=BUCKET)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5d1b2104-ebcc-491f-ac0b-ad1950e7ada2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 1.15 ms (started: 2022-11-27 20:44:26 +00:00)\n"
     ]
    }
   ],
   "source": [
    "## Create a Tensorboard instance\n",
    "#tb = aip.Tensorboard.create(display_name=f\"{APPNAME}_{NOTEBOOK}\", description=NOTEBOOK)\n",
    "#tb.resource_name"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a84a5017-c035-4663-8a9f-934871516195",
   "metadata": {},
   "source": [
    "## 1. Construct the Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9dadb3c2-9bac-41db-b0e6-fd83080ed413",
   "metadata": {},
   "source": [
    "### 1.1. Define pipeline constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ee5c3095-51e3-46b1-9cd2-e5d1549c8c3d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 3.43 ms (started: 2022-11-27 20:44:26 +00:00)\n"
     ]
    }
   ],
   "source": [
    "#TIMESTAMP = datetime.datetime.now().strftime('%Y%m%d%H%M%S')\n",
    "PIPELINE_URI = f\"gs://{BUCKET}/{APPNAME}\"\n",
    "\n",
    "#Data: extract_source_data\n",
    "SOURCE_BQ_TABLE_ID = \"bigquery-public-data.samples.natality\"\n",
    "YEAR = 2003\n",
    "EXTRACTED_BQ_TABLE_ID = f\"{PROJECT}.{BQ_DATASET}.{APPNAME}_extracted_{YEAR}\"\n",
    "\n",
    "#Data: prepped_table_creating_task\n",
    "LIMIT = 500000 #changed to 500,000\n",
    "PREPPED_BQ_TABLE_ID = f\"{PROJECT}.{BQ_DATASET}.{APPNAME}_extracted_{YEAR}_limit_{LIMIT}\" \n",
    "VERTEX_DATASET_DISPLAY_NAME = f\"{APPNAME}_extracted_{YEAR}_limit_{LIMIT}_dataset\"\n",
    "\n",
    "## bqml\n",
    "BQML_MODEL_NAME = f\"bqml_v2_{APPNAME}\"\n",
    "BQML_MODEL_VERSION_ALIASES = BQML_MODEL_NAME\n",
    "VAR_TARGET = \"weight_pounds\"\n",
    "#\n",
    "##automl\n",
    "AUTOML_MODEL_NAME = f\"automl_{APPNAME}\"\n",
    "COLUMN_SPECS = {\n",
    "    \"is_male\": \"categorical\",\n",
    "    \"mother_age\": \"numeric\",\n",
    "    \"plurality\": \"categorical\",\n",
    "    \"gestation_weeks\": \"numeric\",\n",
    "    \"cigarette_use\": \"categorical\",\n",
    "    \"alcohol_use\": \"categorical\",\n",
    "}\n",
    "#\n",
    "##model validation\n",
    "REFERENCE_METRIC_NAME = \"mae\"\n",
    "THRESHOLDS_DICT = {\"mae\": 50.0}\n",
    "#\n",
    "##deployment\n",
    "ENDPOINT_DISPLAY_NAME = f\"{APPNAME}_endpoint\"\n",
    "DEPLOY_IMAGE=\"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-7:latest\"\n",
    "DEPLOY_MACHINE_TYPE = \"n1-highmem-4\"\n",
    "#DEPLOY_MACHINE_TYPE = \"n1-standard-1\"\n",
    "DEPLOY_MIN_REPLICA_COUNT = 1\n",
    "DEPLOY_MAX_REPLICA_COUNT = 1\n",
    "\n",
    "#EXPLANATION_PARAMATERS = {\"sampledShapleyAttribution\": {\"pathCount\": 10}}\n",
    "#EXPLANATION_METADATA_JSON_PATH = f\"{CUSTOM_TRAININGS_URI[5:]}/explanation_metadata.json\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad8434a7-9261-4424-a66c-4249ea72236e",
   "metadata": {},
   "source": [
    "### 1.3. BQML queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d3d20722-d3b8-4510-bc03-1b82ee407aab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'src.pipeline.serving_validation_comp' from '/home/jupyter/babyweight-vertex-ai/src/pipeline/serving_validation_comp.py'>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 82.7 ms (started: 2022-11-27 20:44:26 +00:00)\n"
     ]
    }
   ],
   "source": [
    "from src.pipeline import data_comp\n",
    "from src.pipeline import bqml_comp\n",
    "from src.pipeline import automl_comp\n",
    "from src.pipeline import model_validation_comp\n",
    "from src.pipeline import serving_validation_comp\n",
    "#\n",
    "importlib.reload(data_comp)\n",
    "importlib.reload(bqml_comp)\n",
    "importlib.reload(automl_comp)\n",
    "importlib.reload(model_validation_comp)\n",
    "importlib.reload(serving_validation_comp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b980cdbd-02ba-433a-9774-dbc0b85490bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "QUERY_BUILDING_BQML_MODEL: \n",
      "    CREATE MODEL IF NOT EXISTS `babyweight-prediction.bw_dataset.bqml_v2_bw-prediction`\n",
      "    OPTIONS(\n",
      "        model_type = 'DNN_LINEAR_COMBINED_REGRESSOR',\n",
      "        model_registry = 'vertex_ai', \n",
      "        vertex_ai_model_version_aliases = ['bqml_v2_bw-prediction'],\n",
      "        input_label_cols = ['weight_pounds'],\n",
      "        data_split_col = 'custom_splits',\n",
      "        data_split_method = 'CUSTOM',\n",
      "        HIDDEN_UNITS = [256, 128, 64],\n",
      "        OPTIMIZER = 'adagrad',\n",
      "        BATCH_SIZE = HPARAM_CANDIDATES([16, 32, 64]),\n",
      "        DROPOUT =  HPARAM_CANDIDATES([0, 0.1, 0.2]),\n",
      "        MAX_ITERATIONS = 5,\n",
      "        MAX_PARALLEL_TRIALS = 4,\n",
      "        NUM_TRIALS = 24\n",
      "        ) AS\n",
      "    SELECT * EXCEPT(splits),\n",
      "        CASE\n",
      "            WHEN splits = 'VALIDATE' THEN 'EVAL'\n",
      "            ELSE splits\n",
      "        END AS custom_splits\n",
      "    FROM `babyweight-prediction.bw_dataset.bw-prediction_extracted_2003_limit_500000`\n",
      "    \n",
      "time: 1.01 ms (started: 2022-11-27 20:44:26 +00:00)\n"
     ]
    }
   ],
   "source": [
    "QUERY_BUILDING_BQML_MODEL = bqml_comp.create_query_build_bqml_model(\n",
    "    project=PROJECT,\n",
    "    bq_dataset=BQ_DATASET,\n",
    "    bq_model_name=BQML_MODEL_NAME,\n",
    "    bq_version_aliases=BQML_MODEL_VERSION_ALIASES,\n",
    "    var_target=VAR_TARGET,\n",
    "    bq_train_table_id=PREPPED_BQ_TABLE_ID\n",
    ")\n",
    "print(\"QUERY_BUILDING_BQML_MODEL:\", QUERY_BUILDING_BQML_MODEL)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90cb9a89-7419-48c2-9489-6393ff84a877",
   "metadata": {},
   "source": [
    "### 1.4. Define the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "54438d98-0dd5-41ab-b83b-65fe9a5d6d70",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 3.33 ms (started: 2022-11-27 20:44:26 +00:00)\n"
     ]
    }
   ],
   "source": [
    "@dsl.pipeline(name = APPNAME, pipeline_root = PIPELINE_URI)\n",
    "def vertex_ai_pipeline(\n",
    "    service_account: str=SERVICE_ACCOUNT,\n",
    "    project: str=PROJECT,\n",
    "    region: str=REGION,\n",
    "):\n",
    "    from google_cloud_pipeline_components import aiplatform as gcpc_aip\n",
    "    from google_cloud_pipeline_components.v1.bigquery import BigqueryCreateModelJobOp\n",
    "    from google_cloud_pipeline_components.v1.bigquery import BigqueryEvaluateModelJobOp\n",
    "    \n",
    "    ##################################\n",
    "    #extract data from original natality dataset\n",
    "    extracted_source_data_task = data_comp.extract_source_data(\n",
    "        project=project,\n",
    "        region=region,\n",
    "        year=YEAR,\n",
    "        in_bq_table_id=SOURCE_BQ_TABLE_ID, \n",
    "        out_bq_table_id=EXTRACTED_BQ_TABLE_ID,  \n",
    "    ).set_display_name('create_extracted_table') \n",
    "    \n",
    "    ##################################\n",
    "    \n",
    "    prepped_table_creating_task = data_comp.preprocess_data(\n",
    "        project=project,\n",
    "        region=region,\n",
    "        limit=LIMIT,\n",
    "        in_bq_table_id=EXTRACTED_BQ_TABLE_ID, \n",
    "        out_bq_table_id=PREPPED_BQ_TABLE_ID,  \n",
    "    ).set_display_name('create_prepped_table').after(extracted_source_data_task)\n",
    "    \n",
    "    \n",
    "    vertex_dataset_creating_task = gcc_aip.TabularDatasetCreateOp(\n",
    "        project=project,\n",
    "        display_name=VERTEX_DATASET_DISPLAY_NAME,\n",
    "        bq_source=prepped_table_creating_task.outputs[\"out_bq_table_uri\"],\n",
    "    ).set_display_name('create_vertex_dataset')    \n",
    "\n",
    "    ##################################\n",
    "    \n",
    "    bqml_creating_task = BigqueryCreateModelJobOp(\n",
    "        project=project, \n",
    "        location=\"US\",\n",
    "        query=QUERY_BUILDING_BQML_MODEL,\n",
    "    ).set_display_name('create_bqml_model').after(prepped_table_creating_task)\n",
    "    \n",
    "    bqml_evaluating_task = BigqueryEvaluateModelJobOp(\n",
    "        project=project, \n",
    "        location=\"US\",\n",
    "        model=bqml_creating_task.outputs[\"model\"]\n",
    "    ).set_display_name('evaluate_bqml_model')\n",
    "    \n",
    "    bqml_evaluation_metrics_collecting_task = bqml_comp.collect_eval_metrics_bqml(\n",
    "        eval_metrics_artifact=bqml_evaluating_task.outputs[\"evaluation_metrics\"]\n",
    "    )\n",
    "\n",
    "    ###################################\n",
    "    \n",
    "    automl_creating_task = gcc_aip.AutoMLTabularTrainingJobRunOp(\n",
    "        project=project,\n",
    "        display_name=f\"{AUTOML_MODEL_NAME}_TrainingJob\",\n",
    "        optimization_prediction_type=\"regression\",\n",
    "        optimization_objective=\"minimize-rmse\",\n",
    "        budget_milli_node_hours=3000, #changed from 1,000\n",
    "        model_display_name=AUTOML_MODEL_NAME,\n",
    "        dataset=vertex_dataset_creating_task.outputs[\"dataset\"],\n",
    "        column_specs=COLUMN_SPECS,\n",
    "        target_column=VAR_TARGET,\n",
    "    ).set_display_name('create_automl_model')\n",
    "    \n",
    "    automl_evaluation_metrics_collecting_task = automl_comp.collect_eval_metrics_automl(\n",
    "        region='us-central1', \n",
    "        model=automl_creating_task.outputs[\"model\"]\n",
    "    )\n",
    "    \n",
    "    ###################################\n",
    "    \n",
    "    best_model_task = model_validation_comp.select_best_model(\n",
    "        automl_metrics=automl_evaluation_metrics_collecting_task.outputs[\"metrics\"],\n",
    "        automl_model=automl_creating_task.outputs[\"model\"],\n",
    "        bqml_metrics=bqml_evaluation_metrics_collecting_task.outputs[\"metrics\"],\n",
    "        bqml_model=bqml_creating_task.outputs[\"model\"],  #Argument type \"Model\" is incompatible with the input type \"google.VertexModel\"\n",
    "        reference_metric_name=REFERENCE_METRIC_NAME,\n",
    "        thresholds_dict=THRESHOLDS_DICT,\n",
    "    )\n",
    "    \n",
    "    ##### DEPLOYMENT ############################### \n",
    "    endpoint_creating_task = gcc_aip.EndpointCreateOp(\n",
    "        project=project,\n",
    "        location='us-central1',\n",
    "        #location=region,\n",
    "        display_name=ENDPOINT_DISPLAY_NAME,\n",
    "    ).set_display_name('create_endpoint')\n",
    "\n",
    "    #===============================#    \n",
    "    # If the deploy condition is True, then deploy the best model.\n",
    "    \n",
    "    with dsl.Condition(\n",
    "        best_model_task.outputs[\"deploy_decision\"] == \"true\",\n",
    "        name=\"deploy_decision\",\n",
    "    ):\n",
    "               \n",
    "        #===============================#\n",
    "        with dsl.Condition(\n",
    "            best_model_task.outputs[\"best_model_name\"] == \"automl\",\n",
    "            name=\"deploy_automl\",\n",
    "        ): \n",
    "        \n",
    "            # deploy the model to endpoint\n",
    "            automl_model_deploy_task = gcc_aip.ModelDeployOp(\n",
    "                #model=best_model_task.outputs[\"best_model\"],\n",
    "                model=automl_creating_task.outputs[\"model\"],\n",
    "                endpoint=endpoint_creating_task.outputs[\"endpoint\"],\n",
    "                dedicated_resources_machine_type=DEPLOY_MACHINE_TYPE,\n",
    "                dedicated_resources_min_replica_count=DEPLOY_MIN_REPLICA_COUNT,\n",
    "                dedicated_resources_max_replica_count=DEPLOY_MAX_REPLICA_COUNT,\n",
    "                traffic_split={\"0\": 100},\n",
    "            ).set_display_name('deploy_model_to_endpoint')    \n",
    "            \n",
    "    \n",
    "       \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec151116-54fb-4743-8ecc-b5284081e20b",
   "metadata": {},
   "source": [
    "## 2. Compile the Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7f8a9c2e-87b3-442a-ba26-a0397d907a3d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 242 ms (started: 2022-11-27 20:44:26 +00:00)\n"
     ]
    }
   ],
   "source": [
    "# Create a new (local) directory to store the complied file\n",
    "DIR = f\"temp/{NOTEBOOK}\"\n",
    "!rm -rf {DIR}\n",
    "!mkdir -p {DIR}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "13d5cdea-d6f0-4017-b218-7347bb22c3d0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/kfp/v2/compiler/compiler.py:1293: FutureWarning: APIs imported from the v1 namespace (e.g. kfp.dsl, kfp.components, etc) will not be supported by the v2 compiler since v2.0.0\n",
      "  category=FutureWarning,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 674 ms (started: 2022-11-27 20:44:26 +00:00)\n"
     ]
    }
   ],
   "source": [
    "compiled_package = f\"{DIR}/compiled_package.json\"\n",
    "\n",
    "compiler.Compiler().compile(\n",
    "    pipeline_func = vertex_ai_pipeline,\n",
    "    package_path = compiled_package\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "020ffd92-c26c-41e8-9ba0-76d84a674e25",
   "metadata": {},
   "source": [
    "## 3. Execute the Pipeline on Vertex AI"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca235704-e4bb-4d28-89bb-ba5f57435a9b",
   "metadata": {},
   "source": [
    "### 3.1. Create Vertex AI Pipeline Job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9a5bf56a-2794-4dd6-82df-1e2d04a7147f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 184 ms (started: 2022-11-27 20:44:27 +00:00)\n"
     ]
    }
   ],
   "source": [
    "pipeline_job = aip.PipelineJob(\n",
    "    display_name = f\"{APPNAME}\",\n",
    "    template_path = compiled_package,\n",
    "    pipeline_root=PIPELINE_URI,\n",
    "    parameter_values = {\n",
    "        \"project\": PROJECT,\n",
    "        \"region\": REGION,\n",
    "    },\n",
    "    #enable_caching = False,        #//TRUE, by default//\n",
    "    labels = {\n",
    "        'notebook':f'{NOTEBOOK}',\n",
    "        'app':f'{APPNAME}'}\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ac52c87-23f1-4aa5-ad24-82a1968a20b9",
   "metadata": {},
   "source": [
    "### 3.2. Run the job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "38018f0c-8ec7-405b-a85d-8891489970d3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating PipelineJob\n",
      "PipelineJob created. Resource name: projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427\n",
      "To use this PipelineJob in another session:\n",
      "pipeline_job = aiplatform.PipelineJob.get('projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427')\n",
      "View Pipeline Job:\n",
      "https://console.cloud.google.com/vertex-ai/locations/us-west1/pipelines/runs/bw-prediction-20221127204427?project=69318036822\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n",
      "PipelineJob run completed. Resource name: projects/69318036822/locations/us-west1/pipelineJobs/bw-prediction-20221127204427\n",
      "time: 4h 16min 44s (started: 2022-11-27 20:44:27 +00:00)\n"
     ]
    }
   ],
   "source": [
    "response = pipeline_job.run(\n",
    "    service_account = SERVICE_ACCOUNT,\n",
    "    #sync = True\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb255607-9934-4437-9a28-48667123037d",
   "metadata": {},
   "source": [
    "## 4. View the Pipeline results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2ac7ac6d-b999-4461-a7e9-093453857298",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pipeline_name</th>\n",
       "      <th>run_name</th>\n",
       "      <th>param.input:project</th>\n",
       "      <th>param.input:service_account</th>\n",
       "      <th>param.input:region</th>\n",
       "      <th>metric.rSquared</th>\n",
       "      <th>metric.rootMeanSquaredError</th>\n",
       "      <th>metric.meanAbsoluteError</th>\n",
       "      <th>metric.meanAbsolutePercentageError</th>\n",
       "      <th>metric.rootMeanSquaredLogError</th>\n",
       "      <th>metric.framework</th>\n",
       "      <th>metric.r2_score</th>\n",
       "      <th>metric.mean_squared_log_error</th>\n",
       "      <th>metric.median_absolute_error</th>\n",
       "      <th>metric.mean_absolute_error</th>\n",
       "      <th>metric.mean_squared_error</th>\n",
       "      <th>metric.root_mean_squared_error</th>\n",
       "      <th>metric.explained_variance</th>\n",
       "      <th>metric.trial_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bw-prediction</td>\n",
       "      <td>bw-prediction-20221127204427</td>\n",
       "      <td>babyweight-prediction</td>\n",
       "      <td>69318036822-compute@developer.gserviceaccount.com</td>\n",
       "      <td>us-west1</td>\n",
       "      <td>0.386401</td>\n",
       "      <td>1.010693</td>\n",
       "      <td>0.782775</td>\n",
       "      <td>12.299472</td>\n",
       "      <td>0.136865</td>\n",
       "      <td>AutoML</td>\n",
       "      <td>0.347389</td>\n",
       "      <td>0.022611</td>\n",
       "      <td>0.660123</td>\n",
       "      <td>0.812205</td>\n",
       "      <td>1.093847</td>\n",
       "      <td>1.045872</td>\n",
       "      <td>0.357209</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bw-prediction</td>\n",
       "      <td>bw-prediction-20221127202315</td>\n",
       "      <td>babyweight-prediction</td>\n",
       "      <td>69318036822-compute@developer.gserviceaccount.com</td>\n",
       "      <td>us-west1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>BQML</td>\n",
       "      <td>0.443393</td>\n",
       "      <td>0.021446</td>\n",
       "      <td>0.675051</td>\n",
       "      <td>0.843462</td>\n",
       "      <td>1.173432</td>\n",
       "      <td>1.083250</td>\n",
       "      <td>0.445383</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bw-prediction</td>\n",
       "      <td>bw-prediction-20221127200235</td>\n",
       "      <td>babyweight-prediction</td>\n",
       "      <td>69318036822-compute@developer.gserviceaccount.com</td>\n",
       "      <td>us-west1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>BQML</td>\n",
       "      <td>0.443393</td>\n",
       "      <td>0.021446</td>\n",
       "      <td>0.675051</td>\n",
       "      <td>0.843462</td>\n",
       "      <td>1.173432</td>\n",
       "      <td>1.083250</td>\n",
       "      <td>0.445383</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bw-prediction</td>\n",
       "      <td>bw-prediction-20221124191247</td>\n",
       "      <td>babyweight-prediction</td>\n",
       "      <td>69318036822-compute@developer.gserviceaccount.com</td>\n",
       "      <td>us-west1</td>\n",
       "      <td>0.483891</td>\n",
       "      <td>1.071977</td>\n",
       "      <td>0.823724</td>\n",
       "      <td>13.171675</td>\n",
       "      <td>0.145592</td>\n",
       "      <td>BQML</td>\n",
       "      <td>0.443393</td>\n",
       "      <td>0.021446</td>\n",
       "      <td>0.675051</td>\n",
       "      <td>0.843462</td>\n",
       "      <td>1.173432</td>\n",
       "      <td>1.083250</td>\n",
       "      <td>0.445383</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bw-prediction</td>\n",
       "      <td>bw-prediction-20221124180611</td>\n",
       "      <td>babyweight-prediction</td>\n",
       "      <td>69318036822-compute@developer.gserviceaccount.com</td>\n",
       "      <td>us-west1</td>\n",
       "      <td>0.483891</td>\n",
       "      <td>1.071977</td>\n",
       "      <td>0.823724</td>\n",
       "      <td>13.171675</td>\n",
       "      <td>0.145592</td>\n",
       "      <td>BQML</td>\n",
       "      <td>0.443393</td>\n",
       "      <td>0.021446</td>\n",
       "      <td>0.675051</td>\n",
       "      <td>0.843462</td>\n",
       "      <td>1.173432</td>\n",
       "      <td>1.083250</td>\n",
       "      <td>0.445383</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>bw-prediction</td>\n",
       "      <td>bw-prediction-20221124180033</td>\n",
       "      <td>babyweight-prediction</td>\n",
       "      <td>69318036822-compute@developer.gserviceaccount.com</td>\n",
       "      <td>us-west1</td>\n",
       "      <td>0.483891</td>\n",
       "      <td>1.071977</td>\n",
       "      <td>0.823724</td>\n",
       "      <td>13.171675</td>\n",
       "      <td>0.145592</td>\n",
       "      <td>AutoML</td>\n",
       "      <td>0.443393</td>\n",
       "      <td>0.021446</td>\n",
       "      <td>0.675051</td>\n",
       "      <td>0.843462</td>\n",
       "      <td>1.173432</td>\n",
       "      <td>1.083250</td>\n",
       "      <td>0.445383</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>bw-prediction</td>\n",
       "      <td>bw-prediction-20221124175128</td>\n",
       "      <td>babyweight-prediction</td>\n",
       "      <td>69318036822-compute@developer.gserviceaccount.com</td>\n",
       "      <td>us-west1</td>\n",
       "      <td>0.483891</td>\n",
       "      <td>1.071977</td>\n",
       "      <td>0.823724</td>\n",
       "      <td>13.171675</td>\n",
       "      <td>0.145592</td>\n",
       "      <td>BQML</td>\n",
       "      <td>0.443393</td>\n",
       "      <td>0.021446</td>\n",
       "      <td>0.675051</td>\n",
       "      <td>0.843462</td>\n",
       "      <td>1.173432</td>\n",
       "      <td>1.083250</td>\n",
       "      <td>0.445383</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>bw-prediction</td>\n",
       "      <td>bw-prediction-20221124174620</td>\n",
       "      <td>babyweight-prediction</td>\n",
       "      <td>69318036822-compute@developer.gserviceaccount.com</td>\n",
       "      <td>us-west1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>BQML</td>\n",
       "      <td>0.443393</td>\n",
       "      <td>0.021446</td>\n",
       "      <td>0.675051</td>\n",
       "      <td>0.843462</td>\n",
       "      <td>1.173432</td>\n",
       "      <td>1.083250</td>\n",
       "      <td>0.445383</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>bw-prediction</td>\n",
       "      <td>bw-prediction-20221124045408</td>\n",
       "      <td>babyweight-prediction</td>\n",
       "      <td>69318036822-compute@developer.gserviceaccount.com</td>\n",
       "      <td>us-west1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>BQML</td>\n",
       "      <td>0.443393</td>\n",
       "      <td>0.021446</td>\n",
       "      <td>0.675051</td>\n",
       "      <td>0.843462</td>\n",
       "      <td>1.173432</td>\n",
       "      <td>1.083250</td>\n",
       "      <td>0.445383</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>bw-prediction</td>\n",
       "      <td>bw-prediction-20221124042211</td>\n",
       "      <td>babyweight-prediction</td>\n",
       "      <td>69318036822-compute@developer.gserviceaccount.com</td>\n",
       "      <td>us-west1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>BQML</td>\n",
       "      <td>0.443393</td>\n",
       "      <td>0.021446</td>\n",
       "      <td>0.675051</td>\n",
       "      <td>0.843462</td>\n",
       "      <td>1.173432</td>\n",
       "      <td>1.083250</td>\n",
       "      <td>0.445383</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>bw-prediction</td>\n",
       "      <td>bw-prediction-20221124034906</td>\n",
       "      <td>babyweight-prediction</td>\n",
       "      <td>69318036822-compute@developer.gserviceaccount.com</td>\n",
       "      <td>us-west1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>bw-prediction</td>\n",
       "      <td>bw-prediction-20221124011317</td>\n",
       "      <td>babyweight-prediction</td>\n",
       "      <td>69318036822-compute@developer.gserviceaccount.com</td>\n",
       "      <td>us-west1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>bw-prediction</td>\n",
       "      <td>bw-prediction-20221124005608</td>\n",
       "      <td>babyweight-prediction</td>\n",
       "      <td>69318036822-compute@developer.gserviceaccount.com</td>\n",
       "      <td>us-west1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    pipeline_name                      run_name    param.input:project  \\\n",
       "0   bw-prediction  bw-prediction-20221127204427  babyweight-prediction   \n",
       "1   bw-prediction  bw-prediction-20221127202315  babyweight-prediction   \n",
       "2   bw-prediction  bw-prediction-20221127200235  babyweight-prediction   \n",
       "3   bw-prediction  bw-prediction-20221124191247  babyweight-prediction   \n",
       "4   bw-prediction  bw-prediction-20221124180611  babyweight-prediction   \n",
       "5   bw-prediction  bw-prediction-20221124180033  babyweight-prediction   \n",
       "6   bw-prediction  bw-prediction-20221124175128  babyweight-prediction   \n",
       "7   bw-prediction  bw-prediction-20221124174620  babyweight-prediction   \n",
       "8   bw-prediction  bw-prediction-20221124045408  babyweight-prediction   \n",
       "9   bw-prediction  bw-prediction-20221124042211  babyweight-prediction   \n",
       "10  bw-prediction  bw-prediction-20221124034906  babyweight-prediction   \n",
       "11  bw-prediction  bw-prediction-20221124011317  babyweight-prediction   \n",
       "12  bw-prediction  bw-prediction-20221124005608  babyweight-prediction   \n",
       "\n",
       "                          param.input:service_account param.input:region  \\\n",
       "0   69318036822-compute@developer.gserviceaccount.com           us-west1   \n",
       "1   69318036822-compute@developer.gserviceaccount.com           us-west1   \n",
       "2   69318036822-compute@developer.gserviceaccount.com           us-west1   \n",
       "3   69318036822-compute@developer.gserviceaccount.com           us-west1   \n",
       "4   69318036822-compute@developer.gserviceaccount.com           us-west1   \n",
       "5   69318036822-compute@developer.gserviceaccount.com           us-west1   \n",
       "6   69318036822-compute@developer.gserviceaccount.com           us-west1   \n",
       "7   69318036822-compute@developer.gserviceaccount.com           us-west1   \n",
       "8   69318036822-compute@developer.gserviceaccount.com           us-west1   \n",
       "9   69318036822-compute@developer.gserviceaccount.com           us-west1   \n",
       "10  69318036822-compute@developer.gserviceaccount.com           us-west1   \n",
       "11  69318036822-compute@developer.gserviceaccount.com           us-west1   \n",
       "12  69318036822-compute@developer.gserviceaccount.com           us-west1   \n",
       "\n",
       "    metric.rSquared  metric.rootMeanSquaredError  metric.meanAbsoluteError  \\\n",
       "0          0.386401                     1.010693                  0.782775   \n",
       "1               NaN                          NaN                       NaN   \n",
       "2               NaN                          NaN                       NaN   \n",
       "3          0.483891                     1.071977                  0.823724   \n",
       "4          0.483891                     1.071977                  0.823724   \n",
       "5          0.483891                     1.071977                  0.823724   \n",
       "6          0.483891                     1.071977                  0.823724   \n",
       "7               NaN                          NaN                       NaN   \n",
       "8               NaN                          NaN                       NaN   \n",
       "9               NaN                          NaN                       NaN   \n",
       "10              NaN                          NaN                       NaN   \n",
       "11              NaN                          NaN                       NaN   \n",
       "12              NaN                          NaN                       NaN   \n",
       "\n",
       "    metric.meanAbsolutePercentageError  metric.rootMeanSquaredLogError  \\\n",
       "0                            12.299472                        0.136865   \n",
       "1                                  NaN                             NaN   \n",
       "2                                  NaN                             NaN   \n",
       "3                            13.171675                        0.145592   \n",
       "4                            13.171675                        0.145592   \n",
       "5                            13.171675                        0.145592   \n",
       "6                            13.171675                        0.145592   \n",
       "7                                  NaN                             NaN   \n",
       "8                                  NaN                             NaN   \n",
       "9                                  NaN                             NaN   \n",
       "10                                 NaN                             NaN   \n",
       "11                                 NaN                             NaN   \n",
       "12                                 NaN                             NaN   \n",
       "\n",
       "   metric.framework  metric.r2_score  metric.mean_squared_log_error  \\\n",
       "0            AutoML         0.347389                       0.022611   \n",
       "1              BQML         0.443393                       0.021446   \n",
       "2              BQML         0.443393                       0.021446   \n",
       "3              BQML         0.443393                       0.021446   \n",
       "4              BQML         0.443393                       0.021446   \n",
       "5            AutoML         0.443393                       0.021446   \n",
       "6              BQML         0.443393                       0.021446   \n",
       "7              BQML         0.443393                       0.021446   \n",
       "8              BQML         0.443393                       0.021446   \n",
       "9              BQML         0.443393                       0.021446   \n",
       "10              NaN              NaN                            NaN   \n",
       "11              NaN              NaN                            NaN   \n",
       "12              NaN              NaN                            NaN   \n",
       "\n",
       "    metric.median_absolute_error  metric.mean_absolute_error  \\\n",
       "0                       0.660123                    0.812205   \n",
       "1                       0.675051                    0.843462   \n",
       "2                       0.675051                    0.843462   \n",
       "3                       0.675051                    0.843462   \n",
       "4                       0.675051                    0.843462   \n",
       "5                       0.675051                    0.843462   \n",
       "6                       0.675051                    0.843462   \n",
       "7                       0.675051                    0.843462   \n",
       "8                       0.675051                    0.843462   \n",
       "9                       0.675051                    0.843462   \n",
       "10                           NaN                         NaN   \n",
       "11                           NaN                         NaN   \n",
       "12                           NaN                         NaN   \n",
       "\n",
       "    metric.mean_squared_error  metric.root_mean_squared_error  \\\n",
       "0                    1.093847                        1.045872   \n",
       "1                    1.173432                        1.083250   \n",
       "2                    1.173432                        1.083250   \n",
       "3                    1.173432                        1.083250   \n",
       "4                    1.173432                        1.083250   \n",
       "5                    1.173432                        1.083250   \n",
       "6                    1.173432                        1.083250   \n",
       "7                    1.173432                        1.083250   \n",
       "8                    1.173432                        1.083250   \n",
       "9                    1.173432                        1.083250   \n",
       "10                        NaN                             NaN   \n",
       "11                        NaN                             NaN   \n",
       "12                        NaN                             NaN   \n",
       "\n",
       "    metric.explained_variance  metric.trial_id  \n",
       "0                    0.357209              9.0  \n",
       "1                    0.445383              2.0  \n",
       "2                    0.445383              2.0  \n",
       "3                    0.445383              2.0  \n",
       "4                    0.445383              2.0  \n",
       "5                    0.445383              2.0  \n",
       "6                    0.445383              2.0  \n",
       "7                    0.445383              2.0  \n",
       "8                    0.445383              2.0  \n",
       "9                    0.445383              2.0  \n",
       "10                        NaN              NaN  \n",
       "11                        NaN              NaN  \n",
       "12                        NaN              NaN  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 1.84 s (started: 2022-11-28 01:01:11 +00:00)\n"
     ]
    }
   ],
   "source": [
    "pipeline_df = aip.get_pipeline_df(APPNAME)\n",
    "pipeline_df = pipeline_df[pipeline_df.pipeline_name == APPNAME]\n",
    "pipeline_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "371f5b9e-b32c-4e72-9ca8-7350a7634059",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROJECT_NUMBER: 69318036822\n",
      "PIPELINE_JOB_ID: bw-prediction-20221127204427\n",
      "PIPELINE_TASKS_URI: gs://b_w_bucket/bw-prediction/69318036822/bw-prediction-20221127204427\n",
      "time: 1.32 ms (started: 2022-11-28 01:01:13 +00:00)\n"
     ]
    }
   ],
   "source": [
    "PROJECT_NUMBER = pipeline_job.resource_name.split(\"/\")[1]\n",
    "PIPELINE_JOB_ID = pipeline_job.name\n",
    "PIPELINE_TASKS_URI = f\"{PIPELINE_URI}/{PROJECT_NUMBER}/{PIPELINE_JOB_ID}\"\n",
    "\n",
    "print(\"PROJECT_NUMBER:\",PROJECT_NUMBER)\n",
    "print(\"PIPELINE_JOB_ID:\",PIPELINE_JOB_ID)\n",
    "print(\"PIPELINE_TASKS_URI:\",PIPELINE_TASKS_URI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "10adb13f-a372-48e6-af8b-815474a421d9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All pipeline tasks:\n",
      " 0: tabular-dataset-create\n",
      " 1: bigquery-evaluate-model-job\n",
      " 2: bw-prediction-20221127204427\n",
      " 3: endpoint-create\n",
      " 4: collect-eval-metrics-bqml\n",
      " 5: automl-tabular-training-job\n",
      " 6: model-deploy\n",
      " 7: preprocess-data\n",
      " 8: select-best-model\n",
      " 9: bigquery-create-model-job\n",
      " 10: condition-deploy-decision-1\n",
      " 11: condition-deploy-automl-2\n",
      " 12: collect-eval-metrics-automl\n",
      " 13: extract-source-data\n",
      "--- Index of `endpoint-create` task: 3\n",
      "time: 3.89 ms (started: 2022-11-28 01:01:13 +00:00)\n"
     ]
    }
   ],
   "source": [
    "all_pipeline_tasks = pipeline_job.gca_resource.job_detail.task_details\n",
    "endpoint_create_task_index = 0 \n",
    "print(\"All pipeline tasks:\")\n",
    "for i in range (len(all_pipeline_tasks)):\n",
    "    print(f\" {i}: {all_pipeline_tasks[i].task_name}\")\n",
    "    if all_pipeline_tasks[i].task_name ==\"endpoint-create\": endpoint_create_task_index = i \n",
    "\n",
    "print(\"--- Index of `endpoint-create` task:\", endpoint_create_task_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cff8def9-0fcf-4658-8e8b-c252dad725c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'projects/69318036822/locations/us-central1/endpoints/4074389870305345536'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 5.63 ms (started: 2022-11-28 01:01:13 +00:00)\n"
     ]
    }
   ],
   "source": [
    "endpoint_resource_name = all_pipeline_tasks[endpoint_create_task_index].outputs[\"endpoint\"].artifacts[0].metadata[\"resourceName\"]\n",
    "endpoint_resource_name"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51b0a3df-b67e-4f04-867e-51931ec89a0f",
   "metadata": {},
   "source": [
    "***Click here to verify:***  \n",
    "https://console.cloud.google.com/vertex-ai/locations/us-central1/endpoints/5840539796048445440?project=asc-ahnat-adsi-sandbox"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3519a545-2599-4894-8fd1-f18b865c516e",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17e93af1-23cd-4b81-a756-14738e5c4015",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10ebf5f2-d000-46f5-b745-ee96dc376b1f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe3daaa8-de8b-4d2d-a86a-51cfbca185a3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b589700-21cb-4d04-a9a4-2e85eab36e56",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "tf2-gpu.2-8.m100",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-8:m100"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}