From 66b565ddb645743d2bbdbe35160faaa07e2a62f9 Mon Sep 17 00:00:00 2001 From: Mansi Achuthan Date: Mon, 16 Aug 2021 17:30:28 +0000 Subject: [PATCH 01/13] Added BigQuery location of exported evaluated examples from AutoML Forecasting Training Job in training output --- google/cloud/aiplatform/training_jobs.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/google/cloud/aiplatform/training_jobs.py b/google/cloud/aiplatform/training_jobs.py index 8e89509246..bd9a24c366 100644 --- a/google/cloud/aiplatform/training_jobs.py +++ b/google/cloud/aiplatform/training_jobs.py @@ -447,6 +447,7 @@ def _run_job( model: Optional[gca_model.Model] = None, gcs_destination_uri_prefix: Optional[str] = None, bigquery_destination: Optional[str] = None, + bq_uri: Optional[str] = None ) -> Optional[models.Model]: """Runs the training job. @@ -603,6 +604,9 @@ def _run_job( + self._model_upload_fail_string ) + if bq_uri is not None: + _LOGGER.info("Exported examples available at:\n%s" % bq_uri) + return model def _is_waiting_to_run(self) -> bool: @@ -626,7 +630,7 @@ def state(self) -> Optional[gca_pipeline_state.PipelineState]: self._sync_gca_resource() return self._gca_resource.state - + def get_model(self, sync=True) -> models.Model: """Vertex AI Model produced by this training, if one was produced. @@ -3529,6 +3533,7 @@ def _run( test_fraction_split=0.1, predefined_split_column_name=predefined_split_column_name, model=model, + bq_uri=self.bq_uri ) @property @@ -3539,6 +3544,20 @@ def _model_upload_fail_string(self) -> str: "Model." ) + @property + def bq_uri(self) -> Optional[str]: + """BigQuery location of exported evaluated examples from the Training Job""" + + if self._gca_resource is None: + return None + + meta = getattr((self._gca_resource), "training_task_metadata") + + if meta is not None and "evaluatedDataItemsBigqueryUri" in meta._pb: + return meta._pb["evaluatedDataItemsBigqueryUri"].string_value + else: + return None + def _add_additional_experiments(self, additional_experiments: List[str]): """Add experiment flags to the training job. Args: From b8995117d5174e4353dd0b5bb78d106082cbfa82 Mon Sep 17 00:00:00 2001 From: Mansi Achuthan Date: Mon, 16 Aug 2021 22:40:02 +0000 Subject: [PATCH 02/13] Testing BQ uri output feature for AutoML Forecasting --- google/cloud/aiplatform/training_jobs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/google/cloud/aiplatform/training_jobs.py b/google/cloud/aiplatform/training_jobs.py index bd9a24c366..e7ed0be673 100644 --- a/google/cloud/aiplatform/training_jobs.py +++ b/google/cloud/aiplatform/training_jobs.py @@ -3524,6 +3524,9 @@ def _run( model = gca_model.Model(display_name=model_display_name) + if self.bq_uri is not None: + print("BQ:", self.bq_uri) + return self._run_job( training_task_definition=training_task_definition, training_task_inputs=training_task_inputs_dict, From c225ac7f4b42f071db36fc6c6540b672aa9ae5d0 Mon Sep 17 00:00:00 2001 From: Mansi Achuthan Date: Mon, 16 Aug 2021 23:12:50 +0000 Subject: [PATCH 03/13] Testing BQ uri output feature for AutoML Forecasting --- google/cloud/aiplatform/training_jobs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/google/cloud/aiplatform/training_jobs.py b/google/cloud/aiplatform/training_jobs.py index e7ed0be673..8722bb8bf9 100644 --- a/google/cloud/aiplatform/training_jobs.py +++ b/google/cloud/aiplatform/training_jobs.py @@ -3524,8 +3524,7 @@ def _run( model = gca_model.Model(display_name=model_display_name) - if self.bq_uri is not None: - print("BQ:", self.bq_uri) + print("BQ:", self.bq_uri) return self._run_job( training_task_definition=training_task_definition, From 29e32bf0ca70cba82c6d5abe1504616b0a40100b Mon Sep 17 00:00:00 2001 From: Mansi Achuthan Date: Mon, 16 Aug 2021 23:17:51 +0000 Subject: [PATCH 04/13] Testing BQ uri output feature for AutoML Forecasting --- google/cloud/aiplatform/training_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/aiplatform/training_jobs.py b/google/cloud/aiplatform/training_jobs.py index 8722bb8bf9..f8a8764da9 100644 --- a/google/cloud/aiplatform/training_jobs.py +++ b/google/cloud/aiplatform/training_jobs.py @@ -3524,7 +3524,7 @@ def _run( model = gca_model.Model(display_name=model_display_name) - print("BQ:", self.bq_uri) + _LOGGER.info("Exported exmapled available at:\n%s" % self.bq_uri) return self._run_job( training_task_definition=training_task_definition, From 4dfb8b9b7cfeba06caf067cc5eabd26e6fd2f4a6 Mon Sep 17 00:00:00 2001 From: Mansi Achuthan Date: Tue, 17 Aug 2021 16:34:48 +0000 Subject: [PATCH 05/13] Testing BQ uri output feature for AutoML Forecasting --- google/cloud/aiplatform/training_jobs.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/google/cloud/aiplatform/training_jobs.py b/google/cloud/aiplatform/training_jobs.py index f8a8764da9..13fe633dbe 100644 --- a/google/cloud/aiplatform/training_jobs.py +++ b/google/cloud/aiplatform/training_jobs.py @@ -3524,9 +3524,10 @@ def _run( model = gca_model.Model(display_name=model_display_name) - _LOGGER.info("Exported exmapled available at:\n%s" % self.bq_uri) + _LOGGER.info("Exported examples available at:\n%s" % self.bq_uri) + print("self", self.bq_uri) - return self._run_job( + model = self._run_job( training_task_definition=training_task_definition, training_task_inputs=training_task_inputs_dict, dataset=dataset, @@ -3538,6 +3539,11 @@ def _run( bq_uri=self.bq_uri ) + _LOGGER.info("Exported examples available at:\n%s" % self.bq_uri) + print("self", self.bq_uri) + + return model + @property def _model_upload_fail_string(self) -> str: """Helper property for model upload failure.""" From bf869b6e5f0547f829a45f44d42269e153eabbab Mon Sep 17 00:00:00 2001 From: Mansi Achuthan Date: Tue, 17 Aug 2021 18:15:18 +0000 Subject: [PATCH 06/13] Testing BQ uri output feature for AutoML Forecasting --- google/cloud/aiplatform/training_jobs.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/google/cloud/aiplatform/training_jobs.py b/google/cloud/aiplatform/training_jobs.py index 13fe633dbe..5f8405bba4 100644 --- a/google/cloud/aiplatform/training_jobs.py +++ b/google/cloud/aiplatform/training_jobs.py @@ -446,8 +446,7 @@ def _run_job( predefined_split_column_name: Optional[str] = None, model: Optional[gca_model.Model] = None, gcs_destination_uri_prefix: Optional[str] = None, - bigquery_destination: Optional[str] = None, - bq_uri: Optional[str] = None + bigquery_destination: Optional[str] = None ) -> Optional[models.Model]: """Runs the training job. @@ -604,9 +603,6 @@ def _run_job( + self._model_upload_fail_string ) - if bq_uri is not None: - _LOGGER.info("Exported examples available at:\n%s" % bq_uri) - return model def _is_waiting_to_run(self) -> bool: @@ -3524,9 +3520,6 @@ def _run( model = gca_model.Model(display_name=model_display_name) - _LOGGER.info("Exported examples available at:\n%s" % self.bq_uri) - print("self", self.bq_uri) - model = self._run_job( training_task_definition=training_task_definition, training_task_inputs=training_task_inputs_dict, @@ -3535,12 +3528,11 @@ def _run( validation_fraction_split=0.1, test_fraction_split=0.1, predefined_split_column_name=predefined_split_column_name, - model=model, - bq_uri=self.bq_uri + model=model ) - _LOGGER.info("Exported examples available at:\n%s" % self.bq_uri) - print("self", self.bq_uri) + if self.bq_uri is not None: + _LOGGER.info("Exported examples available at:\n%s" % self.bq_uri) return model From fca235b08e5f3822cfe569e3b04053d8cbe7e6ff Mon Sep 17 00:00:00 2001 From: Mansi Achuthan Date: Tue, 17 Aug 2021 22:35:18 +0000 Subject: [PATCH 07/13] Testing BQ uri output feature for AutoML Forecasting --- google/cloud/aiplatform/training_jobs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google/cloud/aiplatform/training_jobs.py b/google/cloud/aiplatform/training_jobs.py index 5f8405bba4..a30c56510e 100644 --- a/google/cloud/aiplatform/training_jobs.py +++ b/google/cloud/aiplatform/training_jobs.py @@ -3520,7 +3520,7 @@ def _run( model = gca_model.Model(display_name=model_display_name) - model = self._run_job( + new_model = self._run_job( training_task_definition=training_task_definition, training_task_inputs=training_task_inputs_dict, dataset=dataset, @@ -3534,7 +3534,7 @@ def _run( if self.bq_uri is not None: _LOGGER.info("Exported examples available at:\n%s" % self.bq_uri) - return model + return new_model @property def _model_upload_fail_string(self) -> str: From e519e644fec47f8577e718c68982c83436a23579 Mon Sep 17 00:00:00 2001 From: Mansi Achuthan Date: Wed, 18 Aug 2021 16:47:25 +0000 Subject: [PATCH 08/13] Testing BQ uri output feature for AutoML Forecasting --- google/cloud/aiplatform/training_jobs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/google/cloud/aiplatform/training_jobs.py b/google/cloud/aiplatform/training_jobs.py index a30c56510e..82e7785e78 100644 --- a/google/cloud/aiplatform/training_jobs.py +++ b/google/cloud/aiplatform/training_jobs.py @@ -3531,8 +3531,8 @@ def _run( model=model ) - if self.bq_uri is not None: - _LOGGER.info("Exported examples available at:\n%s" % self.bq_uri) + if export_evaluated_data_items: + _LOGGER.info("Exported examples available at:\n%s" % self.evaluated_data_items_bigquery_uri) return new_model @@ -3545,7 +3545,7 @@ def _model_upload_fail_string(self) -> str: ) @property - def bq_uri(self) -> Optional[str]: + def evaluated_data_items_bigquery_uri(self) -> Optional[str]: """BigQuery location of exported evaluated examples from the Training Job""" if self._gca_resource is None: From ef03a2075efc5d7991ec37ece3261314bd2025e7 Mon Sep 17 00:00:00 2001 From: mansiachuthan <55572620+mansiachuthan@users.noreply.github.com> Date: Wed, 25 Aug 2021 12:41:42 -0700 Subject: [PATCH 09/13] Adding changes from dev to master branch --- google/cloud/aiplatform/training_jobs.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/google/cloud/aiplatform/training_jobs.py b/google/cloud/aiplatform/training_jobs.py index 682e8e9f81..9d60e2b124 100644 --- a/google/cloud/aiplatform/training_jobs.py +++ b/google/cloud/aiplatform/training_jobs.py @@ -4147,15 +4147,18 @@ def _model_upload_fail_string(self) -> str: def evaluated_data_items_bigquery_uri(self) -> Optional[str]: """BigQuery location of exported evaluated examples from the Training Job""" - if self._gca_resource is None: - return None - - meta = getattr((self._gca_resource), "training_task_metadata") - - if meta is not None and "evaluatedDataItemsBigqueryUri" in meta._pb: - return meta._pb["evaluatedDataItemsBigqueryUri"].string_value - else: - return None + self._assert_gca_resource_is_available() + + try: + meta = getattr((self._gca_resource), "training_task_metadata") + except ValueError: + raise ValueError("BigQuery uri for evaluated data items does not exist. Must export evaluated data items during training.") + + try: + metadata = self._gca_resource.training_task_metadata + return metadata["evaluatedDataItemsBigqueryUri"] + except (AttributeError, KeyError): + raise ValueError("BigQuery URI for evaluated data items does not exist. Must export evaluated data items during training.") def _add_additional_experiments(self, additional_experiments: List[str]): """Add experiment flags to the training job. From 31d06223f8408e59863284679727d07be0c58437 Mon Sep 17 00:00:00 2001 From: mansiachuthan <55572620+mansiachuthan@users.noreply.github.com> Date: Mon, 30 Aug 2021 16:53:02 -0700 Subject: [PATCH 10/13] Removed extra try/except block Removed the unnecessary try/except block from the evaluated bigquery uri property method in Automl Forecasting --- google/cloud/aiplatform/training_jobs.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/google/cloud/aiplatform/training_jobs.py b/google/cloud/aiplatform/training_jobs.py index 7611baedcb..3e64ccd52d 100644 --- a/google/cloud/aiplatform/training_jobs.py +++ b/google/cloud/aiplatform/training_jobs.py @@ -4148,11 +4148,8 @@ def evaluated_data_items_bigquery_uri(self) -> Optional[str]: """BigQuery location of exported evaluated examples from the Training Job""" self._assert_gca_resource_is_available() - - try: - meta = getattr((self._gca_resource), "training_task_metadata") - except ValueError: - raise ValueError("BigQuery uri for evaluated data items does not exist. Must export evaluated data items during training.") + + meta = getattr((self._gca_resource), "training_task_metadata") try: metadata = self._gca_resource.training_task_metadata From a8c10bb58a4412e04aeb395d994ecf092364feb4 Mon Sep 17 00:00:00 2001 From: Hardik Vala Date: Tue, 31 Aug 2021 19:03:26 -0400 Subject: [PATCH 11/13] Fix lint errors --- google/cloud/aiplatform/training_jobs.py | 27 +++++++++++++----------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/google/cloud/aiplatform/training_jobs.py b/google/cloud/aiplatform/training_jobs.py index 3e64ccd52d..79e7cfb84a 100644 --- a/google/cloud/aiplatform/training_jobs.py +++ b/google/cloud/aiplatform/training_jobs.py @@ -562,7 +562,7 @@ def _run_job( annotation_schema_uri: Optional[str] = None, model: Optional[gca_model.Model] = None, gcs_destination_uri_prefix: Optional[str] = None, - bigquery_destination: Optional[str] = None + bigquery_destination: Optional[str] = None, ) -> Optional[models.Model]: """Runs the training job. @@ -778,7 +778,7 @@ def state(self) -> Optional[gca_pipeline_state.PipelineState]: self._sync_gca_resource() return self._gca_resource.state - + def get_model(self, sync=True) -> models.Model: """Vertex AI Model produced by this training, if one was produced. @@ -4127,11 +4127,14 @@ def _run( validation_fraction_split=None, test_fraction_split=None, predefined_split_column_name=predefined_split_column_name, - model=model + model=model, ) if export_evaluated_data_items: - _LOGGER.info("Exported examples available at:\n%s" % self.evaluated_data_items_bigquery_uri) + _LOGGER.info( + "Exported examples available at:\n%s" + % self.evaluated_data_items_bigquery_uri + ) return new_model @@ -4148,14 +4151,14 @@ def evaluated_data_items_bigquery_uri(self) -> Optional[str]: """BigQuery location of exported evaluated examples from the Training Job""" self._assert_gca_resource_is_available() - - meta = getattr((self._gca_resource), "training_task_metadata") - - try: - metadata = self._gca_resource.training_task_metadata - return metadata["evaluatedDataItemsBigqueryUri"] - except (AttributeError, KeyError): - raise ValueError("BigQuery URI for evaluated data items does not exist. Must export evaluated data items during training.") + + try: + metadata = self._gca_resource.training_task_metadata + return metadata["evaluatedDataItemsBigqueryUri"] + except (AttributeError, KeyError): + raise ValueError( + "BigQuery URI for evaluated data items does not exist. Must export evaluated data items during training." + ) def _add_additional_experiments(self, additional_experiments: List[str]): """Add experiment flags to the training job. From dd3021667a9b7cfb30f7f95dba685f531e77611e Mon Sep 17 00:00:00 2001 From: Hardik Vala Date: Wed, 22 Sep 2021 18:29:54 -0700 Subject: [PATCH 12/13] Address comments --- google/cloud/aiplatform/training_jobs.py | 17 ++++++++++------- .../test_automl_forecasting_training_jobs.py | 3 +++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/google/cloud/aiplatform/training_jobs.py b/google/cloud/aiplatform/training_jobs.py index 79e7cfb84a..9712074c70 100644 --- a/google/cloud/aiplatform/training_jobs.py +++ b/google/cloud/aiplatform/training_jobs.py @@ -4148,17 +4148,20 @@ def _model_upload_fail_string(self) -> str: @property def evaluated_data_items_bigquery_uri(self) -> Optional[str]: - """BigQuery location of exported evaluated examples from the Training Job""" + """BigQuery location of exported evaluated examples from the Training Job + Returns: + str: BigQuery uri for the exported evaluated examples if the export + feature is enabled for training. + None: If the export feature was not enabled for training. + """ self._assert_gca_resource_is_available() - try: - metadata = self._gca_resource.training_task_metadata + metadata = self._gca_resource.training_task_metadata + if "evaluatedDataItemsBigqueryUri" in metadata: return metadata["evaluatedDataItemsBigqueryUri"] - except (AttributeError, KeyError): - raise ValueError( - "BigQuery URI for evaluated data items does not exist. Must export evaluated data items during training." - ) + + return None def _add_additional_experiments(self, additional_experiments: List[str]): """Add experiment flags to the training job. diff --git a/tests/unit/aiplatform/test_automl_forecasting_training_jobs.py b/tests/unit/aiplatform/test_automl_forecasting_training_jobs.py index 8dc1f362ba..88ceb45cc3 100644 --- a/tests/unit/aiplatform/test_automl_forecasting_training_jobs.py +++ b/tests/unit/aiplatform/test_automl_forecasting_training_jobs.py @@ -138,6 +138,9 @@ def mock_pipeline_service_get(): name=_TEST_PIPELINE_RESOURCE_NAME, state=gca_pipeline_state.PipelineState.PIPELINE_STATE_SUCCEEDED, model_to_upload=gca_model.Model(name=_TEST_MODEL_NAME), + training_task_metadata={ + "evaluatedDataItemsBigqueryUri": _TEST_TRAINING_EXPORT_EVALUATED_DATA_ITEMS_BIGQUERY_DESTINATION_URI + }, ) yield mock_get_training_pipeline From b1c3e790540fb338d708cc27feae28b403ed72d2 Mon Sep 17 00:00:00 2001 From: Hardik Vala Date: Mon, 4 Oct 2021 15:42:40 -0700 Subject: [PATCH 13/13] Add metadata existence check --- google/cloud/aiplatform/training_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/aiplatform/training_jobs.py b/google/cloud/aiplatform/training_jobs.py index 9712074c70..847aa6d8de 100644 --- a/google/cloud/aiplatform/training_jobs.py +++ b/google/cloud/aiplatform/training_jobs.py @@ -4158,7 +4158,7 @@ def evaluated_data_items_bigquery_uri(self) -> Optional[str]: self._assert_gca_resource_is_available() metadata = self._gca_resource.training_task_metadata - if "evaluatedDataItemsBigqueryUri" in metadata: + if metadata and "evaluatedDataItemsBigqueryUri" in metadata: return metadata["evaluatedDataItemsBigqueryUri"] return None