From d7450367ac77eccd10a157e7843a4e4f221352ff Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Tue, 8 Dec 2020 23:48:29 +0900 Subject: [PATCH 01/16] Added predict, get_model_evaluation and create_training_pipeline samples for AutoML Forecasting --- ...ing_pipeline_tabular_forecasting_sample.py | 10 +++++ ...l_evaluation_tabular_forecasting_sample.py | 5 +++ .../predict_tabular_forecasting_sample.py | 15 ++++++++ .sample_configs/process_configs.yaml | 3 ++ .sample_configs/variants.yaml | 3 ++ ...ing_pipeline_tabular_forecasting_sample.py | 36 ++++++++++++++++++ ...l_evaluation_tabular_forecasting_sample.py | 37 +++++++++++++++++++ .../predict_tabular_forecasting_sample.py | 36 ++++++++++++++++++ 8 files changed, 145 insertions(+) create mode 100644 .sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py create mode 100644 .sample_configs/param_handlers/get_model_evaluation_tabular_forecasting_sample.py create mode 100644 .sample_configs/param_handlers/predict_tabular_forecasting_sample.py create mode 100644 samples/snippets/create_training_pipeline_tabular_forecasting_sample.py create mode 100644 samples/snippets/get_model_evaluation_tabular_forecasting_sample.py create mode 100644 samples/snippets/predict_tabular_forecasting_sample.py diff --git a/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py b/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py new file mode 100644 index 0000000000..027c088ddb --- /dev/null +++ b/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py @@ -0,0 +1,10 @@ +def make_parent(parent: str) -> str: + parent = parent + + return parent + +def make_training_pipeline(training_pipeline: google.cloud.aiplatform_v1beta1.types.training_pipeline.TrainingPipeline) -> google.cloud.aiplatform_v1beta1.types.training_pipeline.TrainingPipeline: + training_pipeline = training_pipeline + + return training_pipeline + diff --git a/.sample_configs/param_handlers/get_model_evaluation_tabular_forecasting_sample.py b/.sample_configs/param_handlers/get_model_evaluation_tabular_forecasting_sample.py new file mode 100644 index 0000000000..d2637dd609 --- /dev/null +++ b/.sample_configs/param_handlers/get_model_evaluation_tabular_forecasting_sample.py @@ -0,0 +1,5 @@ +def make_name(name: str) -> str: + name = name + + return name + diff --git a/.sample_configs/param_handlers/predict_tabular_forecasting_sample.py b/.sample_configs/param_handlers/predict_tabular_forecasting_sample.py new file mode 100644 index 0000000000..cecaa43950 --- /dev/null +++ b/.sample_configs/param_handlers/predict_tabular_forecasting_sample.py @@ -0,0 +1,15 @@ +def make_endpoint(endpoint: str) -> str: + endpoint = endpoint + + return endpoint + +def make_instances(instances: typing.Sequence[google.protobuf.struct_pb2.Value]) -> typing.Sequence[google.protobuf.struct_pb2.Value]: + instances = instances + + return instances + +def make_parameters(parameters: google.protobuf.struct_pb2.Value) -> google.protobuf.struct_pb2.Value: + parameters = parameters + + return parameters + diff --git a/.sample_configs/process_configs.yaml b/.sample_configs/process_configs.yaml index 7eb80f400f..dd07761c54 100644 --- a/.sample_configs/process_configs.yaml +++ b/.sample_configs/process_configs.yaml @@ -73,6 +73,7 @@ create_training_pipeline_image_classification_sample: {} create_training_pipeline_image_object_detection_sample: {} create_training_pipeline_sample: {} create_training_pipeline_tabular_classification_sample: {} +create_training_pipeline_tabular_forecasting_sample: {} create_training_pipeline_tabular_regression_sample: {} create_training_pipeline_text_classification_sample: {} create_training_pipeline_text_entity_extraction_sample: @@ -182,6 +183,7 @@ get_model_evaluation_sample: - model_explanation get_model_evaluation_slice_sample: {} get_model_evaluation_tabular_classification_sample: {} +get_model_evaluation_tabular_forecasting_sample: {} get_model_evaluation_tabular_regression_sample: {} get_model_evaluation_text_classification_sample: skip: @@ -288,6 +290,7 @@ predict_tabular_classification_sample: comments: predictions: See gs://google-cloud-aiplatform/schema/predict/prediction/tables_classification.yaml for the format of the predictions. +predict_tabular_forecasting_sample: {} predict_tabular_regression_sample: api_endpoint: us-central1-prediction-aiplatform.googleapis.com max_depth: 1 diff --git a/.sample_configs/variants.yaml b/.sample_configs/variants.yaml index 59a0fd78f5..55917d160d 100644 --- a/.sample_configs/variants.yaml +++ b/.sample_configs/variants.yaml @@ -59,6 +59,7 @@ create_training_pipeline: - image_classification - image_object_detection - tabular_classification +- tabular_forecasting - tabular_regression - text_classification - text_entity_extraction @@ -131,6 +132,7 @@ get_model_evaluation: - image_classification - image_object_detection - tabular_classification +- tabular_forecasting - tabular_regression - text_classification - text_entity_extraction @@ -189,6 +191,7 @@ predict: - image_file - image_object_detection - tabular_classification +- tabular_forecasting - tabular_regression - text_classification_single_label - text_entity_extraction diff --git a/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py b/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py new file mode 100644 index 0000000000..043555816b --- /dev/null +++ b/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_training_pipeline_tabular_forecasting_sample] +from google.cloud import aiplatform + + +def create_training_pipeline_tabular_forecasting_sample( + project: str, + training_pipeline: google.cloud.aiplatform_v1beta1.types.training_pipeline.TrainingPipeline, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) + parent = f"projects/{project}/locations/{location}" + response = client.create_training_pipeline( + parent=parent, training_pipeline=training_pipeline + ) + print("response:", response) + + +# [END aiplatform_create_training_pipeline_tabular_forecasting_sample] diff --git a/samples/snippets/get_model_evaluation_tabular_forecasting_sample.py b/samples/snippets/get_model_evaluation_tabular_forecasting_sample.py new file mode 100644 index 0000000000..7d41c8a003 --- /dev/null +++ b/samples/snippets/get_model_evaluation_tabular_forecasting_sample.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_get_model_evaluation_tabular_forecasting_sample] +from google.cloud import aiplatform + + +def get_model_evaluation_tabular_forecasting_sample( + project: str, + model_id: str, + evaluation_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + name = client.model_evaluation_path( + project=project, location=location, model=model_id, evaluation=evaluation_id + ) + response = client.get_model_evaluation(name=name) + print("response:", response) + + +# [END aiplatform_get_model_evaluation_tabular_forecasting_sample] diff --git a/samples/snippets/predict_tabular_forecasting_sample.py b/samples/snippets/predict_tabular_forecasting_sample.py new file mode 100644 index 0000000000..cea88120b9 --- /dev/null +++ b/samples/snippets/predict_tabular_forecasting_sample.py @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_predict_tabular_forecasting_sample] +from google.cloud import aiplatform +from google.protobuf.struct_pb2 import Value + + +def predict_tabular_forecasting_sample( + endpoint: str, + instances: typing.Sequence[google.protobuf.struct_pb2.Value], + parameters: google.protobuf.struct_pb2.Value, + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PredictionServiceClient(client_options=client_options) + response = client.predict( + endpoint=endpoint, instances=instances, parameters=parameters + ) + print("response:", response) + + +# [END aiplatform_predict_tabular_forecasting_sample] From 0fee278297743a752b0df510bf1c6f1c3c3fe217 Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Wed, 9 Dec 2020 00:46:41 +0900 Subject: [PATCH 02/16] Added param handlers --- ...ing_pipeline_tabular_forecasting_sample.py | 51 ++++++++++++++++++- .../predict_tabular_forecasting_sample.py | 29 +++++++++-- ...ing_pipeline_tabular_forecasting_sample.py | 51 ++++++++++++++++++- .../predict_tabular_forecasting_sample.py | 12 ++++- 4 files changed, 133 insertions(+), 10 deletions(-) diff --git a/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py b/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py index 027c088ddb..e1837c7e2e 100644 --- a/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py +++ b/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py @@ -3,8 +3,55 @@ def make_parent(parent: str) -> str: return parent -def make_training_pipeline(training_pipeline: google.cloud.aiplatform_v1beta1.types.training_pipeline.TrainingPipeline) -> google.cloud.aiplatform_v1beta1.types.training_pipeline.TrainingPipeline: - training_pipeline = training_pipeline +def make_training_pipeline(display_name: str, dataset_id: str, model_display_name: str, target_column: str) -> google.cloud.aiplatform_v1alpha1.types.training_pipeline.TrainingPipeline: + # set the columns used for training and their data types + transformations = [ + {"auto": {"column_name": "sepal_width"}}, + {"auto": {"column_name": "sepal_length"}}, + {"auto": {"column_name": "petal_length"}}, + {"auto": {"column_name": "petal_width"}} + ] + + training_task_inputs_dict = { + # required inputs + "targetColumn": target_column, + "predictionType": "classification", + "transformations": transformations, + "trainBudgetMilliNodeHours": 8000, + + # optional inputs + "disableEarlyStopping": False, + # supported binary classification optimisation objectives: + # maximize-au-roc, minimize-log-loss, maximize-au-prc, + # maximize-precision-at-recall, maximize-recall-at-precision + # supported multi-class classification optimisation objective: + # minimize-log-loss + "optimizationObjective": "minimize-log-loss", + + # possibly required inputs + # required when using maximize-precision-at-recall + # "optimizationObjectiveRecallValue": 0.5, # 0.0 - 1.0 + # required when using maximize-recall-at-precision + # "optimizationObjectivePrecisionValue": 0.5, # 0.0 - 1.0 + } + training_task_inputs = to_protobuf_value(training_task_inputs_dict) + + training_pipeline = { + 'display_name': display_name, + 'training_task_definition': "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_tabular_1.0.0.yaml", + 'training_task_inputs': training_task_inputs, + 'input_data_config': { + 'dataset_id': dataset_id, + 'fraction_split': { + 'training_fraction': 0.8, + 'validation_fraction': 0.1, + 'test_fraction': 0.1, + } + }, + 'model_to_upload': { + 'display_name': model_display_name + } + } return training_pipeline diff --git a/.sample_configs/param_handlers/predict_tabular_forecasting_sample.py b/.sample_configs/param_handlers/predict_tabular_forecasting_sample.py index cecaa43950..a1d28ebc5c 100644 --- a/.sample_configs/param_handlers/predict_tabular_forecasting_sample.py +++ b/.sample_configs/param_handlers/predict_tabular_forecasting_sample.py @@ -1,15 +1,34 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + def make_endpoint(endpoint: str) -> str: endpoint = endpoint - + return endpoint -def make_instances(instances: typing.Sequence[google.protobuf.struct_pb2.Value]) -> typing.Sequence[google.protobuf.struct_pb2.Value]: - instances = instances +def make_instances(instance_dict: Dict) -> typing.Sequence[google.protobuf.struct_pb2.Value]: + # for more info on the instance schema, please use get_model_sample.py + # and look at the yaml found in instance_schema_uri + instance = to_protobuf_value(instance_dict) + instances = [instance] return instances -def make_parameters(parameters: google.protobuf.struct_pb2.Value) -> google.protobuf.struct_pb2.Value: - parameters = parameters +def make_parameters() -> google.protobuf.struct_pb2.Value: + parameters_dict = {} + parameters = to_protobuf_value(parameters_dict) return parameters diff --git a/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py b/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py index 043555816b..be10fba3a8 100644 --- a/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py +++ b/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py @@ -14,11 +14,16 @@ # [START aiplatform_create_training_pipeline_tabular_forecasting_sample] from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value def create_training_pipeline_tabular_forecasting_sample( project: str, - training_pipeline: google.cloud.aiplatform_v1beta1.types.training_pipeline.TrainingPipeline, + display_name: str, + dataset_id: str, + model_display_name: str, + target_column: str, location: str = "us-central1", api_endpoint: str = "us-central1-aiplatform.googleapis.com", ): @@ -26,6 +31,50 @@ def create_training_pipeline_tabular_forecasting_sample( # Initialize client that will be used to create and send requests. # This client only needs to be created once, and can be reused for multiple requests. client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) + # set the columns used for training and their data types + transformations = [ + {"auto": {"column_name": "sepal_width"}}, + {"auto": {"column_name": "sepal_length"}}, + {"auto": {"column_name": "petal_length"}}, + {"auto": {"column_name": "petal_width"}}, + ] + + training_task_inputs_dict = { + # required inputs + "targetColumn": target_column, + "predictionType": "classification", + "transformations": transformations, + "trainBudgetMilliNodeHours": 8000, + # optional inputs + "disableEarlyStopping": False, + # supported binary classification optimisation objectives: + # maximize-au-roc, minimize-log-loss, maximize-au-prc, + # maximize-precision-at-recall, maximize-recall-at-precision + # supported multi-class classification optimisation objective: + # minimize-log-loss + "optimizationObjective": "minimize-log-loss", + # possibly required inputs + # required when using maximize-precision-at-recall + # "optimizationObjectiveRecallValue": 0.5, # 0.0 - 1.0 + # required when using maximize-recall-at-precision + # "optimizationObjectivePrecisionValue": 0.5, # 0.0 - 1.0 + } + training_task_inputs = json_format.ParseDict(training_task_inputs_dict, Value()) + + training_pipeline = { + "display_name": display_name, + "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_tabular_1.0.0.yaml", + "training_task_inputs": training_task_inputs, + "input_data_config": { + "dataset_id": dataset_id, + "fraction_split": { + "training_fraction": 0.8, + "validation_fraction": 0.1, + "test_fraction": 0.1, + }, + }, + "model_to_upload": {"display_name": model_display_name}, + } parent = f"projects/{project}/locations/{location}" response = client.create_training_pipeline( parent=parent, training_pipeline=training_pipeline diff --git a/samples/snippets/predict_tabular_forecasting_sample.py b/samples/snippets/predict_tabular_forecasting_sample.py index cea88120b9..ff74a3ec6a 100644 --- a/samples/snippets/predict_tabular_forecasting_sample.py +++ b/samples/snippets/predict_tabular_forecasting_sample.py @@ -13,20 +13,28 @@ # limitations under the License. # [START aiplatform_predict_tabular_forecasting_sample] +from typing import Dict + from google.cloud import aiplatform +from google.protobuf import json_format from google.protobuf.struct_pb2 import Value def predict_tabular_forecasting_sample( endpoint: str, - instances: typing.Sequence[google.protobuf.struct_pb2.Value], - parameters: google.protobuf.struct_pb2.Value, + instance_dict: Dict, api_endpoint: str = "us-central1-aiplatform.googleapis.com", ): client_options = {"api_endpoint": api_endpoint} # Initialize client that will be used to create and send requests. # This client only needs to be created once, and can be reused for multiple requests. client = aiplatform.gapic.PredictionServiceClient(client_options=client_options) + # for more info on the instance schema, please use get_model_sample.py + # and look at the yaml found in instance_schema_uri + instance = json_format.ParseDict(instance_dict, Value()) + instances = [instance] + parameters_dict = {} + parameters = json_format.ParseDict(parameters_dict, Value()) response = client.predict( endpoint=endpoint, instances=instances, parameters=parameters ) From 57318ce10970f608e885953f97f2ec81e5fba627 Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Wed, 9 Dec 2020 01:02:11 +0900 Subject: [PATCH 03/16] Added headers manually --- ...raining_pipeline_tabular_forecasting_sample.py | 15 +++++++++++++++ ...model_evaluation_tabular_forecasting_sample.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py b/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py index e1837c7e2e..90c06bce1b 100644 --- a/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py +++ b/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py @@ -1,3 +1,18 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + def make_parent(parent: str) -> str: parent = parent diff --git a/.sample_configs/param_handlers/get_model_evaluation_tabular_forecasting_sample.py b/.sample_configs/param_handlers/get_model_evaluation_tabular_forecasting_sample.py index d2637dd609..79ffd458f7 100644 --- a/.sample_configs/param_handlers/get_model_evaluation_tabular_forecasting_sample.py +++ b/.sample_configs/param_handlers/get_model_evaluation_tabular_forecasting_sample.py @@ -1,3 +1,18 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + def make_name(name: str) -> str: name = name From a6ac96d40e74f15161e7993718774be552be51fe Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Fri, 11 Dec 2020 00:11:04 +0900 Subject: [PATCH 04/16] fix: Improved forecasting sample --- ...ing_pipeline_tabular_forecasting_sample.py | 62 ++++++++++++------- ...ing_pipeline_tabular_forecasting_sample.py | 54 ++++++++++------ 2 files changed, 74 insertions(+), 42 deletions(-) diff --git a/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py b/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py index 90c06bce1b..debc262f3a 100644 --- a/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py +++ b/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py @@ -18,42 +18,60 @@ def make_parent(parent: str) -> str: return parent -def make_training_pipeline(display_name: str, dataset_id: str, model_display_name: str, target_column: str) -> google.cloud.aiplatform_v1alpha1.types.training_pipeline.TrainingPipeline: +def make_training_pipeline( + display_name: str, + dataset_id: str, + model_display_name: str, + target_column: str, + time_series_identifier_column: str, + time_column: str, + static_columns: str, + time_variant_past_only_columns: str, + time_variant_past_and_future_columns: str, + forecast_window_end: int, + ) -> google.cloud.aiplatform_v1alpha1.types.training_pipeline.TrainingPipeline: # set the columns used for training and their data types transformations = [ - {"auto": {"column_name": "sepal_width"}}, - {"auto": {"column_name": "sepal_length"}}, - {"auto": {"column_name": "petal_length"}}, - {"auto": {"column_name": "petal_width"}} + {"auto": {"column_name": "date"}}, + {"auto": {"column_name": "state_name"}}, + {"auto": {"column_name": "county_fips_code"}}, + {"auto": {"column_name": "confirmed_cases"}}, + {"auto": {"column_name": "deaths"}} ] + period = {"unit": "day", "quantity": 1} + training_task_inputs_dict = { # required inputs "targetColumn": target_column, - "predictionType": "classification", + "timeSeriesIdentifierColumn": time_series_identifier_column, + "timeColumn": time_column, "transformations": transformations, - "trainBudgetMilliNodeHours": 8000, - - # optional inputs - "disableEarlyStopping": False, - # supported binary classification optimisation objectives: - # maximize-au-roc, minimize-log-loss, maximize-au-prc, - # maximize-precision-at-recall, maximize-recall-at-precision - # supported multi-class classification optimisation objective: - # minimize-log-loss - "optimizationObjective": "minimize-log-loss", + "period": period, - # possibly required inputs - # required when using maximize-precision-at-recall - # "optimizationObjectiveRecallValue": 0.5, # 0.0 - 1.0 - # required when using maximize-recall-at-precision - # "optimizationObjectivePrecisionValue": 0.5, # 0.0 - 1.0 + # Objective function the model is to be optimized towards. + # The training process creates a Model that optimizes the value of the objective + # function over the validation set. The supported optimization objectives: + # "minimize-rmse" (default) - Minimize root-mean-squared error (RMSE). + # "minimize-mae" - Minimize mean-absolute error (MAE). + # "minimize-rmsle" - Minimize root-mean-squared log error (RMSLE). + # "minimize-rmspe" - Minimize root-mean-squared percentage error (RMSPE). + # "minimize-wape-mae" - Minimize the combination of weighted absolute percentage error (WAPE) + # and mean-absolute-error (MAE). + # "minimize-quantile-loss" - Minimize the quantile loss at the defined quantiles. + "optimizationObjective": "minimize-rmse", + "budgetMilliNodeHours": 8000, + "staticColumns": static_columns, + "timeVariantPastOnlyColumns": time_variant_past_only_columns, + "timeVariantPastAndFutureColumns": time_variant_past_and_future_columns, + "forecastWindowEnd": forecast_window_end, } + training_task_inputs = to_protobuf_value(training_task_inputs_dict) training_pipeline = { 'display_name': display_name, - 'training_task_definition': "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_tabular_1.0.0.yaml", + 'training_task_definition': "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_time_series_forecasting_1.0.0.yaml", 'training_task_inputs': training_task_inputs, 'input_data_config': { 'dataset_id': dataset_id, diff --git a/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py b/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py index be10fba3a8..be8e1ee4e0 100644 --- a/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py +++ b/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py @@ -24,6 +24,12 @@ def create_training_pipeline_tabular_forecasting_sample( dataset_id: str, model_display_name: str, target_column: str, + time_series_identifier_column: str, + time_column: str, + static_columns: str, + time_variant_past_only_columns: str, + time_variant_past_and_future_columns: str, + forecast_window_end: int, location: str = "us-central1", api_endpoint: str = "us-central1-aiplatform.googleapis.com", ): @@ -33,37 +39,45 @@ def create_training_pipeline_tabular_forecasting_sample( client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) # set the columns used for training and their data types transformations = [ - {"auto": {"column_name": "sepal_width"}}, - {"auto": {"column_name": "sepal_length"}}, - {"auto": {"column_name": "petal_length"}}, - {"auto": {"column_name": "petal_width"}}, + {"auto": {"column_name": "date"}}, + {"auto": {"column_name": "state_name"}}, + {"auto": {"column_name": "county_fips_code"}}, + {"auto": {"column_name": "confirmed_cases"}}, + {"auto": {"column_name": "deaths"}}, ] + period = {"unit": "day", "quantity": 1} + training_task_inputs_dict = { # required inputs "targetColumn": target_column, - "predictionType": "classification", + "timeSeriesIdentifierColumn": time_series_identifier_column, + "timeColumn": time_column, "transformations": transformations, - "trainBudgetMilliNodeHours": 8000, - # optional inputs - "disableEarlyStopping": False, - # supported binary classification optimisation objectives: - # maximize-au-roc, minimize-log-loss, maximize-au-prc, - # maximize-precision-at-recall, maximize-recall-at-precision - # supported multi-class classification optimisation objective: - # minimize-log-loss - "optimizationObjective": "minimize-log-loss", - # possibly required inputs - # required when using maximize-precision-at-recall - # "optimizationObjectiveRecallValue": 0.5, # 0.0 - 1.0 - # required when using maximize-recall-at-precision - # "optimizationObjectivePrecisionValue": 0.5, # 0.0 - 1.0 + "period": period, + # Objective function the model is to be optimized towards. + # The training process creates a Model that optimizes the value of the objective + # function over the validation set. The supported optimization objectives: + # "minimize-rmse" (default) - Minimize root-mean-squared error (RMSE). + # "minimize-mae" - Minimize mean-absolute error (MAE). + # "minimize-rmsle" - Minimize root-mean-squared log error (RMSLE). + # "minimize-rmspe" - Minimize root-mean-squared percentage error (RMSPE). + # "minimize-wape-mae" - Minimize the combination of weighted absolute percentage error (WAPE) + # and mean-absolute-error (MAE). + # "minimize-quantile-loss" - Minimize the quantile loss at the defined quantiles. + "optimizationObjective": "minimize-rmse", + "budgetMilliNodeHours": 8000, + "staticColumns": static_columns, + "timeVariantPastOnlyColumns": time_variant_past_only_columns, + "timeVariantPastAndFutureColumns": time_variant_past_and_future_columns, + "forecastWindowEnd": forecast_window_end, } + training_task_inputs = json_format.ParseDict(training_task_inputs_dict, Value()) training_pipeline = { "display_name": display_name, - "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_tabular_1.0.0.yaml", + "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_time_series_forecasting_1.0.0.yaml", "training_task_inputs": training_task_inputs, "input_data_config": { "dataset_id": dataset_id, From 42fcc42dce45fb85cf5bdcd42518d412e0ff9f2c Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Fri, 11 Dec 2020 01:47:55 +0900 Subject: [PATCH 05/16] Added forecasting test --- ...ipeline_tabular_forecasting_sample_test.py | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 samples/snippets/create_training_pipeline_tabular_forecasting_sample_test.py diff --git a/samples/snippets/create_training_pipeline_tabular_forecasting_sample_test.py b/samples/snippets/create_training_pipeline_tabular_forecasting_sample_test.py new file mode 100644 index 0000000000..86db6849a3 --- /dev/null +++ b/samples/snippets/create_training_pipeline_tabular_forecasting_sample_test.py @@ -0,0 +1,87 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from uuid import uuid4 + +from google.cloud import aiplatform +import pytest + +import cancel_training_pipeline_sample +import create_training_pipeline_tabular_forecasting_sample +import delete_training_pipeline_sample +import helpers + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +DATASET_ID = "7172228697192136704" # COVID Dataset +DISPLAY_NAME = f"temp_create_training_pipeline_test_{uuid4()}" +TARGET_COLUMN = "deaths" +PREDICTION_TYPE = "forecasting" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + training_pipeline_id = shared_state["training_pipeline_name"].split("/")[-1] + + # Stop the training pipeline + cancel_training_pipeline_sample.cancel_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + client_options = {"api_endpoint": "us-central1-aiplatform.googleapis.com"} + pipeline_client = aiplatform.gapic.PipelineServiceClient( + client_options=client_options + ) + + # Waiting for training pipeline to be in CANCELLED state + helpers.wait_for_job_state( + get_job_method=pipeline_client.get_training_pipeline, + name=shared_state["training_pipeline_name"], + ) + + # Delete the training pipeline + delete_training_pipeline_sample.delete_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + +def test_ucaip_generated_create_training_pipeline_sample(capsys, shared_state): + + create_training_pipeline_tabular_forecasting_sample.create_training_pipeline_tabular_forecasting_sample( + project=PROJECT_ID, + display_name=DISPLAY_NAME, + dataset_id=DATASET_ID, + model_display_name=f"Temp Model for {DISPLAY_NAME}", + target_column=TARGET_COLUMN, + time_series_identifier_column="county", + time_column="date", + static_columns=["state_name"], + time_variant_past_only_columns=["deaths"], + time_variant_past_and_future_columns=[], + forecast_window_end=10 + ) + + out, _ = capsys.readouterr() + assert "response:" in out + + # Save resource name of the newly created training pipeline + shared_state["training_pipeline_name"] = helpers.get_name(out) From e75b88092d1ad1b136630ae6b7bc98fb46648738 Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Fri, 11 Dec 2020 20:52:06 +0900 Subject: [PATCH 06/16] Added tests for predict and get_model_evaluation --- ...luation_tabular_forecasting_sample_test.py | 30 +++++++++++++++ ...predict_tabular_forecasting_sample_test.py | 38 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 samples/snippets/get_model_evaluation_tabular_forecasting_sample_test.py create mode 100644 samples/snippets/predict_tabular_forecasting_sample_test.py diff --git a/samples/snippets/get_model_evaluation_tabular_forecasting_sample_test.py b/samples/snippets/get_model_evaluation_tabular_forecasting_sample_test.py new file mode 100644 index 0000000000..e1186431fc --- /dev/null +++ b/samples/snippets/get_model_evaluation_tabular_forecasting_sample_test.py @@ -0,0 +1,30 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +import get_model_evaluation_tabular_forecasting_sample + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +MODEL_ID = "TODO" # permanent_safe_driver_model +EVALUATION_ID = "TODO" # permanent_safe_driver_model Evaluation + + +def test_ucaip_generated_get_model_evaluation_tabular_forecasting_sample(capsys): + get_model_evaluation_tabular_forecasting_sample.get_model_evaluation_tabular_forecasting_sample( + project=PROJECT_ID, model_id=MODEL_ID, evaluation_id=EVALUATION_ID + ) + out, _ = capsys.readouterr() + assert "metrics_schema_uri" in out diff --git a/samples/snippets/predict_tabular_forecasting_sample_test.py b/samples/snippets/predict_tabular_forecasting_sample_test.py new file mode 100644 index 0000000000..4d6fe4801c --- /dev/null +++ b/samples/snippets/predict_tabular_forecasting_sample_test.py @@ -0,0 +1,38 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +import predict_tabular_classification_sample + +ENDPOINT_ID = "TODO" +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") + +INSTANCE = { + "petal_length": "1.4", + "petal_width": "1.3", + "sepal_length": "5.1", + "sepal_width": "2.8", +} + + +def test_ucaip_generated_predict_tabular_classification_sample(capsys): + + predict_tabular_classification_sample.predict_tabular_classification_sample( + instance_dict=INSTANCE, project=PROJECT_ID, endpoint_id=ENDPOINT_ID + ) + + out, _ = capsys.readouterr() + assert 'string_value: "setosa"' in out From 7e8f7ba99103337522a62c5443b283f0d479bf08 Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Tue, 15 Dec 2020 01:36:54 +0900 Subject: [PATCH 07/16] fix: Fixed create_training_pipeline_sample --- .../create_training_pipeline_tabular_forecasting_sample.py | 4 ++-- .../create_training_pipeline_tabular_forecasting_sample.py | 4 ++-- ...ate_training_pipeline_tabular_forecasting_sample_test.py | 6 +++--- samples/snippets/predict_tabular_forecasting_sample_test.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py b/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py index debc262f3a..460ea1a0e2 100644 --- a/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py +++ b/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py @@ -60,7 +60,7 @@ def make_training_pipeline( # and mean-absolute-error (MAE). # "minimize-quantile-loss" - Minimize the quantile loss at the defined quantiles. "optimizationObjective": "minimize-rmse", - "budgetMilliNodeHours": 8000, + "trainBudgetMilliNodeHours": 8000, "staticColumns": static_columns, "timeVariantPastOnlyColumns": time_variant_past_only_columns, "timeVariantPastAndFutureColumns": time_variant_past_and_future_columns, @@ -71,7 +71,7 @@ def make_training_pipeline( training_pipeline = { 'display_name': display_name, - 'training_task_definition': "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_time_series_forecasting_1.0.0.yaml", + 'training_task_definition': "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_forecasting_1.0.0.yaml", 'training_task_inputs': training_task_inputs, 'input_data_config': { 'dataset_id': dataset_id, diff --git a/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py b/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py index be8e1ee4e0..dddc6d9380 100644 --- a/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py +++ b/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py @@ -66,7 +66,7 @@ def create_training_pipeline_tabular_forecasting_sample( # and mean-absolute-error (MAE). # "minimize-quantile-loss" - Minimize the quantile loss at the defined quantiles. "optimizationObjective": "minimize-rmse", - "budgetMilliNodeHours": 8000, + "trainBudgetMilliNodeHours": 8000, "staticColumns": static_columns, "timeVariantPastOnlyColumns": time_variant_past_only_columns, "timeVariantPastAndFutureColumns": time_variant_past_and_future_columns, @@ -77,7 +77,7 @@ def create_training_pipeline_tabular_forecasting_sample( training_pipeline = { "display_name": display_name, - "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_time_series_forecasting_1.0.0.yaml", + "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_forecasting_1.0.0.yaml", "training_task_inputs": training_task_inputs, "input_data_config": { "dataset_id": dataset_id, diff --git a/samples/snippets/create_training_pipeline_tabular_forecasting_sample_test.py b/samples/snippets/create_training_pipeline_tabular_forecasting_sample_test.py index 86db6849a3..1bd0f3558e 100644 --- a/samples/snippets/create_training_pipeline_tabular_forecasting_sample_test.py +++ b/samples/snippets/create_training_pipeline_tabular_forecasting_sample_test.py @@ -70,14 +70,14 @@ def test_ucaip_generated_create_training_pipeline_sample(capsys, shared_state): project=PROJECT_ID, display_name=DISPLAY_NAME, dataset_id=DATASET_ID, - model_display_name=f"Temp Model for {DISPLAY_NAME}", + model_display_name=f"permanent_tabular_forecasting_model", target_column=TARGET_COLUMN, time_series_identifier_column="county", time_column="date", static_columns=["state_name"], time_variant_past_only_columns=["deaths"], - time_variant_past_and_future_columns=[], - forecast_window_end=10 + time_variant_past_and_future_columns=["date"], + forecast_window_end=10, ) out, _ = capsys.readouterr() diff --git a/samples/snippets/predict_tabular_forecasting_sample_test.py b/samples/snippets/predict_tabular_forecasting_sample_test.py index 4d6fe4801c..ad92844536 100644 --- a/samples/snippets/predict_tabular_forecasting_sample_test.py +++ b/samples/snippets/predict_tabular_forecasting_sample_test.py @@ -28,7 +28,7 @@ } -def test_ucaip_generated_predict_tabular_classification_sample(capsys): +def test_ucaip_generated_predict_tabular_forecasting_sample(capsys): predict_tabular_classification_sample.predict_tabular_classification_sample( instance_dict=INSTANCE, project=PROJECT_ID, endpoint_id=ENDPOINT_ID From 848ee55ee4a393adfce1b03d1917aa8ca8b091d7 Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Tue, 15 Dec 2020 15:56:32 +0900 Subject: [PATCH 08/16] feat: Added list_model_evaluations_tabular_forecasting_sample and test, fixed get_model_evaluation_tabular_forecasting_sample, and fixed create_training_pipeline_tabular_forecasting_sample --- ..._evaluations_tabular_forecasting_sample.py | 5 ++ .sample_configs/process_configs.yaml | 1 + .sample_configs/variants.yaml | 2 +- ...ipeline_tabular_forecasting_sample_test.py | 2 +- ...luation_tabular_forecasting_sample_test.py | 6 +-- ..._evaluations_tabular_forecasting_sample.py | 35 +++++++++++++ ...uations_tabular_forecasting_sample_test.py | 49 +++++++++++++++++++ 7 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 .sample_configs/param_handlers/list_model_evaluations_tabular_forecasting_sample.py create mode 100644 samples/snippets/list_model_evaluations_tabular_forecasting_sample.py create mode 100644 samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py diff --git a/.sample_configs/param_handlers/list_model_evaluations_tabular_forecasting_sample.py b/.sample_configs/param_handlers/list_model_evaluations_tabular_forecasting_sample.py new file mode 100644 index 0000000000..5e33a89bc0 --- /dev/null +++ b/.sample_configs/param_handlers/list_model_evaluations_tabular_forecasting_sample.py @@ -0,0 +1,5 @@ +def make_parent(parent: str) -> str: + parent = parent + + return parent + diff --git a/.sample_configs/process_configs.yaml b/.sample_configs/process_configs.yaml index dd07761c54..b33753d39a 100644 --- a/.sample_configs/process_configs.yaml +++ b/.sample_configs/process_configs.yaml @@ -248,6 +248,7 @@ list_endpoints_sample: {} list_hyperparameter_tuning_jobs_sample: {} list_model_evaluation_slices_sample: {} list_model_evaluations_sample: {} +list_model_evaluations_tabular_forecasting_sample: {} list_models_sample: {} list_specialist_pools_sample: {} list_training_pipelines_sample: {} diff --git a/.sample_configs/variants.yaml b/.sample_configs/variants.yaml index 55917d160d..457f9be7f7 100644 --- a/.sample_configs/variants.yaml +++ b/.sample_configs/variants.yaml @@ -177,7 +177,7 @@ list_hyperparameter_tuning_jobs: list_model_evaluation_slices: - '' list_model_evaluations: -- '' +- tabular_forecasting list_models: - '' list_specialist_pools: diff --git a/samples/snippets/create_training_pipeline_tabular_forecasting_sample_test.py b/samples/snippets/create_training_pipeline_tabular_forecasting_sample_test.py index 1bd0f3558e..4599c2f539 100644 --- a/samples/snippets/create_training_pipeline_tabular_forecasting_sample_test.py +++ b/samples/snippets/create_training_pipeline_tabular_forecasting_sample_test.py @@ -24,7 +24,7 @@ import helpers PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") -DATASET_ID = "7172228697192136704" # COVID Dataset +DATASET_ID = "3003302817130610688" # COVID Dataset DISPLAY_NAME = f"temp_create_training_pipeline_test_{uuid4()}" TARGET_COLUMN = "deaths" PREDICTION_TYPE = "forecasting" diff --git a/samples/snippets/get_model_evaluation_tabular_forecasting_sample_test.py b/samples/snippets/get_model_evaluation_tabular_forecasting_sample_test.py index e1186431fc..f40a713feb 100644 --- a/samples/snippets/get_model_evaluation_tabular_forecasting_sample_test.py +++ b/samples/snippets/get_model_evaluation_tabular_forecasting_sample_test.py @@ -17,9 +17,9 @@ import get_model_evaluation_tabular_forecasting_sample -PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") -MODEL_ID = "TODO" # permanent_safe_driver_model -EVALUATION_ID = "TODO" # permanent_safe_driver_model Evaluation +PROJECT_ID = "ucaip-sample-tests" # os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +MODEL_ID = "8531330622239539200" # COVID model +EVALUATION_ID = "5651454056920760520" def test_ucaip_generated_get_model_evaluation_tabular_forecasting_sample(capsys): diff --git a/samples/snippets/list_model_evaluations_tabular_forecasting_sample.py b/samples/snippets/list_model_evaluations_tabular_forecasting_sample.py new file mode 100644 index 0000000000..4d949d227f --- /dev/null +++ b/samples/snippets/list_model_evaluations_tabular_forecasting_sample.py @@ -0,0 +1,35 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_list_model_evaluations_tabular_forecasting_sample] +from google.cloud import aiplatform + + +def list_model_evaluations_tabular_forecasting_sample( + project: str, + model_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + parent = client.model_path(project=project, location=location, model=model_id) + response = client.list_model_evaluations(parent=parent) + for model_evaluation in response: + print("model_evaluation:", model_evaluation) + + +# [END aiplatform_list_model_evaluations_tabular_forecasting_sample] diff --git a/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py b/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py new file mode 100644 index 0000000000..6738116da1 --- /dev/null +++ b/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py @@ -0,0 +1,49 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from uuid import uuid4 + +from google.cloud import aiplatform +import pytest + +import list_model_evaluations_tabular_forecasting_sample +import helpers + +PROJECT_ID = "ucaip-sample-tests" # TODO: Revert this to os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +MODEL_ID = "8531330622239539200" # COVID Dataset + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + +def test_ucaip_generated_create_training_pipeline_sample(capsys, shared_state): + + list_model_evaluations_tabular_forecasting_sample.list_model_evaluations_tabular_forecasting_sample( + project=PROJECT_ID, model_id=MODEL_ID + ) + + out, _ = capsys.readouterr() + assert "response:" in out + + # Save resource name of the newly created training pipeline + shared_state["training_pipeline_name"] = helpers.get_name(out) From 669a664ef3701964a57b74c80b3354c66ff3c5eb Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Tue, 15 Dec 2020 17:03:20 +0900 Subject: [PATCH 09/16] fix: Reverted back to generated BUILD_SPECIFIC_GCLOUD_PROJECT --- .../get_model_evaluation_tabular_forecasting_sample_test.py | 2 +- .../list_model_evaluations_tabular_forecasting_sample_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/get_model_evaluation_tabular_forecasting_sample_test.py b/samples/snippets/get_model_evaluation_tabular_forecasting_sample_test.py index f40a713feb..705aee3ed3 100644 --- a/samples/snippets/get_model_evaluation_tabular_forecasting_sample_test.py +++ b/samples/snippets/get_model_evaluation_tabular_forecasting_sample_test.py @@ -17,7 +17,7 @@ import get_model_evaluation_tabular_forecasting_sample -PROJECT_ID = "ucaip-sample-tests" # os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") MODEL_ID = "8531330622239539200" # COVID model EVALUATION_ID = "5651454056920760520" diff --git a/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py b/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py index 6738116da1..cecbbafb6e 100644 --- a/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py +++ b/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py @@ -21,7 +21,7 @@ import list_model_evaluations_tabular_forecasting_sample import helpers -PROJECT_ID = "ucaip-sample-tests" # TODO: Revert this to os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") MODEL_ID = "8531330622239539200" # COVID Dataset From a779c4f4dd9e03682765bfe1d21a84d95ad78fcf Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Tue, 15 Dec 2020 18:33:29 +0900 Subject: [PATCH 10/16] fix: Fixed name of test --- .../list_model_evaluations_tabular_forecasting_sample_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py b/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py index cecbbafb6e..d18d9e88d6 100644 --- a/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py +++ b/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py @@ -36,7 +36,9 @@ def teardown(shared_state): yield -def test_ucaip_generated_create_training_pipeline_sample(capsys, shared_state): +def test_ucaip_generated_list_model_evaluations_tabular_forecasting_sample( + capsys, shared_state +): list_model_evaluations_tabular_forecasting_sample.list_model_evaluations_tabular_forecasting_sample( project=PROJECT_ID, model_id=MODEL_ID From b08ac1e24e6be6131f25cc2d8148ebce89f5b1f2 Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Tue, 15 Dec 2020 23:23:16 +0900 Subject: [PATCH 11/16] fix: Fixed lint errors --- ...reate_training_pipeline_tabular_forecasting_sample_test.py | 2 +- .../list_model_evaluations_tabular_forecasting_sample_test.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/samples/snippets/create_training_pipeline_tabular_forecasting_sample_test.py b/samples/snippets/create_training_pipeline_tabular_forecasting_sample_test.py index 4599c2f539..d5c58a7992 100644 --- a/samples/snippets/create_training_pipeline_tabular_forecasting_sample_test.py +++ b/samples/snippets/create_training_pipeline_tabular_forecasting_sample_test.py @@ -70,7 +70,7 @@ def test_ucaip_generated_create_training_pipeline_sample(capsys, shared_state): project=PROJECT_ID, display_name=DISPLAY_NAME, dataset_id=DATASET_ID, - model_display_name=f"permanent_tabular_forecasting_model", + model_display_name="permanent_tabular_forecasting_model", target_column=TARGET_COLUMN, time_series_identifier_column="county", time_column="date", diff --git a/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py b/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py index d18d9e88d6..3f06343e76 100644 --- a/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py +++ b/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py @@ -13,13 +13,11 @@ # limitations under the License. import os -from uuid import uuid4 -from google.cloud import aiplatform import pytest -import list_model_evaluations_tabular_forecasting_sample import helpers +import list_model_evaluations_tabular_forecasting_sample PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") MODEL_ID = "8531330622239539200" # COVID Dataset From fd87673c1a884052c1405b91ae6e30adbaf0f745 Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Wed, 16 Dec 2020 00:37:59 +0900 Subject: [PATCH 12/16] fix: Fixed assertion --- .../list_model_evaluations_tabular_forecasting_sample_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py b/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py index 3f06343e76..4fc3afbebc 100644 --- a/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py +++ b/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py @@ -43,7 +43,7 @@ def test_ucaip_generated_list_model_evaluations_tabular_forecasting_sample( ) out, _ = capsys.readouterr() - assert "response:" in out + assert "model_evaluation:" in out # Save resource name of the newly created training pipeline shared_state["training_pipeline_name"] = helpers.get_name(out) From 2fc3f903e922e8a36f4475e9f38e3f5fcaa290bb Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Wed, 16 Dec 2020 05:31:14 +0900 Subject: [PATCH 13/16] fix: Removed predict samples --- .../predict_tabular_forecasting_sample.py | 34 -------------- .sample_configs/variants.yaml | 1 - .../predict_tabular_forecasting_sample.py | 44 ------------------- ...predict_tabular_forecasting_sample_test.py | 38 ---------------- 4 files changed, 117 deletions(-) delete mode 100644 .sample_configs/param_handlers/predict_tabular_forecasting_sample.py delete mode 100644 samples/snippets/predict_tabular_forecasting_sample.py delete mode 100644 samples/snippets/predict_tabular_forecasting_sample_test.py diff --git a/.sample_configs/param_handlers/predict_tabular_forecasting_sample.py b/.sample_configs/param_handlers/predict_tabular_forecasting_sample.py deleted file mode 100644 index a1d28ebc5c..0000000000 --- a/.sample_configs/param_handlers/predict_tabular_forecasting_sample.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -def make_endpoint(endpoint: str) -> str: - endpoint = endpoint - - return endpoint - -def make_instances(instance_dict: Dict) -> typing.Sequence[google.protobuf.struct_pb2.Value]: - # for more info on the instance schema, please use get_model_sample.py - # and look at the yaml found in instance_schema_uri - instance = to_protobuf_value(instance_dict) - instances = [instance] - - return instances - -def make_parameters() -> google.protobuf.struct_pb2.Value: - parameters_dict = {} - parameters = to_protobuf_value(parameters_dict) - - return parameters - diff --git a/.sample_configs/variants.yaml b/.sample_configs/variants.yaml index 457f9be7f7..8d57d05206 100644 --- a/.sample_configs/variants.yaml +++ b/.sample_configs/variants.yaml @@ -191,7 +191,6 @@ predict: - image_file - image_object_detection - tabular_classification -- tabular_forecasting - tabular_regression - text_classification_single_label - text_entity_extraction diff --git a/samples/snippets/predict_tabular_forecasting_sample.py b/samples/snippets/predict_tabular_forecasting_sample.py deleted file mode 100644 index ff74a3ec6a..0000000000 --- a/samples/snippets/predict_tabular_forecasting_sample.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START aiplatform_predict_tabular_forecasting_sample] -from typing import Dict - -from google.cloud import aiplatform -from google.protobuf import json_format -from google.protobuf.struct_pb2 import Value - - -def predict_tabular_forecasting_sample( - endpoint: str, - instance_dict: Dict, - api_endpoint: str = "us-central1-aiplatform.googleapis.com", -): - client_options = {"api_endpoint": api_endpoint} - # Initialize client that will be used to create and send requests. - # This client only needs to be created once, and can be reused for multiple requests. - client = aiplatform.gapic.PredictionServiceClient(client_options=client_options) - # for more info on the instance schema, please use get_model_sample.py - # and look at the yaml found in instance_schema_uri - instance = json_format.ParseDict(instance_dict, Value()) - instances = [instance] - parameters_dict = {} - parameters = json_format.ParseDict(parameters_dict, Value()) - response = client.predict( - endpoint=endpoint, instances=instances, parameters=parameters - ) - print("response:", response) - - -# [END aiplatform_predict_tabular_forecasting_sample] diff --git a/samples/snippets/predict_tabular_forecasting_sample_test.py b/samples/snippets/predict_tabular_forecasting_sample_test.py deleted file mode 100644 index ad92844536..0000000000 --- a/samples/snippets/predict_tabular_forecasting_sample_test.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - - -import predict_tabular_classification_sample - -ENDPOINT_ID = "TODO" -PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") - -INSTANCE = { - "petal_length": "1.4", - "petal_width": "1.3", - "sepal_length": "5.1", - "sepal_width": "2.8", -} - - -def test_ucaip_generated_predict_tabular_forecasting_sample(capsys): - - predict_tabular_classification_sample.predict_tabular_classification_sample( - instance_dict=INSTANCE, project=PROJECT_ID, endpoint_id=ENDPOINT_ID - ) - - out, _ = capsys.readouterr() - assert 'string_value: "setosa"' in out From 087ff3928a1ade7fb631d7433f44b2eb204f1895 Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Tue, 22 Dec 2020 00:43:56 +0900 Subject: [PATCH 14/16] Consolidated samples --- ...ing_pipeline_tabular_forecasting_sample.py | 62 ++++++++----------- ..._evaluations_tabular_forecasting_sample.py | 5 -- .sample_configs/process_configs.yaml | 1 + .sample_configs/variants.yaml | 4 +- ...l_evaluation_tabular_forecasting_sample.py | 37 ----------- ...luation_tabular_forecasting_sample_test.py | 30 --------- 6 files changed, 29 insertions(+), 110 deletions(-) delete mode 100644 .sample_configs/param_handlers/list_model_evaluations_tabular_forecasting_sample.py delete mode 100644 samples/snippets/get_model_evaluation_tabular_forecasting_sample.py delete mode 100644 samples/snippets/get_model_evaluation_tabular_forecasting_sample_test.py diff --git a/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py b/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py index 460ea1a0e2..5c0f63a781 100644 --- a/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py +++ b/.sample_configs/param_handlers/create_training_pipeline_tabular_forecasting_sample.py @@ -13,34 +13,37 @@ # limitations under the License. # + def make_parent(parent: str) -> str: parent = parent return parent + def make_training_pipeline( - display_name: str, - dataset_id: str, - model_display_name: str, - target_column: str, - time_series_identifier_column: str, - time_column: str, - static_columns: str, - time_variant_past_only_columns: str, - time_variant_past_and_future_columns: str, - forecast_window_end: int, - ) -> google.cloud.aiplatform_v1alpha1.types.training_pipeline.TrainingPipeline: + display_name: str, + dataset_id: str, + model_display_name: str, + target_column: str, + time_series_identifier_column: str, + time_column: str, + static_columns: str, + time_variant_past_only_columns: str, + time_variant_past_and_future_columns: str, + forecast_window_end: int, +) -> google.cloud.aiplatform_v1alpha1.types.training_pipeline.TrainingPipeline: # set the columns used for training and their data types transformations = [ {"auto": {"column_name": "date"}}, {"auto": {"column_name": "state_name"}}, {"auto": {"column_name": "county_fips_code"}}, {"auto": {"column_name": "confirmed_cases"}}, - {"auto": {"column_name": "deaths"}} + {"auto": {"column_name": "deaths"}}, ] period = {"unit": "day", "quantity": 1} + # the inputs should be formatted according to the training_task_definition yaml file training_task_inputs_dict = { # required inputs "targetColumn": target_column, @@ -48,17 +51,6 @@ def make_training_pipeline( "timeColumn": time_column, "transformations": transformations, "period": period, - - # Objective function the model is to be optimized towards. - # The training process creates a Model that optimizes the value of the objective - # function over the validation set. The supported optimization objectives: - # "minimize-rmse" (default) - Minimize root-mean-squared error (RMSE). - # "minimize-mae" - Minimize mean-absolute error (MAE). - # "minimize-rmsle" - Minimize root-mean-squared log error (RMSLE). - # "minimize-rmspe" - Minimize root-mean-squared percentage error (RMSPE). - # "minimize-wape-mae" - Minimize the combination of weighted absolute percentage error (WAPE) - # and mean-absolute-error (MAE). - # "minimize-quantile-loss" - Minimize the quantile loss at the defined quantiles. "optimizationObjective": "minimize-rmse", "trainBudgetMilliNodeHours": 8000, "staticColumns": static_columns, @@ -70,20 +62,18 @@ def make_training_pipeline( training_task_inputs = to_protobuf_value(training_task_inputs_dict) training_pipeline = { - 'display_name': display_name, - 'training_task_definition': "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_forecasting_1.0.0.yaml", - 'training_task_inputs': training_task_inputs, - 'input_data_config': { - 'dataset_id': dataset_id, - 'fraction_split': { - 'training_fraction': 0.8, - 'validation_fraction': 0.1, - 'test_fraction': 0.1, - } + "display_name": display_name, + "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_forecasting_1.0.0.yaml", + "training_task_inputs": training_task_inputs, + "input_data_config": { + "dataset_id": dataset_id, + "fraction_split": { + "training_fraction": 0.8, + "validation_fraction": 0.1, + "test_fraction": 0.1, + }, }, - 'model_to_upload': { - 'display_name': model_display_name - } + "model_to_upload": {"display_name": model_display_name}, } return training_pipeline diff --git a/.sample_configs/param_handlers/list_model_evaluations_tabular_forecasting_sample.py b/.sample_configs/param_handlers/list_model_evaluations_tabular_forecasting_sample.py deleted file mode 100644 index 5e33a89bc0..0000000000 --- a/.sample_configs/param_handlers/list_model_evaluations_tabular_forecasting_sample.py +++ /dev/null @@ -1,5 +0,0 @@ -def make_parent(parent: str) -> str: - parent = parent - - return parent - diff --git a/.sample_configs/process_configs.yaml b/.sample_configs/process_configs.yaml index b33753d39a..8ba791d311 100644 --- a/.sample_configs/process_configs.yaml +++ b/.sample_configs/process_configs.yaml @@ -18,6 +18,7 @@ create_batch_prediction_job_custom_image_explain_sample: {} create_batch_prediction_job_custom_tabular_explain_sample: {} create_batch_prediction_job_sample: {} create_batch_prediction_job_tabular_explain_sample: {} +create_batch_prediction_job_tabular_forecasting_sample: {} create_batch_prediction_job_text_classification_sample: {} create_batch_prediction_job_text_entity_extraction_sample: {} create_batch_prediction_job_text_sentiment_analysis_sample: {} diff --git a/.sample_configs/variants.yaml b/.sample_configs/variants.yaml index 8d57d05206..0ef9cf7bc7 100644 --- a/.sample_configs/variants.yaml +++ b/.sample_configs/variants.yaml @@ -22,6 +22,7 @@ create_batch_prediction_job: - custom_image_explain - custom_tabular_explain - tabular_explain +- tabular_forecasting - text_classification - text_entity_extraction - text_sentiment_analysis @@ -132,7 +133,6 @@ get_model_evaluation: - image_classification - image_object_detection - tabular_classification -- tabular_forecasting - tabular_regression - text_classification - text_entity_extraction @@ -177,7 +177,7 @@ list_hyperparameter_tuning_jobs: list_model_evaluation_slices: - '' list_model_evaluations: -- tabular_forecasting +- '' list_models: - '' list_specialist_pools: diff --git a/samples/snippets/get_model_evaluation_tabular_forecasting_sample.py b/samples/snippets/get_model_evaluation_tabular_forecasting_sample.py deleted file mode 100644 index 7d41c8a003..0000000000 --- a/samples/snippets/get_model_evaluation_tabular_forecasting_sample.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START aiplatform_get_model_evaluation_tabular_forecasting_sample] -from google.cloud import aiplatform - - -def get_model_evaluation_tabular_forecasting_sample( - project: str, - model_id: str, - evaluation_id: str, - location: str = "us-central1", - api_endpoint: str = "us-central1-aiplatform.googleapis.com", -): - client_options = {"api_endpoint": api_endpoint} - # Initialize client that will be used to create and send requests. - # This client only needs to be created once, and can be reused for multiple requests. - client = aiplatform.gapic.ModelServiceClient(client_options=client_options) - name = client.model_evaluation_path( - project=project, location=location, model=model_id, evaluation=evaluation_id - ) - response = client.get_model_evaluation(name=name) - print("response:", response) - - -# [END aiplatform_get_model_evaluation_tabular_forecasting_sample] diff --git a/samples/snippets/get_model_evaluation_tabular_forecasting_sample_test.py b/samples/snippets/get_model_evaluation_tabular_forecasting_sample_test.py deleted file mode 100644 index 705aee3ed3..0000000000 --- a/samples/snippets/get_model_evaluation_tabular_forecasting_sample_test.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - - -import get_model_evaluation_tabular_forecasting_sample - -PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") -MODEL_ID = "8531330622239539200" # COVID model -EVALUATION_ID = "5651454056920760520" - - -def test_ucaip_generated_get_model_evaluation_tabular_forecasting_sample(capsys): - get_model_evaluation_tabular_forecasting_sample.get_model_evaluation_tabular_forecasting_sample( - project=PROJECT_ID, model_id=MODEL_ID, evaluation_id=EVALUATION_ID - ) - out, _ = capsys.readouterr() - assert "metrics_schema_uri" in out From ddd443eb48dc9837ee5922419eea69d54a9924c0 Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Tue, 22 Dec 2020 05:27:36 +0900 Subject: [PATCH 15/16] fix: Removed list_model_evaluations_tabular_forecasting --- ..._evaluations_tabular_forecasting_sample.py | 35 ------------- ...uations_tabular_forecasting_sample_test.py | 49 ------------------- 2 files changed, 84 deletions(-) delete mode 100644 samples/snippets/list_model_evaluations_tabular_forecasting_sample.py delete mode 100644 samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py diff --git a/samples/snippets/list_model_evaluations_tabular_forecasting_sample.py b/samples/snippets/list_model_evaluations_tabular_forecasting_sample.py deleted file mode 100644 index 4d949d227f..0000000000 --- a/samples/snippets/list_model_evaluations_tabular_forecasting_sample.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START aiplatform_list_model_evaluations_tabular_forecasting_sample] -from google.cloud import aiplatform - - -def list_model_evaluations_tabular_forecasting_sample( - project: str, - model_id: str, - location: str = "us-central1", - api_endpoint: str = "us-central1-aiplatform.googleapis.com", -): - client_options = {"api_endpoint": api_endpoint} - # Initialize client that will be used to create and send requests. - # This client only needs to be created once, and can be reused for multiple requests. - client = aiplatform.gapic.ModelServiceClient(client_options=client_options) - parent = client.model_path(project=project, location=location, model=model_id) - response = client.list_model_evaluations(parent=parent) - for model_evaluation in response: - print("model_evaluation:", model_evaluation) - - -# [END aiplatform_list_model_evaluations_tabular_forecasting_sample] diff --git a/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py b/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py deleted file mode 100644 index 4fc3afbebc..0000000000 --- a/samples/snippets/list_model_evaluations_tabular_forecasting_sample_test.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import pytest - -import helpers -import list_model_evaluations_tabular_forecasting_sample - -PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") -MODEL_ID = "8531330622239539200" # COVID Dataset - - -@pytest.fixture -def shared_state(): - state = {} - yield state - - -@pytest.fixture(scope="function", autouse=True) -def teardown(shared_state): - yield - - -def test_ucaip_generated_list_model_evaluations_tabular_forecasting_sample( - capsys, shared_state -): - - list_model_evaluations_tabular_forecasting_sample.list_model_evaluations_tabular_forecasting_sample( - project=PROJECT_ID, model_id=MODEL_ID - ) - - out, _ = capsys.readouterr() - assert "model_evaluation:" in out - - # Save resource name of the newly created training pipeline - shared_state["training_pipeline_name"] = helpers.get_name(out) From f12db5b5ad92f20f97901edc448d4fa4ef2e7706 Mon Sep 17 00:00:00 2001 From: Ivan Cheung Date: Tue, 22 Dec 2020 05:36:04 +0900 Subject: [PATCH 16/16] fix: tweaks --- ...l_evaluation_tabular_forecasting_sample.py | 20 ------------------- ...ing_pipeline_tabular_forecasting_sample.py | 11 +--------- 2 files changed, 1 insertion(+), 30 deletions(-) delete mode 100644 .sample_configs/param_handlers/get_model_evaluation_tabular_forecasting_sample.py diff --git a/.sample_configs/param_handlers/get_model_evaluation_tabular_forecasting_sample.py b/.sample_configs/param_handlers/get_model_evaluation_tabular_forecasting_sample.py deleted file mode 100644 index 79ffd458f7..0000000000 --- a/.sample_configs/param_handlers/get_model_evaluation_tabular_forecasting_sample.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -def make_name(name: str) -> str: - name = name - - return name - diff --git a/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py b/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py index dddc6d9380..5c04fccc79 100644 --- a/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py +++ b/samples/snippets/create_training_pipeline_tabular_forecasting_sample.py @@ -48,6 +48,7 @@ def create_training_pipeline_tabular_forecasting_sample( period = {"unit": "day", "quantity": 1} + # the inputs should be formatted according to the training_task_definition yaml file training_task_inputs_dict = { # required inputs "targetColumn": target_column, @@ -55,16 +56,6 @@ def create_training_pipeline_tabular_forecasting_sample( "timeColumn": time_column, "transformations": transformations, "period": period, - # Objective function the model is to be optimized towards. - # The training process creates a Model that optimizes the value of the objective - # function over the validation set. The supported optimization objectives: - # "minimize-rmse" (default) - Minimize root-mean-squared error (RMSE). - # "minimize-mae" - Minimize mean-absolute error (MAE). - # "minimize-rmsle" - Minimize root-mean-squared log error (RMSLE). - # "minimize-rmspe" - Minimize root-mean-squared percentage error (RMSPE). - # "minimize-wape-mae" - Minimize the combination of weighted absolute percentage error (WAPE) - # and mean-absolute-error (MAE). - # "minimize-quantile-loss" - Minimize the quantile loss at the defined quantiles. "optimizationObjective": "minimize-rmse", "trainBudgetMilliNodeHours": 8000, "staticColumns": static_columns,