From eac907b890f744e4015f87984dfb315ff4a8bdc1 Mon Sep 17 00:00:00 2001 From: Vinny Senthil Date: Tue, 14 Sep 2021 14:57:42 -0700 Subject: [PATCH 1/3] Add tests to check XAI metadata builder and Model.upload compatibility --- ...n_saved_model_metadata_builder_tf1_test.py | 30 ++++++++++++++++++ ...n_saved_model_metadata_builder_tf2_test.py | 31 ++++++++++++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf1_test.py b/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf1_test.py index 41ff8bb68e..a4912185ab 100644 --- a/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf1_test.py +++ b/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf1_test.py @@ -15,13 +15,18 @@ # limitations under the License. # +import pytest import tensorflow.compat.v1 as tf +from google.cloud.aiplatform import models from google.cloud.aiplatform.explain.metadata.tf.v1 import saved_model_metadata_builder from google.cloud.aiplatform.compat.types import ( explanation_metadata_v1beta1 as explanation_metadata, ) +import test_models +from test_models import upload_model_mock, get_model_mock # noqa: F401 + class SavedModelMetadataBuilderTF1Test(tf.test.TestCase): def _set_up(self): @@ -108,3 +113,28 @@ def test_get_metadata_protobuf_double_output(self): ) assert md_builder.get_metadata_protobuf() == expected_object + + @pytest.mark.usefixtures("upload_model_mock", "get_model_mock") + def test_model_upload_compatibility(self): + self._set_up() + md_builder = saved_model_metadata_builder.SavedModelMetadataBuilder( + self.model_path, tags=[tf.saved_model.tag_constants.SERVING] + ) + + generated_md = md_builder.get_metadata() + + try: + models.Model.upload( + display_name=test_models._TEST_MODEL_NAME, + serving_container_image_uri=test_models._TEST_SERVING_CONTAINER_IMAGE, + explanation_parameters=test_models._TEST_EXPLANATION_PARAMETERS, + explanation_metadata=generated_md, # Test metadata from builder + labels=test_models._TEST_LABEL, + ) + except TypeError as e: + if "Parameter to MergeFrom() must be instance of same class" in str(e): + pytest.fail( + f"Model.upload() expects different proto version, more info: {e}" + ) + else: + raise e diff --git a/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf2_test.py b/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf2_test.py index 5ebc0a9af7..2e84dcf1e2 100644 --- a/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf2_test.py +++ b/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf2_test.py @@ -15,15 +15,19 @@ # limitations under the License. # - +import pytest import tensorflow as tf import numpy as np +from google.cloud.aiplatform import models from google.cloud.aiplatform.explain.metadata.tf.v2 import saved_model_metadata_builder from google.cloud.aiplatform.compat.types import ( explanation_metadata_v1beta1 as explanation_metadata, ) +import test_models +from test_models import upload_model_mock, get_model_mock # noqa: F401 + class SavedModelMetadataBuilderTF2Test(tf.test.TestCase): def _set_up_sequential(self): @@ -184,3 +188,28 @@ def test_model_with_feature_column(self): "outputs": {"output_1": {"outputTensorName": "output_1"}}, } assert expected_md == generated_md + + @pytest.mark.usefixtures("upload_model_mock", "get_model_mock") + def test_model_upload_compatibility(self): + self._set_up_sequential() + + builder = saved_model_metadata_builder.SavedModelMetadataBuilder( + self.saved_model_path + ) + generated_md = builder.get_metadata() + + try: + models.Model.upload( + display_name=test_models._TEST_MODEL_NAME, + serving_container_image_uri=test_models._TEST_SERVING_CONTAINER_IMAGE, + explanation_parameters=test_models._TEST_EXPLANATION_PARAMETERS, + explanation_metadata=generated_md, # Test metadata from builder + labels=test_models._TEST_LABEL, + ) + except TypeError as e: + if "Parameter to MergeFrom() must be instance of same class" in str(e): + pytest.fail( + f"Model.upload() expects different proto version, more info: {e}" + ) + else: + raise e From ca00316ee386d45ae363f99103c7b770e14fac13 Mon Sep 17 00:00:00 2001 From: Vinny Senthil Date: Tue, 14 Sep 2021 15:03:03 -0700 Subject: [PATCH 2/3] Use get_metadata_protobuf in test --- .../test_explain_saved_model_metadata_builder_tf1_test.py | 2 +- .../test_explain_saved_model_metadata_builder_tf2_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf1_test.py b/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf1_test.py index a4912185ab..8f5d669895 100644 --- a/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf1_test.py +++ b/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf1_test.py @@ -121,7 +121,7 @@ def test_model_upload_compatibility(self): self.model_path, tags=[tf.saved_model.tag_constants.SERVING] ) - generated_md = md_builder.get_metadata() + generated_md = md_builder.get_metadata_protobuf() try: models.Model.upload( diff --git a/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf2_test.py b/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf2_test.py index 2e84dcf1e2..761fe155c5 100644 --- a/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf2_test.py +++ b/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf2_test.py @@ -196,7 +196,7 @@ def test_model_upload_compatibility(self): builder = saved_model_metadata_builder.SavedModelMetadataBuilder( self.saved_model_path ) - generated_md = builder.get_metadata() + generated_md = builder.get_metadata_protobuf() try: models.Model.upload( From 152d87fe5237ed9d20e6daae3f2beca5acf3364e Mon Sep 17 00:00:00 2001 From: Vinny Senthil Date: Tue, 14 Sep 2021 15:04:33 -0700 Subject: [PATCH 3/3] Fix version mismatch bug, tests passing --- .../explain/metadata/tf/v1/saved_model_metadata_builder.py | 4 +--- .../explain/metadata/tf/v2/saved_model_metadata_builder.py | 4 +--- .../test_explain_saved_model_metadata_builder_tf1_test.py | 4 +--- .../test_explain_saved_model_metadata_builder_tf2_test.py | 4 +--- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/google/cloud/aiplatform/explain/metadata/tf/v1/saved_model_metadata_builder.py b/google/cloud/aiplatform/explain/metadata/tf/v1/saved_model_metadata_builder.py index 89261f8c1f..6f0af6d93b 100644 --- a/google/cloud/aiplatform/explain/metadata/tf/v1/saved_model_metadata_builder.py +++ b/google/cloud/aiplatform/explain/metadata/tf/v1/saved_model_metadata_builder.py @@ -17,9 +17,7 @@ from google.protobuf import json_format from typing import Any, Dict, List, Optional -from google.cloud.aiplatform.compat.types import ( - explanation_metadata_v1beta1 as explanation_metadata, -) +from google.cloud.aiplatform.compat.types import explanation_metadata from google.cloud.aiplatform.explain.metadata import metadata_builder diff --git a/google/cloud/aiplatform/explain/metadata/tf/v2/saved_model_metadata_builder.py b/google/cloud/aiplatform/explain/metadata/tf/v2/saved_model_metadata_builder.py index 36f520d7b0..dd7f2b8d0a 100644 --- a/google/cloud/aiplatform/explain/metadata/tf/v2/saved_model_metadata_builder.py +++ b/google/cloud/aiplatform/explain/metadata/tf/v2/saved_model_metadata_builder.py @@ -18,9 +18,7 @@ from typing import Optional, List, Dict, Any, Tuple from google.cloud.aiplatform.explain.metadata import metadata_builder -from google.cloud.aiplatform.compat.types import ( - explanation_metadata_v1beta1 as explanation_metadata, -) +from google.cloud.aiplatform.compat.types import explanation_metadata class SavedModelMetadataBuilder(metadata_builder.MetadataBuilder): diff --git a/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf1_test.py b/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf1_test.py index 8f5d669895..8c83b3b087 100644 --- a/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf1_test.py +++ b/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf1_test.py @@ -20,9 +20,7 @@ from google.cloud.aiplatform import models from google.cloud.aiplatform.explain.metadata.tf.v1 import saved_model_metadata_builder -from google.cloud.aiplatform.compat.types import ( - explanation_metadata_v1beta1 as explanation_metadata, -) +from google.cloud.aiplatform.compat.types import explanation_metadata import test_models from test_models import upload_model_mock, get_model_mock # noqa: F401 diff --git a/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf2_test.py b/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf2_test.py index 761fe155c5..a18eed243c 100644 --- a/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf2_test.py +++ b/tests/unit/aiplatform/test_explain_saved_model_metadata_builder_tf2_test.py @@ -21,9 +21,7 @@ from google.cloud.aiplatform import models from google.cloud.aiplatform.explain.metadata.tf.v2 import saved_model_metadata_builder -from google.cloud.aiplatform.compat.types import ( - explanation_metadata_v1beta1 as explanation_metadata, -) +from google.cloud.aiplatform.compat.types import explanation_metadata import test_models from test_models import upload_model_mock, get_model_mock # noqa: F401