From 5cf38596d115da63cdddc8958b6ae8f455bdb9a6 Mon Sep 17 00:00:00 2001 From: Andrew Ferlitsch Date: Thu, 10 Dec 2020 13:56:35 -0800 Subject: [PATCH] feat: xai samples (#83) --- samples/snippets/explain_tabular_sample.py | 68 +++++++++++++++ .../snippets/explain_tabular_sample_test.py | 37 ++++++++ ..._explain_image_managed_container_sample.py | 83 ++++++++++++++++++ ...ain_image_managed_container_sample_test.py | 52 +++++++++++ ...xplain_tabular_managed_container_sample.py | 87 +++++++++++++++++++ ...n_tabular_managed_container_sample_test.py | 54 ++++++++++++ 6 files changed, 381 insertions(+) create mode 100644 samples/snippets/explain_tabular_sample.py create mode 100644 samples/snippets/explain_tabular_sample_test.py create mode 100644 samples/snippets/upload_model_explain_image_managed_container_sample.py create mode 100644 samples/snippets/upload_model_explain_image_managed_container_sample_test.py create mode 100644 samples/snippets/upload_model_explain_tabular_managed_container_sample.py create mode 100644 samples/snippets/upload_model_explain_tabular_managed_container_sample_test.py diff --git a/samples/snippets/explain_tabular_sample.py b/samples/snippets/explain_tabular_sample.py new file mode 100644 index 0000000000..c0997d77b6 --- /dev/null +++ b/samples/snippets/explain_tabular_sample.py @@ -0,0 +1,68 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_explain_tabular_sample] +from typing import Dict + +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def explain_tabular_sample( + project: str, + endpoint_id: str, + instance_dict: Dict, + location: str = "us-central1", + api_endpoint: str = "us-central1-prediction-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PredictionServiceClient(client_options=client_options) + # The format of each instance should conform to the deployed model's prediction input schema. + instance = json_format.ParseDict(instance_dict, Value()) + instances = [instance] + # tabular models do not have additional parameters + parameters_dict = {} + parameters = json_format.ParseDict(parameters_dict, Value()) + endpoint = client.endpoint_path( + project=project, location=location, endpoint=endpoint_id + ) + response = client.explain( + endpoint=endpoint, instances=instances, parameters=parameters + ) + print("response") + print(" deployed_model_id:", response.deployed_model_id) + explanations = response.explanations + for explanation in explanations: + print(" explanation") + # Feature attributions. + attributions = explanation.attributions + for attribution in attributions: + print(" attribution") + print(" baseline_output_value:", attribution.baseline_output_value) + print(" instance_output_value:", attribution.instance_output_value) + print(" output_display_name:", attribution.output_display_name) + print(" approximation_error:", attribution.approximation_error) + print(" output_name:", attribution.output_name) + output_index = attribution.output_index + for output_index in output_index: + print(" output_index:", output_index) + predictions = response.predictions + for prediction in predictions: + print(" prediction:", dict(prediction)) + + +# [END aiplatform_explain_tabular_sample] diff --git a/samples/snippets/explain_tabular_sample_test.py b/samples/snippets/explain_tabular_sample_test.py new file mode 100644 index 0000000000..910107ac37 --- /dev/null +++ b/samples/snippets/explain_tabular_sample_test.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import explain_tabular_sample + +ENDPOINT_ID = "4966625964059525120" # iris 1000 +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") + +INSTANCE = { + "petal_length": "1.4", + "petal_width": "1.3", + "sepal_length": "5.1", + "sepal_width": "2.8", +} + + +def test_ucaip_generated_explain_tabular_sample(capsys): + + explain_tabular_sample.explain_tabular_sample( + instance_dict=INSTANCE, project=PROJECT_ID, endpoint_id=ENDPOINT_ID + ) + + out, _ = capsys.readouterr() + assert 'attribution' in out diff --git a/samples/snippets/upload_model_explain_image_managed_container_sample.py b/samples/snippets/upload_model_explain_image_managed_container_sample.py new file mode 100644 index 0000000000..0b5f46533a --- /dev/null +++ b/samples/snippets/upload_model_explain_image_managed_container_sample.py @@ -0,0 +1,83 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_upload_model_explain_image_managed_container_sample] +from google.cloud import aiplatform + + +def upload_model_explain_image_managed_container_sample( + project: str, + display_name: str, + container_spec_image_uri: str, + artifact_uri: str, + input_tensor_name: str, + output_tensor_name: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + + # Container specification for deploying the model + container_spec = {"image_uri": container_spec_image_uri, "command": [], "args": []} + + # The explainabilty method and corresponding parameters + parameters = aiplatform.gapic.ExplanationParameters( + {"xrai_attribution": {"step_count": 1}} + ) + + # The input tensor for feature attribution to the output + # For single input model, y = f(x), this will be the serving input layer. + input_metadata = aiplatform.gapic.ExplanationMetadata.InputMetadata( + { + "input_tensor_name": input_tensor_name, + # Input is image data + "modality": "image", + } + ) + + # The output tensor to explain + # For single output model, y = f(x), this will be the serving output layer. + output_metadata = aiplatform.gapic.ExplanationMetadata.OutputMetadata( + {"output_tensor_name": output_tensor_name} + ) + + # Assemble the explanation metadata + metadata = aiplatform.gapic.ExplanationMetadata( + inputs={"image": input_metadata}, outputs={"prediction": output_metadata} + ) + + # Assemble the explanation specification + explanation_spec = aiplatform.gapic.ExplanationSpec( + parameters=parameters, metadata=metadata + ) + + model = aiplatform.gapic.Model( + display_name=display_name, + # The Cloud Storage location of the custom model + artifact_uri=artifact_uri, + explanation_spec=explanation_spec, + container_spec=container_spec, + ) + parent = f"projects/{project}/locations/{location}" + response = client.upload_model(parent=parent, model=model) + print("Long running operation:", response.operation.name) + upload_model_response = response.result(timeout=timeout) + print("upload_model_response:", upload_model_response) + + +# [END aiplatform_upload_model_explain_image_managed_container_sample] diff --git a/samples/snippets/upload_model_explain_image_managed_container_sample_test.py b/samples/snippets/upload_model_explain_image_managed_container_sample_test.py new file mode 100644 index 0000000000..e93b9348ec --- /dev/null +++ b/samples/snippets/upload_model_explain_image_managed_container_sample_test.py @@ -0,0 +1,52 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from uuid import uuid4 + +import pytest + +import helpers + +import upload_model_explain_image_managed_container_sample + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +IMAGE_URI = "gcr.io/cloud-aiplatform/prediction/tf2-cpu.2-1:latest" +ARTIFACT_URI = "gs://ucaip-samples-us-central1/model/cifar" +DISPLAY_NAME = f"temp_upload_model_explain_image_managed_container_sample_{uuid4()}" + +INPUT_TENSOR_NAME = "bytes_inputs" +OUTPUT_TENSOR_NAME = "output_0" + + +@pytest.fixture(scope="function", autouse=True) +def teardown(teardown_model): + yield + + +def test_ucaip_generated_upload_model_explain_image_managed_container_sample(capsys, shared_state): + + upload_model_explain_image_managed_container_sample.upload_model_explain_image_managed_container_sample( + display_name=DISPLAY_NAME, + artifact_uri=ARTIFACT_URI, + container_spec_image_uri=IMAGE_URI, + project=PROJECT_ID, + input_tensor_name=INPUT_TENSOR_NAME, + output_tensor_name=OUTPUT_TENSOR_NAME + ) + + out, _ = capsys.readouterr() + + shared_state["model_name"] = helpers.get_name(out, key="model") diff --git a/samples/snippets/upload_model_explain_tabular_managed_container_sample.py b/samples/snippets/upload_model_explain_tabular_managed_container_sample.py new file mode 100644 index 0000000000..5d2d56a868 --- /dev/null +++ b/samples/snippets/upload_model_explain_tabular_managed_container_sample.py @@ -0,0 +1,87 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_upload_model_explain_tabular_managed_container_sample] +from google.cloud import aiplatform + + +def upload_model_explain_tabular_managed_container_sample( + project: str, + display_name: str, + container_spec_image_uri: str, + artifact_uri: str, + input_tensor_name: str, + output_tensor_name: str, + feature_names: list, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + + # Container specification for deploying the model + container_spec = {"image_uri": container_spec_image_uri, "command": [], "args": []} + + # The explainabilty method and corresponding parameters + parameters = aiplatform.gapic.ExplanationParameters( + {"xrai_attribution": {"step_count": 1}} + ) + + # The input tensor for feature attribution to the output + # For single input model, y = f(x), this will be the serving input layer. + input_metadata = aiplatform.gapic.ExplanationMetadata.InputMetadata( + { + "input_tensor_name": input_tensor_name, + # Input is tabular data + "modality": "numeric", + # Assign feature names to the inputs for explanation + "encoding": "BAG_OF_FEATURES", + "index_feature_mapping": feature_names, + } + ) + + # The output tensor to explain + # For single output model, y = f(x), this will be the serving output layer. + output_metadata = aiplatform.gapic.ExplanationMetadata.OutputMetadata( + {"output_tensor_name": output_tensor_name} + ) + + # Assemble the explanation metadata + metadata = aiplatform.gapic.ExplanationMetadata( + inputs={"features": input_metadata}, outputs={"prediction": output_metadata} + ) + + # Assemble the explanation specification + explanation_spec = aiplatform.gapic.ExplanationSpec( + parameters=parameters, metadata=metadata + ) + + model = aiplatform.gapic.Model( + display_name=display_name, + # The Cloud Storage location of the custom model + artifact_uri=artifact_uri, + explanation_spec=explanation_spec, + container_spec=container_spec, + ) + parent = f"projects/{project}/locations/{location}" + response = client.upload_model(parent=parent, model=model) + print("Long running operation:", response.operation.name) + upload_model_response = response.result(timeout=timeout) + print("upload_model_response:", upload_model_response) + + +# [END aiplatform_upload_model_explain_tabular_managed_container_sample] diff --git a/samples/snippets/upload_model_explain_tabular_managed_container_sample_test.py b/samples/snippets/upload_model_explain_tabular_managed_container_sample_test.py new file mode 100644 index 0000000000..581a11e5b4 --- /dev/null +++ b/samples/snippets/upload_model_explain_tabular_managed_container_sample_test.py @@ -0,0 +1,54 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from uuid import uuid4 + +import pytest + +import helpers + +import upload_model_explain_tabular_managed_container_sample + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +IMAGE_URI = "gcr.io/cloud-aiplatform/prediction/tf2-cpu.2-1:latest" +ARTIFACT_URI = "gs://ucaip-samples-us-central1/model/boston_housing/" +DISPLAY_NAME = f"temp_upload_model_test_{uuid4()}" + +INPUT_TENSOR_NAME = "dense_input" +OUTPUT_TENSOR_NAME = "dense_2" + + +@pytest.fixture(scope="function", autouse=True) +def teardown(teardown_model): + yield + + +def test_ucaip_generated_upload_model_explain_tabular_managed_constainer_sample(capsys, shared_state): + + upload_model_explain_tabular_managed_container_sample.upload_model_explain_tabular_managed_container_sample( + display_name=DISPLAY_NAME, + artifact_uri=ARTIFACT_URI, + container_spec_image_uri=IMAGE_URI, + project=PROJECT_ID, + input_tensor_name=INPUT_TENSOR_NAME, + output_tensor_name=OUTPUT_TENSOR_NAME, + feature_names=["crim", "zn", "indus", "chas", "nox", "rm", "age", + "dis", "rad", "tax", "ptratio", "b", "lstat"] + ) + + out, _ = capsys.readouterr() + + shared_state["model_name"] = helpers.get_name(out, key="model")