diff --git a/samples/snippets/cancel_batch_prediction_job_sample.py b/samples/snippets/cancel_batch_prediction_job_sample.py new file mode 100644 index 0000000000..7b7b95a8c3 --- /dev/null +++ b/samples/snippets/cancel_batch_prediction_job_sample.py @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_cancel_batch_prediction_job_sample] +from google.cloud import aiplatform + + +def cancel_batch_prediction_job_sample( + project: str, + batch_prediction_job_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + name = client.batch_prediction_job_path( + project=project, location=location, batch_prediction_job=batch_prediction_job_id + ) + response = client.cancel_batch_prediction_job(name=name) + print("response:", response) + + +# [END aiplatform_cancel_batch_prediction_job_sample] diff --git a/samples/snippets/cancel_custom_job_sample.py b/samples/snippets/cancel_custom_job_sample.py new file mode 100644 index 0000000000..237d872a08 --- /dev/null +++ b/samples/snippets/cancel_custom_job_sample.py @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_cancel_custom_job_sample] +from google.cloud import aiplatform + + +def cancel_custom_job_sample( + project: str, + custom_job_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + name = client.custom_job_path( + project=project, location=location, custom_job=custom_job_id + ) + response = client.cancel_custom_job(name=name) + print("response:", response) + + +# [END aiplatform_cancel_custom_job_sample] diff --git a/samples/snippets/cancel_data_labeling_job_sample.py b/samples/snippets/cancel_data_labeling_job_sample.py new file mode 100644 index 0000000000..4ad232812b --- /dev/null +++ b/samples/snippets/cancel_data_labeling_job_sample.py @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_cancel_data_labeling_job_sample] +from google.cloud import aiplatform + + +def cancel_data_labeling_job_sample( + project: str, + data_labeling_job_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + name = client.data_labeling_job_path( + project=project, location=location, data_labeling_job=data_labeling_job_id + ) + response = client.cancel_data_labeling_job(name=name) + print("response:", response) + + +# [END aiplatform_cancel_data_labeling_job_sample] diff --git a/samples/snippets/cancel_hyperparameter_tuning_job_sample.py b/samples/snippets/cancel_hyperparameter_tuning_job_sample.py new file mode 100644 index 0000000000..120e257c36 --- /dev/null +++ b/samples/snippets/cancel_hyperparameter_tuning_job_sample.py @@ -0,0 +1,38 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_cancel_hyperparameter_tuning_job_sample] +from google.cloud import aiplatform + + +def cancel_hyperparameter_tuning_job_sample( + project: str, + hyperparameter_tuning_job_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + name = client.hyperparameter_tuning_job_path( + project=project, + location=location, + hyperparameter_tuning_job=hyperparameter_tuning_job_id, + ) + response = client.cancel_hyperparameter_tuning_job(name=name) + print("response:", response) + + +# [END aiplatform_cancel_hyperparameter_tuning_job_sample] diff --git a/samples/snippets/cancel_training_pipeline_sample.py b/samples/snippets/cancel_training_pipeline_sample.py new file mode 100644 index 0000000000..6ecc71bde7 --- /dev/null +++ b/samples/snippets/cancel_training_pipeline_sample.py @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_cancel_training_pipeline_sample] +from google.cloud import aiplatform + + +def cancel_training_pipeline_sample( + project: str, + training_pipeline_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) + name = client.training_pipeline_path( + project=project, location=location, training_pipeline=training_pipeline_id + ) + response = client.cancel_training_pipeline(name=name) + print("response:", response) + + +# [END aiplatform_cancel_training_pipeline_sample] diff --git a/samples/snippets/cancel_training_pipeline_sample_test.py b/samples/snippets/cancel_training_pipeline_sample_test.py new file mode 100644 index 0000000000..9ffb601e42 --- /dev/null +++ b/samples/snippets/cancel_training_pipeline_sample_test.py @@ -0,0 +1,78 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 +import pytest +import os + +import helpers + +import create_training_pipeline_sample +import cancel_training_pipeline_sample +import delete_training_pipeline_sample +import get_training_pipeline_sample + +from google.cloud import aiplatform + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +LOCATION = "us-central1" +DATASET_ID = "1084241610289446912" # Permanent 50 Flowers Dataset +DISPLAY_NAME = f"temp_create_training_pipeline_test_{uuid4()}" +TRAINING_DEFINITION_GCS_PATH = "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_image_classification_1.0.0.yaml" + + +@pytest.fixture(scope="function") +def training_pipeline_id(capsys): + create_training_pipeline_sample.create_training_pipeline_sample( + project=PROJECT_ID, + display_name=DISPLAY_NAME, + training_task_definition=TRAINING_DEFINITION_GCS_PATH, + dataset_id=DATASET_ID, + model_display_name=f"Temp Model for {DISPLAY_NAME}", + ) + + out, _ = capsys.readouterr() + + training_pipeline_name = helpers.get_name(out) + + assert "/" in training_pipeline_name + + training_pipeline_id = training_pipeline_name.split("/")[-1] + + yield training_pipeline_id + + delete_training_pipeline_sample.delete_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + +def test_ucaip_generated_cancel_training_pipeline_sample(capsys, training_pipeline_id): + # Run cancel pipeline sample + cancel_training_pipeline_sample.cancel_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + pipeline_client = aiplatform.gapic.PipelineServiceClient( + client_options={"api_endpoint": "us-central1-aiplatform.googleapis.com"} + ) + + # Waiting for training pipeline to be in CANCELLED state, otherwise raise error + helpers.wait_for_job_state( + get_job_method=pipeline_client.get_training_pipeline, + name=pipeline_client.training_pipeline_path( + project=PROJECT_ID, + location=LOCATION, + training_pipeline=training_pipeline_id, + ), + ) diff --git a/samples/snippets/create_batch_prediction_job_video_classification_sample.py b/samples/snippets/create_batch_prediction_job_video_classification_sample.py new file mode 100644 index 0000000000..96d8c89503 --- /dev/null +++ b/samples/snippets/create_batch_prediction_job_video_classification_sample.py @@ -0,0 +1,64 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_batch_prediction_job_video_classification_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_batch_prediction_job_video_classification_sample( + project: str, + display_name: str, + model_name: str, + gcs_source_uri: str, + gcs_destination_output_uri_prefix: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + model_parameters_dict = { + "confidenceThreshold": 0.5, + "maxPredictions": 10000, + "segmentClassification": True, + "shotClassification": True, + "oneSecIntervalClassification": True, + } + model_parameters = json_format.ParseDict(model_parameters_dict, Value()) + + batch_prediction_job = { + "display_name": display_name, + # Format: 'projects/{project}/locations/{location}/models/{model_id}' + "model": model_name, + "model_parameters": model_parameters, + "input_config": { + "instances_format": "jsonl", + "gcs_source": {"uris": [gcs_source_uri]}, + }, + "output_config": { + "predictions_format": "jsonl", + "gcs_destination": {"output_uri_prefix": gcs_destination_output_uri_prefix}, + }, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_batch_prediction_job( + parent=parent, batch_prediction_job=batch_prediction_job + ) + print("response:", response) + + +# [END aiplatform_create_batch_prediction_job_video_classification_sample] diff --git a/samples/snippets/create_batch_prediction_job_video_classification_sample_test.py b/samples/snippets/create_batch_prediction_job_video_classification_sample_test.py new file mode 100644 index 0000000000..41752dc12b --- /dev/null +++ b/samples/snippets/create_batch_prediction_job_video_classification_sample_test.py @@ -0,0 +1,85 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 +import pytest +import os + +import helpers + +import create_batch_prediction_job_video_classification_sample +import cancel_batch_prediction_job_sample +import delete_batch_prediction_job_sample + +from google.cloud import aiplatform + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +LOCATION = "us-central1" +MODEL_ID = "667940119734386688" # Permanent 5 class sports model +DISPLAY_NAME = f"temp_create_batch_prediction_vcn_test_{uuid4()}" +GCS_SOURCE_URI = ( + "gs://ucaip-samples-test-output/inputs/vcn_40_batch_prediction_input.jsonl" +) +GCS_OUTPUT_URI = "gs://ucaip-samples-test-output/" + + +@pytest.fixture(scope="function") +def shared_state(): + + shared_state = {} + + yield shared_state + + assert "/" in shared_state["batch_prediction_job_name"] + + batch_prediction_job = shared_state["batch_prediction_job_name"].split("/")[-1] + + # Stop the batch prediction job + cancel_batch_prediction_job_sample.cancel_batch_prediction_job_sample( + project=PROJECT_ID, batch_prediction_job_id=batch_prediction_job + ) + + job_client = aiplatform.gapic.JobServiceClient( + client_options={"api_endpoint": "us-central1-aiplatform.googleapis.com"} + ) + + # Waiting for batch prediction job to be in CANCELLED state + helpers.wait_for_job_state( + get_job_method=job_client.get_batch_prediction_job, + name=shared_state["batch_prediction_job_name"], + ) + + # Delete the batch prediction job + delete_batch_prediction_job_sample.delete_batch_prediction_job_sample( + project=PROJECT_ID, batch_prediction_job_id=batch_prediction_job + ) + + +# Creating AutoML Video Classification batch prediction job +def test_ucaip_generated_create_batch_prediction_vcn_sample(capsys, shared_state): + + model_name = f"projects/{PROJECT_ID}/locations/{LOCATION}/models/{MODEL_ID}" + + create_batch_prediction_job_video_classification_sample.create_batch_prediction_job_video_classification_sample( + project=PROJECT_ID, + display_name=DISPLAY_NAME, + model_name=model_name, + gcs_source_uri=GCS_SOURCE_URI, + gcs_destination_output_uri_prefix=GCS_OUTPUT_URI, + ) + + out, _ = capsys.readouterr() + + # Save resource name of the newly created batch prediction job + shared_state["batch_prediction_job_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_batch_prediction_job_video_object_tracking_sample.py b/samples/snippets/create_batch_prediction_job_video_object_tracking_sample.py new file mode 100644 index 0000000000..2ccbfc9087 --- /dev/null +++ b/samples/snippets/create_batch_prediction_job_video_object_tracking_sample.py @@ -0,0 +1,58 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_batch_prediction_job_video_object_tracking_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_batch_prediction_job_video_object_tracking_sample( + project: str, + display_name: str, + model_name: str, + gcs_source_uri: str, + gcs_destination_output_uri_prefix: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + model_parameters_dict = {"confidenceThreshold": 0.0} + model_parameters = json_format.ParseDict(model_parameters_dict, Value()) + + batch_prediction_job = { + "display_name": display_name, + # Format: 'projects/{project}/locations/{location}/models/{model_id}' + "model": model_name, + "model_parameters": model_parameters, + "input_config": { + "instances_format": "jsonl", + "gcs_source": {"uris": [gcs_source_uri]}, + }, + "output_config": { + "predictions_format": "jsonl", + "gcs_destination": {"output_uri_prefix": gcs_destination_output_uri_prefix}, + }, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_batch_prediction_job( + parent=parent, batch_prediction_job=batch_prediction_job + ) + print("response:", response) + + +# [END aiplatform_create_batch_prediction_job_video_object_tracking_sample] diff --git a/samples/snippets/create_batch_prediction_job_video_object_tracking_sample_test.py b/samples/snippets/create_batch_prediction_job_video_object_tracking_sample_test.py new file mode 100644 index 0000000000..6ac8660193 --- /dev/null +++ b/samples/snippets/create_batch_prediction_job_video_object_tracking_sample_test.py @@ -0,0 +1,88 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 +import pytest +import os + +import helpers + +import create_batch_prediction_job_video_object_tracking_sample +import cancel_batch_prediction_job_sample +import delete_batch_prediction_job_sample + +from google.cloud import aiplatform + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +LOCATION = "us-central1" +MODEL_ID = "20547673299877888" # Permanent horses model +DISPLAY_NAME = f"temp_create_batch_prediction_vot_test_{uuid4()}" +GCS_SOURCE_URI = ( + "gs://ucaip-samples-test-output/inputs/vot_batch_prediction_input.jsonl" +) +GCS_OUTPUT_URI = "gs://ucaip-samples-test-output/" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + assert "/" in shared_state["batch_prediction_job_name"] + + batch_prediction_job = shared_state["batch_prediction_job_name"].split("/")[-1] + + # Stop the batch prediction job + cancel_batch_prediction_job_sample.cancel_batch_prediction_job_sample( + project=PROJECT_ID, batch_prediction_job_id=batch_prediction_job + ) + + job_client = aiplatform.gapic.JobServiceClient( + client_options={"api_endpoint": "us-central1-aiplatform.googleapis.com"} + ) + + # Waiting for batch prediction job to be in CANCELLED state + helpers.wait_for_job_state( + get_job_method=job_client.get_batch_prediction_job, + name=shared_state["batch_prediction_job_name"], + ) + + # Delete the batch prediction job + delete_batch_prediction_job_sample.delete_batch_prediction_job_sample( + project=PROJECT_ID, batch_prediction_job_id=batch_prediction_job + ) + + +# Creating AutoML Video Object Tracking batch prediction job +def test_ucaip_generated_create_batch_prediction_vcn_sample(capsys, shared_state): + + model_name = f"projects/{PROJECT_ID}/locations/{LOCATION}/models/{MODEL_ID}" + + create_batch_prediction_job_video_object_tracking_sample.create_batch_prediction_job_video_object_tracking_sample( + project=PROJECT_ID, + display_name=DISPLAY_NAME, + model_name=model_name, + gcs_source_uri=GCS_SOURCE_URI, + gcs_destination_output_uri_prefix=GCS_OUTPUT_URI, + ) + + out, _ = capsys.readouterr() + + # Save resource name of the newly created batch prediction job + shared_state["batch_prediction_job_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_custom_job_sample.py b/samples/snippets/create_custom_job_sample.py new file mode 100644 index 0000000000..1493a22ae3 --- /dev/null +++ b/samples/snippets/create_custom_job_sample.py @@ -0,0 +1,55 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_custom_job_sample] +from google.cloud import aiplatform + + +def create_custom_job_sample( + project: str, + display_name: str, + container_image_uri: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + custom_job = { + "display_name": display_name, + "job_spec": { + "worker_pool_specs": [ + { + "machine_spec": { + "machine_type": "n1-standard-4", + "accelerator_type": aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, + "accelerator_count": 1, + }, + "replica_count": 1, + "container_spec": { + "image_uri": container_image_uri, + "command": [], + "args": [], + }, + } + ] + }, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_custom_job(parent=parent, custom_job=custom_job) + print("response:", response) + + +# [END aiplatform_create_custom_job_sample] diff --git a/samples/snippets/create_custom_job_sample_test.py b/samples/snippets/create_custom_job_sample_test.py new file mode 100644 index 0000000000..affc2895f0 --- /dev/null +++ b/samples/snippets/create_custom_job_sample_test.py @@ -0,0 +1,72 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +import uuid +import os + +import helpers + +import cancel_custom_job_sample +import create_custom_job_sample +import delete_custom_job_sample + +from google.cloud import aiplatform + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +CONTAINER_IMAGE_URI = "gcr.io/ucaip-test/ucaip-training-test:latest" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + custom_job_id = shared_state["custom_job_name"].split("/")[-1] + + # Cancel the created custom job + cancel_custom_job_sample.cancel_custom_job_sample( + project=PROJECT_ID, custom_job_id=custom_job_id + ) + + client_options = {"api_endpoint": "us-central1-aiplatform.googleapis.com"} + job_client = aiplatform.gapic.JobServiceClient(client_options=client_options) + + # Waiting for custom job to be in CANCELLED state + helpers.wait_for_job_state( + get_job_method=job_client.get_custom_job, name=shared_state["custom_job_name"], + ) + + # Delete the created custom job + delete_custom_job_sample.delete_custom_job_sample( + project=PROJECT_ID, custom_job_id=custom_job_id + ) + + +def test_ucaip_generated_create_custom_job(capsys, shared_state): + create_custom_job_sample.create_custom_job_sample( + display_name=f"temp_create_custom_job_test_{uuid.uuid4()}", + container_image_uri=CONTAINER_IMAGE_URI, + project=PROJECT_ID, + ) + out, _ = capsys.readouterr() + assert "response" in out + + shared_state["custom_job_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_data_labeling_job_images_sample.py b/samples/snippets/create_data_labeling_job_images_sample.py new file mode 100644 index 0000000000..396b0d1823 --- /dev/null +++ b/samples/snippets/create_data_labeling_job_images_sample.py @@ -0,0 +1,57 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_data_labeling_job_images_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_data_labeling_job_images_sample( + project: str, + display_name: str, + dataset: str, + instruction_uri: str, + annotation_spec: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + inputs_dict = {"annotation_specs": [annotation_spec]} + inputs = json_format.ParseDict(inputs_dict, Value()) + + data_labeling_job = { + "display_name": display_name, + # Full resource name: projects/{project_id}/locations/{location}/datasets/{dataset_id} + "datasets": [dataset], + # labeler_count must be 1, 3, or 5 + "labeler_count": 1, + "instruction_uri": instruction_uri, + "inputs_schema_uri": "gs://google-cloud-aiplatform/schema/datalabelingjob/inputs/image_classification_1.0.0.yaml", + "inputs": inputs, + "annotation_labels": { + "aiplatform.googleapis.com/annotation_set_name": "my_test_saved_query" + }, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_data_labeling_job( + parent=parent, data_labeling_job=data_labeling_job + ) + print("response:", response) + + +# [END aiplatform_create_data_labeling_job_images_sample] diff --git a/samples/snippets/create_data_labeling_job_images_sample_test.py b/samples/snippets/create_data_labeling_job_images_sample_test.py new file mode 100644 index 0000000000..8d99132f18 --- /dev/null +++ b/samples/snippets/create_data_labeling_job_images_sample_test.py @@ -0,0 +1,91 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os +from uuid import uuid4 +from google.cloud import aiplatform + +import helpers + +import create_data_labeling_job_images_sample +import cancel_data_labeling_job_sample +import delete_data_labeling_job_sample + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +API_ENDPOINT = os.getenv("DATA_LABELING_API_ENDPOINT") +LOCATION = "us-central1" +DATASET_ID = "1905673553261363200" +DISPLAY_NAME = f"temp_create_data_labeling_job_test_{uuid4()}" +INSTRUCTIONS_GCS_URI = ( + "gs://ucaip-sample-resources/images/datalabeling_instructions.pdf" +) +ANNOTATION_SPEC = "daisy" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(capsys, shared_state): + yield + + assert "/" in shared_state["data_labeling_job_name"] + + data_labeling_job_id = shared_state["data_labeling_job_name"].split("/")[-1] + + client_options = {"api_endpoint": API_ENDPOINT} + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + + name = client.data_labeling_job_path( + project=PROJECT_ID, location=LOCATION, data_labeling_job=data_labeling_job_id + ) + client.cancel_data_labeling_job(name=name) + + # Verify Data Labelling Job is cancelled, or timeout after 400 seconds + helpers.wait_for_job_state( + get_job_method=client.get_data_labeling_job, name=name, timeout=400, freq=10 + ) + + # Delete the data labeling job + response = client.delete_data_labeling_job(name=name) + print("Delete LRO:", response.operation.name) + delete_data_labeling_job_response = response.result(timeout=300) + print("delete_data_labeling_job_response", delete_data_labeling_job_response) + + out, _ = capsys.readouterr() + assert "delete_data_labeling_job_response" in out + + +# Creating a data labeling job for images +def test_ucaip_generated_create_data_labeling_job_sample(capsys, shared_state): + + dataset_name = f"projects/{PROJECT_ID}/locations/{LOCATION}/datasets/{DATASET_ID}" + + create_data_labeling_job_images_sample.create_data_labeling_job_images_sample( + project=PROJECT_ID, + display_name=DISPLAY_NAME, + instruction_uri=INSTRUCTIONS_GCS_URI, + dataset=dataset_name, + annotation_spec=ANNOTATION_SPEC, + api_endpoint=API_ENDPOINT, + ) + + out, _ = capsys.readouterr() + + # Save resource name of the newly created data labeing job + shared_state["data_labeling_job_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_data_labeling_job_sample.py b/samples/snippets/create_data_labeling_job_sample.py new file mode 100644 index 0000000000..37dbde2073 --- /dev/null +++ b/samples/snippets/create_data_labeling_job_sample.py @@ -0,0 +1,58 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_data_labeling_job_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_data_labeling_job_sample( + project: str, + display_name: str, + dataset_name: str, + instruction_uri: str, + inputs_schema_uri: str, + annotation_spec: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + inputs_dict = {"annotation_specs": [annotation_spec]} + inputs = json_format.ParseDict(inputs_dict, Value()) + + data_labeling_job = { + "display_name": display_name, + # Full resource name: projects/{project_id}/locations/{location}/datasets/{dataset_id} + "datasets": [dataset_name], + # labeler_count must be 1, 3, or 5 + "labeler_count": 1, + "instruction_uri": instruction_uri, + "inputs_schema_uri": inputs_schema_uri, + "inputs": inputs, + "annotation_labels": { + "aiplatform.googleapis.com/annotation_set_name": "my_test_saved_query" + }, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_data_labeling_job( + parent=parent, data_labeling_job=data_labeling_job + ) + print("response:", response) + + +# [END aiplatform_create_data_labeling_job_sample] diff --git a/samples/snippets/create_data_labeling_job_sample_test.py b/samples/snippets/create_data_labeling_job_sample_test.py new file mode 100644 index 0000000000..dc9a0baacb --- /dev/null +++ b/samples/snippets/create_data_labeling_job_sample_test.py @@ -0,0 +1,93 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os +from uuid import uuid4 +from google.cloud import aiplatform + +import helpers + +import create_data_labeling_job_sample +import cancel_data_labeling_job_sample +import delete_data_labeling_job_sample + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +API_ENDPOINT = os.getenv("DATA_LABELING_API_ENDPOINT") +LOCATION = "us-central1" +DATASET_ID = "1905673553261363200" +DISPLAY_NAME = f"temp_create_data_labeling_job_test_{uuid4()}" +INPUTS_SCHEMA_URI = "gs://google-cloud-aiplatform/schema/datalabelingjob/inputs/image_classification.yaml" +INSTRUCTIONS_GCS_URI = ( + "gs://ucaip-sample-resources/images/datalabeling_instructions.pdf" +) +ANNOTATION_SPEC = "daisy" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(capsys, shared_state): + yield + + assert "/" in shared_state["data_labeling_job_name"] + + data_labeling_job_id = shared_state["data_labeling_job_name"].split("/")[-1] + + client_options = {"api_endpoint": API_ENDPOINT} + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + + name = client.data_labeling_job_path( + project=PROJECT_ID, location=LOCATION, data_labeling_job=data_labeling_job_id + ) + client.cancel_data_labeling_job(name=name) + + # Verify Data Labelling Job is cancelled, or timeout after 400 seconds + helpers.wait_for_job_state( + get_job_method=client.get_data_labeling_job, name=name, timeout=400, freq=10 + ) + + # Delete the data labeling job + response = client.delete_data_labeling_job(name=name) + print("Delete LRO:", response.operation.name) + delete_data_labeling_job_response = response.result(timeout=300) + print("delete_data_labeling_job_response", delete_data_labeling_job_response) + + out, _ = capsys.readouterr() + assert "delete_data_labeling_job_response" in out + + +# Creating a data labeling job for images +def test_ucaip_generated_create_data_labeling_job_sample(capsys, shared_state): + + dataset_name = f"projects/{PROJECT_ID}/locations/{LOCATION}/datasets/{DATASET_ID}" + + create_data_labeling_job_sample.create_data_labeling_job_sample( + project=PROJECT_ID, + display_name=DISPLAY_NAME, + instruction_uri=INSTRUCTIONS_GCS_URI, + dataset_name=dataset_name, + inputs_schema_uri=INPUTS_SCHEMA_URI, + annotation_spec=ANNOTATION_SPEC, + api_endpoint=API_ENDPOINT, + ) + + out, _ = capsys.readouterr() + + # Save resource name of the newly created data labeing job + shared_state["data_labeling_job_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_data_labeling_job_video_sample.py b/samples/snippets/create_data_labeling_job_video_sample.py new file mode 100644 index 0000000000..91edb975c7 --- /dev/null +++ b/samples/snippets/create_data_labeling_job_video_sample.py @@ -0,0 +1,57 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_data_labeling_job_video_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_data_labeling_job_video_sample( + project: str, + display_name: str, + dataset: str, + instruction_uri: str, + annotation_spec: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + inputs_dict = {"annotation_specs": [annotation_spec]} + inputs = json_format.ParseDict(inputs_dict, Value()) + + data_labeling_job = { + "display_name": display_name, + # Full resource name: projects/{project_id}/locations/{location}/datasets/{dataset_id} + "datasets": [dataset], + # labeler_count must be 1, 3, or 5 + "labeler_count": 1, + "instruction_uri": instruction_uri, + "inputs_schema_uri": "gs://google-cloud-aiplatform/schema/datalabelingjob/inputs/video_classification_1.0.0.yaml", + "inputs": inputs, + "annotation_labels": { + "aiplatform.googleapis.com/annotation_set_name": "my_test_saved_query" + }, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_data_labeling_job( + parent=parent, data_labeling_job=data_labeling_job + ) + print("response:", response) + + +# [END aiplatform_create_data_labeling_job_video_sample] diff --git a/samples/snippets/create_data_labeling_job_video_sample_test.py b/samples/snippets/create_data_labeling_job_video_sample_test.py new file mode 100644 index 0000000000..74b20147f9 --- /dev/null +++ b/samples/snippets/create_data_labeling_job_video_sample_test.py @@ -0,0 +1,88 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os +from uuid import uuid4 +from google.cloud import aiplatform + +import helpers + +import create_data_labeling_job_video_sample +import cancel_data_labeling_job_sample +import delete_data_labeling_job_sample + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +API_ENDPOINT = os.getenv("DATA_LABELING_API_ENDPOINT") +LOCATION = "us-central1" +DATASET_ID = "3459133949727473664" +DISPLAY_NAME = f"temp_create_data_labeling_job_test_{uuid4()}" +INSTRUCTIONS_GCS_URI = ( + "gs://ucaip-sample-resources/images/datalabeling_instructions.pdf" +) +ANNOTATION_SPEC = "cartwheel" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + assert "/" in shared_state["data_labeling_job_name"] + + data_labeling_job_id = shared_state["data_labeling_job_name"].split("/")[-1] + + client_options = {"api_endpoint": API_ENDPOINT} + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + + name = client.data_labeling_job_path( + project=PROJECT_ID, location=LOCATION, data_labeling_job=data_labeling_job_id + ) + client.cancel_data_labeling_job(name=name) + + # Verify Data Labelling Job is cancelled, or timeout after 400 seconds + helpers.wait_for_job_state( + get_job_method=client.get_data_labeling_job, name=name, timeout=400, freq=10 + ) + + # Delete the data labeling job + response = client.delete_data_labeling_job(name=name) + print("Delete LRO:", response.operation.name) + delete_data_labeling_job_response = response.result(timeout=300) + print("delete_data_labeling_job_response", delete_data_labeling_job_response) + + +# Creating a data labeling job for images +def test_ucaip_generated_create_data_labeling_job_sample(capsys, shared_state): + + dataset_name = f"projects/{PROJECT_ID}/locations/{LOCATION}/datasets/{DATASET_ID}" + + create_data_labeling_job_video_sample.create_data_labeling_job_video_sample( + project=PROJECT_ID, + display_name=DISPLAY_NAME, + instruction_uri=INSTRUCTIONS_GCS_URI, + dataset=dataset_name, + annotation_spec=ANNOTATION_SPEC, + api_endpoint=API_ENDPOINT, + ) + + out, _ = capsys.readouterr() + + # Save resource name of the newly created data labeing job + shared_state["data_labeling_job_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_dataset_image_sample.py b/samples/snippets/create_dataset_image_sample.py new file mode 100644 index 0000000000..513b8a7f83 --- /dev/null +++ b/samples/snippets/create_dataset_image_sample.py @@ -0,0 +1,41 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_dataset_image_sample] +from google.cloud import aiplatform + + +def create_dataset_image_sample( + project: str, + display_name: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + dataset = { + "display_name": display_name, + "metadata_schema_uri": "gs://google-cloud-aiplatform/schema/dataset/metadata/image_1.0.0.yaml", + } + parent = f"projects/{project}/locations/{location}" + response = client.create_dataset(parent=parent, dataset=dataset) + print("Long running operation:", response.operation.name) + create_dataset_response = response.result(timeout=timeout) + print("create_dataset_response:", create_dataset_response) + + +# [END aiplatform_create_dataset_image_sample] diff --git a/samples/snippets/create_dataset_image_sample_test.py b/samples/snippets/create_dataset_image_sample_test.py new file mode 100644 index 0000000000..ea2f2607ca --- /dev/null +++ b/samples/snippets/create_dataset_image_sample_test.py @@ -0,0 +1,56 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 + +import pytest +import os + +import helpers + +import create_dataset_image_sample +import delete_dataset_sample + + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") + + +@pytest.fixture +def shared_state(): + shared_state = {} + yield shared_state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + assert "/" in shared_state["dataset_name"] + + dataset_id = shared_state["dataset_name"].split("/")[-1] + + # Delete the created dataset + delete_dataset_sample.delete_dataset_sample( + project=PROJECT_ID, dataset_id=dataset_id + ) + + +def test_ucaip_generated_create_dataset_image(capsys, shared_state): + create_dataset_image_sample.create_dataset_image_sample( + display_name=f"temp_create_dataset_image_test_{uuid4()}", project=PROJECT_ID + ) + out, _ = capsys.readouterr() + assert "create_dataset_response" in out + + shared_state["dataset_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_dataset_sample.py b/samples/snippets/create_dataset_sample.py new file mode 100644 index 0000000000..97535ca4ea --- /dev/null +++ b/samples/snippets/create_dataset_sample.py @@ -0,0 +1,42 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_dataset_sample] +from google.cloud import aiplatform + + +def create_dataset_sample( + project: str, + display_name: str, + metadata_schema_uri: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + dataset = { + "display_name": display_name, + "metadata_schema_uri": metadata_schema_uri, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_dataset(parent=parent, dataset=dataset) + print("Long running operation:", response.operation.name) + create_dataset_response = response.result(timeout=timeout) + print("create_dataset_response:", create_dataset_response) + + +# [END aiplatform_create_dataset_sample] diff --git a/samples/snippets/create_dataset_sample_test.py b/samples/snippets/create_dataset_sample_test.py new file mode 100644 index 0000000000..3508092e8c --- /dev/null +++ b/samples/snippets/create_dataset_sample_test.py @@ -0,0 +1,61 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 + +import pytest +import os + +import helpers + +import create_dataset_sample +import delete_dataset_sample + + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +IMAGE_METADATA_SCHEMA_URI = ( + "gs://google-cloud-aiplatform/schema/dataset/metadata/image_1.0.0.yaml" +) + + +@pytest.fixture +def shared_state(): + shared_state = {} + yield shared_state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + assert "/" in shared_state["dataset_name"] + + dataset_id = shared_state["dataset_name"].split("/")[-1] + + # Delete the created dataset + delete_dataset_sample.delete_dataset_sample( + project=PROJECT_ID, dataset_id=dataset_id + ) + + +def test_ucaip_generated_create_dataset_sample_vision(capsys, shared_state): + create_dataset_sample.create_dataset_sample( + display_name=f"temp_create_dataset_test_{uuid4()}", + metadata_schema_uri=IMAGE_METADATA_SCHEMA_URI, + project=PROJECT_ID, + ) + out, _ = capsys.readouterr() + assert "create_dataset_response" in out + + shared_state["dataset_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_dataset_tabular_bigquery_sample.py b/samples/snippets/create_dataset_tabular_bigquery_sample.py new file mode 100644 index 0000000000..821eb00b68 --- /dev/null +++ b/samples/snippets/create_dataset_tabular_bigquery_sample.py @@ -0,0 +1,48 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_dataset_tabular_bigquery_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_dataset_tabular_bigquery_sample( + project: str, + display_name: str, + bigquery_uri: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + metadata_dict = {"input_config": {"bigquery_source": {"uri": bigquery_uri}}} + metadata = json_format.ParseDict(metadata_dict, Value()) + + dataset = { + "display_name": display_name, + "metadata_schema_uri": "gs://google-cloud-aiplatform/schema/dataset/metadata/tabular_1.0.0.yaml", + "metadata": metadata, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_dataset(parent=parent, dataset=dataset) + print("Long running operation:", response.operation.name) + create_dataset_response = response.result(timeout=timeout) + print("create_dataset_response:", create_dataset_response) + + +# [END aiplatform_create_dataset_tabular_bigquery_sample] diff --git a/samples/snippets/create_dataset_tabular_bigquery_sample_test.py b/samples/snippets/create_dataset_tabular_bigquery_sample_test.py new file mode 100644 index 0000000000..a582760cfe --- /dev/null +++ b/samples/snippets/create_dataset_tabular_bigquery_sample_test.py @@ -0,0 +1,59 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 + +import pytest +import os + +import helpers + +import create_dataset_tabular_bigquery_sample +import delete_dataset_sample + + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +BIGQUERY_URI = "bq://ucaip-sample-tests.table_test.all_bq_types" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + assert "/" in shared_state["dataset_name"] + + dataset_id = shared_state["dataset_name"].split("/")[-1] + + # Delete the created dataset + delete_dataset_sample.delete_dataset_sample( + project=PROJECT_ID, dataset_id=dataset_id + ) + + +def test_ucaip_generated_create_dataset_tabular_bigquery(capsys, shared_state): + create_dataset_tabular_bigquery_sample.create_dataset_tabular_bigquery_sample( + display_name=f"temp_create_dataset_test_{uuid4()}", + bigquery_uri=BIGQUERY_URI, + project=PROJECT_ID, + ) + out, _ = capsys.readouterr() + assert "create_dataset_response" in out + + shared_state["dataset_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_dataset_tabular_gcs_sample.py b/samples/snippets/create_dataset_tabular_gcs_sample.py new file mode 100644 index 0000000000..f16aac2c24 --- /dev/null +++ b/samples/snippets/create_dataset_tabular_gcs_sample.py @@ -0,0 +1,48 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_dataset_tabular_gcs_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_dataset_tabular_gcs_sample( + project: str, + display_name: str, + gcs_uri: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + metadata_dict = {"input_config": {"gcs_source": {"uri": [gcs_uri]}}} + metadata = json_format.ParseDict(metadata_dict, Value()) + + dataset = { + "display_name": display_name, + "metadata_schema_uri": "gs://google-cloud-aiplatform/schema/dataset/metadata/tabular_1.0.0.yaml", + "metadata": metadata, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_dataset(parent=parent, dataset=dataset) + print("Long running operation:", response.operation.name) + create_dataset_response = response.result(timeout=timeout) + print("create_dataset_response:", create_dataset_response) + + +# [END aiplatform_create_dataset_tabular_gcs_sample] diff --git a/samples/snippets/create_dataset_tabular_gcs_sample_test.py b/samples/snippets/create_dataset_tabular_gcs_sample_test.py new file mode 100644 index 0000000000..3206a56e97 --- /dev/null +++ b/samples/snippets/create_dataset_tabular_gcs_sample_test.py @@ -0,0 +1,59 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 + +import pytest +import os + +import helpers + +import create_dataset_tabular_gcs_sample +import delete_dataset_sample + + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +GCS_URI = "gs://ucaip-sample-resources/iris_1000.csv" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + assert "/" in shared_state["dataset_name"] + + dataset_id = shared_state["dataset_name"].split("/")[-1] + + # Delete the created dataset + delete_dataset_sample.delete_dataset_sample( + project=PROJECT_ID, dataset_id=dataset_id + ) + + +def test_ucaip_generated_create_dataset_tabular_gcs(capsys, shared_state): + create_dataset_tabular_gcs_sample.create_dataset_tabular_gcs_sample( + display_name=f"temp_create_dataset_test_{uuid4()}", + gcs_uri=GCS_URI, + project=PROJECT_ID, + ) + out, _ = capsys.readouterr() + assert "create_dataset_response" in out + + shared_state["dataset_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_dataset_text_sample.py b/samples/snippets/create_dataset_text_sample.py new file mode 100644 index 0000000000..b83f7cfa34 --- /dev/null +++ b/samples/snippets/create_dataset_text_sample.py @@ -0,0 +1,41 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_dataset_text_sample] +from google.cloud import aiplatform + + +def create_dataset_text_sample( + project: str, + display_name: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + dataset = { + "display_name": display_name, + "metadata_schema_uri": "gs://google-cloud-aiplatform/schema/dataset/metadata/text_1.0.0.yaml", + } + parent = f"projects/{project}/locations/{location}" + response = client.create_dataset(parent=parent, dataset=dataset) + print("Long running operation:", response.operation.name) + create_dataset_response = response.result(timeout=timeout) + print("create_dataset_response:", create_dataset_response) + + +# [END aiplatform_create_dataset_text_sample] diff --git a/samples/snippets/create_dataset_video_sample.py b/samples/snippets/create_dataset_video_sample.py new file mode 100644 index 0000000000..af363bde8e --- /dev/null +++ b/samples/snippets/create_dataset_video_sample.py @@ -0,0 +1,41 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_dataset_video_sample] +from google.cloud import aiplatform + + +def create_dataset_video_sample( + project: str, + display_name: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + dataset = { + "display_name": display_name, + "metadata_schema_uri": "gs://google-cloud-aiplatform/schema/dataset/metadata/video_1.0.0.yaml", + } + parent = f"projects/{project}/locations/{location}" + response = client.create_dataset(parent=parent, dataset=dataset) + print("Long running operation:", response.operation.name) + create_dataset_response = response.result(timeout=timeout) + print("create_dataset_response:", create_dataset_response) + + +# [END aiplatform_create_dataset_video_sample] diff --git a/samples/snippets/create_dataset_video_sample_test.py b/samples/snippets/create_dataset_video_sample_test.py new file mode 100644 index 0000000000..844a25c204 --- /dev/null +++ b/samples/snippets/create_dataset_video_sample_test.py @@ -0,0 +1,59 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 + +import pytest +import os + +import helpers + +import create_dataset_video_sample +import delete_dataset_sample + + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +VIDEO_METADATA_SCHEMA_URI = ( + "gs://google-cloud-aiplatform/schema/dataset/metadata/video_1.0.0.yaml" +) + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + assert "/" in shared_state["dataset_name"] + + dataset_id = shared_state["dataset_name"].split("/")[-1] + + # Delete the created dataset + delete_dataset_sample.delete_dataset_sample( + project=PROJECT_ID, dataset_id=dataset_id + ) + + +def test_ucaip_generated_create_dataset_video_sample_vision(capsys, shared_state): + create_dataset_video_sample.create_dataset_video_sample( + display_name=f"temp_create_dataset_test_{uuid4()}", project=PROJECT_ID + ) + out, _ = capsys.readouterr() + assert "create_dataset_response" in out + + shared_state["dataset_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_endpoint_sample.py b/samples/snippets/create_endpoint_sample.py new file mode 100644 index 0000000000..82c5ab1eb2 --- /dev/null +++ b/samples/snippets/create_endpoint_sample.py @@ -0,0 +1,38 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_endpoint_sample] +from google.cloud import aiplatform + + +def create_endpoint_sample( + project: str, + display_name: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.EndpointServiceClient(client_options=client_options) + endpoint = {"display_name": display_name} + parent = f"projects/{project}/locations/{location}" + response = client.create_endpoint(parent=parent, endpoint=endpoint) + print("Long running operation:", response.operation.name) + create_endpoint_response = response.result(timeout=timeout) + print("create_endpoint_response:", create_endpoint_response) + + +# [END aiplatform_create_endpoint_sample] diff --git a/samples/snippets/create_endpoint_sample_test.py b/samples/snippets/create_endpoint_sample_test.py new file mode 100644 index 0000000000..4f66945908 --- /dev/null +++ b/samples/snippets/create_endpoint_sample_test.py @@ -0,0 +1,54 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os +from uuid import uuid4 + +import helpers + +import create_endpoint_sample, delete_endpoint_sample + +DISPLAY_NAME = f"temp_create_endpoint_test_{uuid4()}" +PROJECT = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + endpoint_id = shared_state["endpoint_name"].split("/")[-1] + + # Delete the endpoint that was just created + delete_endpoint_sample.delete_endpoint_sample( + project=PROJECT, endpoint_id=endpoint_id + ) + + +def test_ucaip_generated_create_endpoint_sample(capsys, shared_state): + + create_endpoint_sample.create_endpoint_sample( + display_name=DISPLAY_NAME, project=PROJECT + ) + + out, _ = capsys.readouterr() + assert "create_endpoint_response" in out + + shared_state["endpoint_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_hyperparameter_tuning_job_sample.py b/samples/snippets/create_hyperparameter_tuning_job_sample.py new file mode 100644 index 0000000000..38b1b720f9 --- /dev/null +++ b/samples/snippets/create_hyperparameter_tuning_job_sample.py @@ -0,0 +1,75 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_hyperparameter_tuning_job_sample] +from google.cloud import aiplatform + + +def create_hyperparameter_tuning_job_sample( + project: str, + display_name: str, + container_image_uri: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + hyperparameter_tuning_job = { + "display_name": display_name, + "max_trial_count": 2, + "parallel_trial_count": 1, + "max_failed_trial_count": 1, + "study_spec": { + "metrics": [ + { + "metric_id": "accuracy", + "goal": aiplatform.gapic.StudySpec.MetricSpec.GoalType.MAXIMIZE, + } + ], + "parameters": [ + { + # Learning rate. + "parameter_id": "lr", + "double_value_spec": {"min_value": 0.001, "max_value": 0.1}, + }, + ], + }, + "trial_job_spec": { + "worker_pool_specs": [ + { + "machine_spec": { + "machine_type": "n1-standard-4", + "accelerator_type": aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, + "accelerator_count": 1, + }, + "replica_count": 1, + "container_spec": { + "image_uri": container_image_uri, + "command": [], + "args": [], + }, + } + ] + }, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_hyperparameter_tuning_job( + parent=parent, hyperparameter_tuning_job=hyperparameter_tuning_job + ) + print("response:", response) + + +# [END aiplatform_create_hyperparameter_tuning_job_sample] diff --git a/samples/snippets/create_hyperparameter_tuning_job_sample_test.py b/samples/snippets/create_hyperparameter_tuning_job_sample_test.py new file mode 100644 index 0000000000..21b16121fb --- /dev/null +++ b/samples/snippets/create_hyperparameter_tuning_job_sample_test.py @@ -0,0 +1,75 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid +import pytest +import os + +import helpers + +import cancel_hyperparameter_tuning_job_sample +import create_hyperparameter_tuning_job_sample +import delete_hyperparameter_tuning_job_sample + +from google.cloud import aiplatform + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +CONTAINER_IMAGE_URI = "gcr.io/ucaip-test/ucaip-training-test:latest" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + hyperparameter_tuning_job_id = shared_state["hyperparameter_tuning_job_name"].split( + "/" + )[-1] + + # Cancel the created hyperparameter tuning job + cancel_hyperparameter_tuning_job_sample.cancel_hyperparameter_tuning_job_sample( + project=PROJECT_ID, hyperparameter_tuning_job_id=hyperparameter_tuning_job_id + ) + + job_client = aiplatform.gapic.JobServiceClient( + client_options={"api_endpoint": "us-central1-aiplatform.googleapis.com"} + ) + + # Waiting for hyperparameter tuning job to be in CANCELLED state + helpers.wait_for_job_state( + get_job_method=job_client.get_hyperparameter_tuning_job, + name=shared_state["hyperparameter_tuning_job_name"], + ) + + # Delete the created hyperparameter tuning job + delete_hyperparameter_tuning_job_sample.delete_hyperparameter_tuning_job_sample( + project=PROJECT_ID, hyperparameter_tuning_job_id=hyperparameter_tuning_job_id + ) + + +def test_ucaip_generated_create_hyperparameter_tuning_job(capsys, shared_state): + create_hyperparameter_tuning_job_sample.create_hyperparameter_tuning_job_sample( + display_name=f"temp_create_hyperparameter_tuning_job_test_{uuid.uuid4()}", + container_image_uri=CONTAINER_IMAGE_URI, + project=PROJECT_ID, + ) + out, _ = capsys.readouterr() + assert "response" in out + + shared_state["hyperparameter_tuning_job_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_training_pipeline_image_classification_sample.py b/samples/snippets/create_training_pipeline_image_classification_sample.py new file mode 100644 index 0000000000..8542a6b843 --- /dev/null +++ b/samples/snippets/create_training_pipeline_image_classification_sample.py @@ -0,0 +1,55 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_training_pipeline_image_classification_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_training_pipeline_image_classification_sample( + project: str, + display_name: str, + dataset_id: str, + model_display_name: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) + training_task_inputs_dict = { + "multiLabel": True, + "modelType": "CLOUD", + "budgetMilliNodeHours": 8000, + "disableEarlyStopping": False, + } + training_task_inputs = json_format.ParseDict(training_task_inputs_dict, Value()) + + training_pipeline = { + "display_name": display_name, + "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_image_classification_1.0.0.yaml", + "training_task_inputs": training_task_inputs, + "input_data_config": {"dataset_id": dataset_id}, + "model_to_upload": {"display_name": model_display_name}, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_training_pipeline( + parent=parent, training_pipeline=training_pipeline + ) + print("response:", response) + + +# [END aiplatform_create_training_pipeline_image_classification_sample] diff --git a/samples/snippets/create_training_pipeline_image_classification_sample_test.py b/samples/snippets/create_training_pipeline_image_classification_sample_test.py new file mode 100644 index 0000000000..2c4ecab9b1 --- /dev/null +++ b/samples/snippets/create_training_pipeline_image_classification_sample_test.py @@ -0,0 +1,80 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 +import pytest +import os + +import helpers + +import create_training_pipeline_image_classification_sample +import cancel_training_pipeline_sample +import delete_training_pipeline_sample + +from google.cloud import aiplatform + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +DATASET_ID = "1084241610289446912" # Permanent 50 Flowers Dataset +DISPLAY_NAME = f"temp_create_training_pipeline_image_classification_test_{uuid4()}" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + training_pipeline_id = shared_state["training_pipeline_name"].split("/")[-1] + + # Stop the training pipeline + cancel_training_pipeline_sample.cancel_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + pipeline_client = aiplatform.gapic.PipelineServiceClient( + client_options={"api_endpoint": "us-central1-aiplatform.googleapis.com"} + ) + + # Waiting for training pipeline to be in CANCELLED state + helpers.wait_for_job_state( + get_job_method=pipeline_client.get_training_pipeline, + name=shared_state["training_pipeline_name"], + ) + + # Delete the training pipeline + delete_training_pipeline_sample.delete_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + +def test_ucaip_generated_create_training_pipeline_video_classification_sample( + capsys, shared_state +): + + create_training_pipeline_image_classification_sample.create_training_pipeline_image_classification_sample( + project=PROJECT_ID, + display_name=DISPLAY_NAME, + dataset_id=DATASET_ID, + model_display_name=f"Temp Model for {DISPLAY_NAME}", + ) + + out, _ = capsys.readouterr() + assert "response:" in out + + # Save resource name of the newly created training pipeline + shared_state["training_pipeline_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_training_pipeline_image_object_detection_sample.py b/samples/snippets/create_training_pipeline_image_object_detection_sample.py new file mode 100644 index 0000000000..2e6ca494f9 --- /dev/null +++ b/samples/snippets/create_training_pipeline_image_object_detection_sample.py @@ -0,0 +1,54 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_training_pipeline_image_object_detection_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_training_pipeline_image_object_detection_sample( + project: str, + display_name: str, + dataset_id: str, + model_display_name: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) + training_task_inputs_dict = { + "modelType": "CLOUD_HIGH_ACCURACY_1", + "budgetMilliNodeHours": 20000, + "disableEarlyStopping": False, + } + training_task_inputs = json_format.ParseDict(training_task_inputs_dict, Value()) + + training_pipeline = { + "display_name": display_name, + "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_image_object_detection_1.0.0.yaml", + "training_task_inputs": training_task_inputs, + "input_data_config": {"dataset_id": dataset_id}, + "model_to_upload": {"display_name": model_display_name}, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_training_pipeline( + parent=parent, training_pipeline=training_pipeline + ) + print("response:", response) + + +# [END aiplatform_create_training_pipeline_image_object_detection_sample] diff --git a/samples/snippets/create_training_pipeline_image_object_detection_sample_test.py b/samples/snippets/create_training_pipeline_image_object_detection_sample_test.py new file mode 100644 index 0000000000..0e76ae02f1 --- /dev/null +++ b/samples/snippets/create_training_pipeline_image_object_detection_sample_test.py @@ -0,0 +1,80 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 +import pytest +import os + +import helpers + +import create_training_pipeline_image_object_detection_sample +import cancel_training_pipeline_sample +import delete_training_pipeline_sample + +from google.cloud import aiplatform + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +DATASET_ID = "3555732643297361920" # permanent_salad_object_detection_dataset +DISPLAY_NAME = f"temp_create_training_pipeline_image_obj_detection_test_{uuid4()}" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + training_pipeline_id = shared_state["training_pipeline_name"].split("/")[-1] + + # Stop the training pipeline + cancel_training_pipeline_sample.cancel_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + pipeline_client = aiplatform.gapic.PipelineServiceClient( + client_options={"api_endpoint": "us-central1-aiplatform.googleapis.com"} + ) + + # Waiting for training pipeline to be in CANCELLED state + helpers.wait_for_job_state( + get_job_method=pipeline_client.get_training_pipeline, + name=shared_state["training_pipeline_name"], + ) + + # Delete the training pipeline + delete_training_pipeline_sample.delete_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + +def test_ucaip_generated_create_training_pipeline_image_object_dectection( + capsys, shared_state +): + + create_training_pipeline_image_object_detection_sample.create_training_pipeline_image_object_detection_sample( + project=PROJECT_ID, + display_name=DISPLAY_NAME, + dataset_id=DATASET_ID, + model_display_name=f"Temp Model for {DISPLAY_NAME}", + ) + + out, _ = capsys.readouterr() + assert "response:" in out + + # Save resource name of the newly created training pipeline + shared_state["training_pipeline_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_training_pipeline_sample.py b/samples/snippets/create_training_pipeline_sample.py new file mode 100644 index 0000000000..eddcf7b775 --- /dev/null +++ b/samples/snippets/create_training_pipeline_sample.py @@ -0,0 +1,56 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_training_pipeline_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_training_pipeline_sample( + project: str, + display_name: str, + training_task_definition: str, + dataset_id: str, + model_display_name: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) + training_task_inputs_dict = { + "multiLabel": True, + "modelType": "CLOUD", + "budgetMilliNodeHours": 8000, + "disableEarlyStopping": False, + } + training_task_inputs = json_format.ParseDict(training_task_inputs_dict, Value()) + + training_pipeline = { + "display_name": display_name, + "training_task_definition": training_task_definition, + "training_task_inputs": training_task_inputs, + "input_data_config": {"dataset_id": dataset_id}, + "model_to_upload": {"display_name": model_display_name}, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_training_pipeline( + parent=parent, training_pipeline=training_pipeline + ) + print("response:", response) + + +# [END aiplatform_create_training_pipeline_sample] diff --git a/samples/snippets/create_training_pipeline_sample_test.py b/samples/snippets/create_training_pipeline_sample_test.py new file mode 100644 index 0000000000..1903ff4ce3 --- /dev/null +++ b/samples/snippets/create_training_pipeline_sample_test.py @@ -0,0 +1,81 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 +import pytest +import os + +import helpers + +import create_training_pipeline_sample +import cancel_training_pipeline_sample +import delete_training_pipeline_sample + +from google.cloud import aiplatform + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +DATASET_ID = "1084241610289446912" # Permanent 50 Flowers Dataset +DISPLAY_NAME = f"temp_create_training_pipeline_test_{uuid4()}" +TRAINING_DEFINITION_GCS_PATH = "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_image_classification_1.0.0.yaml" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + training_pipeline_id = shared_state["training_pipeline_name"].split("/")[-1] + + # Stop the training pipeline + cancel_training_pipeline_sample.cancel_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + pipeline_client = aiplatform.gapic.PipelineServiceClient( + client_options={"api_endpoint": "us-central1-aiplatform.googleapis.com"} + ) + + # Waiting for training pipeline to be in CANCELLED state + helpers.wait_for_job_state( + get_job_method=pipeline_client.get_training_pipeline, + name=shared_state["training_pipeline_name"], + ) + + # Delete the training pipeline + delete_training_pipeline_sample.delete_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + +# Training AutoML Vision Model +def test_ucaip_generated_create_training_pipeline_sample(capsys, shared_state): + + create_training_pipeline_sample.create_training_pipeline_sample( + project=PROJECT_ID, + display_name=DISPLAY_NAME, + training_task_definition=TRAINING_DEFINITION_GCS_PATH, + dataset_id=DATASET_ID, + model_display_name=f"Temp Model for {DISPLAY_NAME}", + ) + + out, _ = capsys.readouterr() + assert "response:" in out + + # Save resource name of the newly created training pipeline + shared_state["training_pipeline_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_training_pipeline_tabular_classification_sample.py b/samples/snippets/create_training_pipeline_tabular_classification_sample.py new file mode 100644 index 0000000000..717e499e5a --- /dev/null +++ b/samples/snippets/create_training_pipeline_tabular_classification_sample.py @@ -0,0 +1,85 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_training_pipeline_tabular_classification_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_training_pipeline_tabular_classification_sample( + project: str, + display_name: str, + dataset_id: str, + model_display_name: str, + target_column: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) + # set the columns used for training and their data types + transformations = [ + {"auto": {"column_name": "sepal_width"}}, + {"auto": {"column_name": "sepal_length"}}, + {"auto": {"column_name": "petal_length"}}, + {"auto": {"column_name": "petal_width"}}, + ] + + training_task_inputs_dict = { + # required inputs + "targetColumn": target_column, + "predictionType": "classification", + "transformations": transformations, + "trainBudgetMilliNodeHours": 8000, + # optional inputs + "disableEarlyStopping": False, + # supported binary classification optimisation objectives: + # maximize-au-roc, minimize-log-loss, maximize-au-prc, + # maximize-precision-at-recall, maximize-recall-at-precision + # supported multi-class classification optimisation objective: + # minimize-log-loss + "optimizationObjective": "minimize-log-loss", + # possibly required inputs + # required when using maximize-precision-at-recall + # "optimizationObjectiveRecallValue": 0.5, # 0.0 - 1.0 + # required when using maximize-recall-at-precision + # "optimizationObjectivePrecisionValue": 0.5, # 0.0 - 1.0 + } + training_task_inputs = json_format.ParseDict(training_task_inputs_dict, Value()) + + training_pipeline = { + "display_name": display_name, + "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_tabular_1.0.0.yaml", + "training_task_inputs": training_task_inputs, + "input_data_config": { + "dataset_id": dataset_id, + "fraction_split": { + "training_fraction": 0.8, + "validation_fraction": 0.1, + "test_fraction": 0.1, + }, + }, + "model_to_upload": {"display_name": model_display_name}, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_training_pipeline( + parent=parent, training_pipeline=training_pipeline + ) + print("response:", response) + + +# [END aiplatform_create_training_pipeline_tabular_classification_sample] diff --git a/samples/snippets/create_training_pipeline_tabular_classification_sample_test.py b/samples/snippets/create_training_pipeline_tabular_classification_sample_test.py new file mode 100644 index 0000000000..66cedbbd09 --- /dev/null +++ b/samples/snippets/create_training_pipeline_tabular_classification_sample_test.py @@ -0,0 +1,82 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 +import pytest +import os + +import helpers + +import create_training_pipeline_tabular_classification_sample +import cancel_training_pipeline_sample +import delete_training_pipeline_sample + +from google.cloud import aiplatform + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +DATASET_ID = "2438839935709478912" # iris 1000 +DISPLAY_NAME = f"temp_create_training_pipeline_test_{uuid4()}" +TARGET_COLUMN = "species" +PREDICTION_TYPE = "classification" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + training_pipeline_id = shared_state["training_pipeline_name"].split("/")[-1] + + # Stop the training pipeline + cancel_training_pipeline_sample.cancel_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + client_options = {"api_endpoint": "us-central1-aiplatform.googleapis.com"} + pipeline_client = aiplatform.gapic.PipelineServiceClient( + client_options=client_options + ) + + # Waiting for training pipeline to be in CANCELLED state + helpers.wait_for_job_state( + get_job_method=pipeline_client.get_training_pipeline, + name=shared_state["training_pipeline_name"], + ) + + # Delete the training pipeline + delete_training_pipeline_sample.delete_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + +def test_ucaip_generated_create_training_pipeline_sample(capsys, shared_state): + + create_training_pipeline_tabular_classification_sample.create_training_pipeline_tabular_classification_sample( + project=PROJECT_ID, + display_name=DISPLAY_NAME, + dataset_id=DATASET_ID, + model_display_name=f"Temp Model for {DISPLAY_NAME}", + target_column=TARGET_COLUMN, + ) + + out, _ = capsys.readouterr() + assert "response:" in out + + # Save resource name of the newly created training pipeline + shared_state["training_pipeline_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_training_pipeline_tabular_regression_sample.py b/samples/snippets/create_training_pipeline_tabular_regression_sample.py new file mode 100644 index 0000000000..d2583bc706 --- /dev/null +++ b/samples/snippets/create_training_pipeline_tabular_regression_sample.py @@ -0,0 +1,101 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_training_pipeline_tabular_regression_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_training_pipeline_tabular_regression_sample( + project: str, + display_name: str, + dataset_id: str, + model_display_name: str, + target_column: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) + # set the columns used for training and their data types + transformations = [ + {"auto": {"column_name": "STRING_5000unique_NULLABLE"}}, + {"auto": {"column_name": "INTEGER_5000unique_NULLABLE"}}, + {"auto": {"column_name": "FLOAT_5000unique_NULLABLE"}}, + {"auto": {"column_name": "FLOAT_5000unique_REPEATED"}}, + {"auto": {"column_name": "NUMERIC_5000unique_NULLABLE"}}, + {"auto": {"column_name": "BOOLEAN_2unique_NULLABLE"}}, + { + "timestamp": { + "column_name": "TIMESTAMP_1unique_NULLABLE", + "invalid_values_allowed": True, + } + }, + {"auto": {"column_name": "DATE_1unique_NULLABLE"}}, + {"auto": {"column_name": "TIME_1unique_NULLABLE"}}, + { + "timestamp": { + "column_name": "DATETIME_1unique_NULLABLE", + "invalid_values_allowed": True, + } + }, + {"auto": {"column_name": "STRUCT_NULLABLE.STRING_5000unique_NULLABLE"}}, + {"auto": {"column_name": "STRUCT_NULLABLE.INTEGER_5000unique_NULLABLE"}}, + {"auto": {"column_name": "STRUCT_NULLABLE.FLOAT_5000unique_NULLABLE"}}, + {"auto": {"column_name": "STRUCT_NULLABLE.FLOAT_5000unique_REQUIRED"}}, + {"auto": {"column_name": "STRUCT_NULLABLE.FLOAT_5000unique_REPEATED"}}, + {"auto": {"column_name": "STRUCT_NULLABLE.NUMERIC_5000unique_NULLABLE"}}, + {"auto": {"column_name": "STRUCT_NULLABLE.BOOLEAN_2unique_NULLABLE"}}, + {"auto": {"column_name": "STRUCT_NULLABLE.TIMESTAMP_1unique_NULLABLE"}}, + ] + + training_task_inputs_dict = { + # required inputs + "targetColumn": target_column, + "predictionType": "regression", + "transformations": transformations, + "trainBudgetMilliNodeHours": 8000, + # optional inputs + "disableEarlyStopping": False, + # supported regression optimisation objectives: minimize-rmse, + # minimize-mae, minimize-rmsle + "optimizationObjective": "minimize-rmse", + } + training_task_inputs = json_format.ParseDict(training_task_inputs_dict, Value()) + + training_pipeline = { + "display_name": display_name, + "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_tabular_1.0.0.yaml", + "training_task_inputs": training_task_inputs, + "input_data_config": { + "dataset_id": dataset_id, + "fraction_split": { + "training_fraction": 0.8, + "validation_fraction": 0.1, + "test_fraction": 0.1, + }, + }, + "model_to_upload": {"display_name": model_display_name}, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_training_pipeline( + parent=parent, training_pipeline=training_pipeline + ) + print("response:", response) + + +# [END aiplatform_create_training_pipeline_tabular_regression_sample] diff --git a/samples/snippets/create_training_pipeline_tabular_regression_sample_test.py b/samples/snippets/create_training_pipeline_tabular_regression_sample_test.py new file mode 100644 index 0000000000..1b43876902 --- /dev/null +++ b/samples/snippets/create_training_pipeline_tabular_regression_sample_test.py @@ -0,0 +1,81 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 +import pytest +import os + +import helpers + +import create_training_pipeline_tabular_regression_sample +import cancel_training_pipeline_sample +import delete_training_pipeline_sample + +from google.cloud import aiplatform + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +DATASET_ID = "3019804287640272896" # bq all +DISPLAY_NAME = f"temp_create_training_pipeline_test_{uuid4()}" +TARGET_COLUMN = "FLOAT_5000unique_REQUIRED" +PREDICTION_TYPE = "regression" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + training_pipeline_id = shared_state["training_pipeline_name"].split("/")[-1] + + # Stop the training pipeline + cancel_training_pipeline_sample.cancel_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + pipeline_client = aiplatform.gapic.PipelineServiceClient( + client_options={"api_endpoint": "us-central1-aiplatform.googleapis.com"} + ) + + # Waiting for training pipeline to be in CANCELLED state + helpers.wait_for_job_state( + get_job_method=pipeline_client.get_training_pipeline, + name=shared_state["training_pipeline_name"], + ) + + # Delete the training pipeline + delete_training_pipeline_sample.delete_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + +def test_ucaip_generated_create_training_pipeline_sample(capsys, shared_state): + + create_training_pipeline_tabular_regression_sample.create_training_pipeline_tabular_regression_sample( + project=PROJECT_ID, + display_name=DISPLAY_NAME, + dataset_id=DATASET_ID, + model_display_name=f"Temp Model for {DISPLAY_NAME}", + target_column=TARGET_COLUMN, + ) + + out, _ = capsys.readouterr() + assert "response:" in out + + # Save resource name of the newly created training pipeline + shared_state["training_pipeline_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_training_pipeline_text_classification_sample.py b/samples/snippets/create_training_pipeline_text_classification_sample.py new file mode 100644 index 0000000000..bc58db2b15 --- /dev/null +++ b/samples/snippets/create_training_pipeline_text_classification_sample.py @@ -0,0 +1,50 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_training_pipeline_text_classification_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_training_pipeline_text_classification_sample( + project: str, + display_name: str, + dataset_id: str, + model_display_name: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) + training_task_inputs_dict = {} + training_task_inputs = json_format.ParseDict(training_task_inputs_dict, Value()) + + training_pipeline = { + "display_name": display_name, + "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_text_classification_1.0.0.yaml", + "training_task_inputs": training_task_inputs, + "input_data_config": {"dataset_id": dataset_id}, + "model_to_upload": {"display_name": model_display_name}, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_training_pipeline( + parent=parent, training_pipeline=training_pipeline + ) + print("response:", response) + + +# [END aiplatform_create_training_pipeline_text_classification_sample] diff --git a/samples/snippets/create_training_pipeline_text_entity_extraction_sample.py b/samples/snippets/create_training_pipeline_text_entity_extraction_sample.py new file mode 100644 index 0000000000..3140ef76ec --- /dev/null +++ b/samples/snippets/create_training_pipeline_text_entity_extraction_sample.py @@ -0,0 +1,51 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_training_pipeline_text_entity_extraction_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_training_pipeline_text_entity_extraction_sample( + project: str, + display_name: str, + dataset_id: str, + model_display_name: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) + training_task_inputs_dict = {} + training_task_inputs = json_format.ParseDict(training_task_inputs_dict, Value()) + + training_pipeline = { + "display_name": display_name, + "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_text_extraction_1.0.0.yaml", + # Training task inputs are empty for text entity extraction + "training_task_inputs": training_task_inputs, + "input_data_config": {"dataset_id": dataset_id}, + "model_to_upload": {"display_name": model_display_name}, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_training_pipeline( + parent=parent, training_pipeline=training_pipeline + ) + print("response:", response) + + +# [END aiplatform_create_training_pipeline_text_entity_extraction_sample] diff --git a/samples/snippets/create_training_pipeline_text_entity_extraction_sample_test.py b/samples/snippets/create_training_pipeline_text_entity_extraction_sample_test.py new file mode 100644 index 0000000000..a904d3d114 --- /dev/null +++ b/samples/snippets/create_training_pipeline_text_entity_extraction_sample_test.py @@ -0,0 +1,81 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 +import pytest +import os + +import helpers + +import create_training_pipeline_text_entity_extraction_sample +import cancel_training_pipeline_sample +import delete_training_pipeline_sample + +from google.cloud import aiplatform + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +DATASET_ID = "6203215905493614592" # Permanent text entity extraction dataset +DISPLAY_NAME = f"temp_create_training_pipeline_ten_test_{uuid4()}" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + training_pipeline_id = shared_state["training_pipeline_name"].split("/")[-1] + + # Stop the training pipeline + cancel_training_pipeline_sample.cancel_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + pipeline_client = aiplatform.gapic.PipelineServiceClient( + client_options={"api_endpoint": "us-central1-aiplatform.googleapis.com"} + ) + + # Waiting for training pipeline to be in CANCELLED state + helpers.wait_for_job_state( + get_job_method=pipeline_client.get_training_pipeline, + name=shared_state["training_pipeline_name"], + ) + + # Delete the training pipeline + delete_training_pipeline_sample.delete_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + +# Training Text Entity Extraction Model +def test_ucaip_generated_create_training_pipeline_text_entity_extraction_sample( + capsys, shared_state +): + + create_training_pipeline_text_entity_extraction_sample.create_training_pipeline_text_entity_extraction_sample( + project=PROJECT_ID, + display_name=DISPLAY_NAME, + dataset_id=DATASET_ID, + model_display_name=f"Temp Model for {DISPLAY_NAME}", + ) + + out, _ = capsys.readouterr() + assert "response:" in out + + # Save resource name of the newly created training pipeline + shared_state["training_pipeline_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_training_pipeline_text_sentiment_analysis_sample.py b/samples/snippets/create_training_pipeline_text_sentiment_analysis_sample.py new file mode 100644 index 0000000000..1908828e4b --- /dev/null +++ b/samples/snippets/create_training_pipeline_text_sentiment_analysis_sample.py @@ -0,0 +1,51 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_training_pipeline_text_sentiment_analysis_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_training_pipeline_text_sentiment_analysis_sample( + project: str, + display_name: str, + dataset_id: str, + model_display_name: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) + # Use sentiment_max of 4 + training_task_inputs_dict = {"sentiment_max": 4} + training_task_inputs = json_format.ParseDict(training_task_inputs_dict, Value()) + + training_pipeline = { + "display_name": display_name, + "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_text_sentiment_1.0.0.yaml", + "training_task_inputs": training_task_inputs, + "input_data_config": {"dataset_id": dataset_id}, + "model_to_upload": {"display_name": model_display_name}, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_training_pipeline( + parent=parent, training_pipeline=training_pipeline + ) + print("response:", response) + + +# [END aiplatform_create_training_pipeline_text_sentiment_analysis_sample] diff --git a/samples/snippets/create_training_pipeline_text_sentiment_analysis_sample_test.py b/samples/snippets/create_training_pipeline_text_sentiment_analysis_sample_test.py new file mode 100644 index 0000000000..cb86ce3c24 --- /dev/null +++ b/samples/snippets/create_training_pipeline_text_sentiment_analysis_sample_test.py @@ -0,0 +1,81 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 +import pytest +import os + +import helpers + +import create_training_pipeline_text_sentiment_analysis_sample +import cancel_training_pipeline_sample +import delete_training_pipeline_sample + +from google.cloud import aiplatform + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +DATASET_ID = "5148529167758786560" # Permanent text sentiment analysis dataset +DISPLAY_NAME = f"temp_create_training_pipeline_tsa_test_{uuid4()}" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + training_pipeline_id = shared_state["training_pipeline_name"].split("/")[-1] + + # Stop the training pipeline + cancel_training_pipeline_sample.cancel_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + pipeline_client = aiplatform.gapic.PipelineServiceClient( + client_options={"api_endpoint": "us-central1-aiplatform.googleapis.com"} + ) + + # Waiting for training pipeline to be in CANCELLED state + helpers.wait_for_job_state( + get_job_method=pipeline_client.get_training_pipeline, + name=shared_state["training_pipeline_name"], + ) + + # Delete the training pipeline + delete_training_pipeline_sample.delete_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + +# Training Text Sentiment Analysis Model +def test_ucaip_generated_create_training_pipeline_text_sentiment_analysis_sample( + capsys, shared_state +): + + create_training_pipeline_text_sentiment_analysis_sample.create_training_pipeline_text_sentiment_analysis_sample( + project=PROJECT_ID, + display_name=DISPLAY_NAME, + dataset_id=DATASET_ID, + model_display_name=f"Temp Model for {DISPLAY_NAME}", + ) + + out, _ = capsys.readouterr() + assert "response:" in out + + # Save resource name of the newly created training pipeline + shared_state["training_pipeline_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_training_pipeline_video_classification_sample.py b/samples/snippets/create_training_pipeline_video_classification_sample.py new file mode 100644 index 0000000000..6d427d8b78 --- /dev/null +++ b/samples/snippets/create_training_pipeline_video_classification_sample.py @@ -0,0 +1,51 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_training_pipeline_video_classification_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_training_pipeline_video_classification_sample( + project: str, + display_name: str, + dataset_id: str, + model_display_name: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) + training_task_inputs_dict = {} + training_task_inputs = json_format.ParseDict(training_task_inputs_dict, Value()) + + training_pipeline = { + "display_name": display_name, + "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_video_classification_1.0.0.yaml", + # Training task inputs are empty for video classification + "training_task_inputs": training_task_inputs, + "input_data_config": {"dataset_id": dataset_id}, + "model_to_upload": {"display_name": model_display_name}, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_training_pipeline( + parent=parent, training_pipeline=training_pipeline + ) + print("response:", response) + + +# [END aiplatform_create_training_pipeline_video_classification_sample] diff --git a/samples/snippets/create_training_pipeline_video_classification_sample_test.py b/samples/snippets/create_training_pipeline_video_classification_sample_test.py new file mode 100644 index 0000000000..630a3830b6 --- /dev/null +++ b/samples/snippets/create_training_pipeline_video_classification_sample_test.py @@ -0,0 +1,81 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 +import pytest +import os + +import helpers + +import create_training_pipeline_video_classification_sample +import cancel_training_pipeline_sample +import delete_training_pipeline_sample + +from google.cloud import aiplatform + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +DATASET_ID = "3757409464110546944" # Permanent 5 class sports dataset +DISPLAY_NAME = f"temp_create_training_pipeline_vcn_test_{uuid4()}" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + training_pipeline_id = shared_state["training_pipeline_name"].split("/")[-1] + + # Stop the training pipeline + cancel_training_pipeline_sample.cancel_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + pipeline_client = aiplatform.gapic.PipelineServiceClient( + client_options={"api_endpoint": "us-central1-aiplatform.googleapis.com"} + ) + + # Waiting for training pipeline to be in CANCELLED state + helpers.wait_for_job_state( + get_job_method=pipeline_client.get_training_pipeline, + name=shared_state["training_pipeline_name"], + ) + + # Delete the training pipeline + delete_training_pipeline_sample.delete_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + +# Training AutoML Vision Model +def test_ucaip_generated_create_training_pipeline_video_classification_sample( + capsys, shared_state +): + + create_training_pipeline_video_classification_sample.create_training_pipeline_video_classification_sample( + project=PROJECT_ID, + display_name=DISPLAY_NAME, + dataset_id=DATASET_ID, + model_display_name=f"Temp Model for {DISPLAY_NAME}", + ) + + out, _ = capsys.readouterr() + assert "response:" in out + + # Save resource name of the newly created training pipeline + shared_state["training_pipeline_name"] = helpers.get_name(out) diff --git a/samples/snippets/create_training_pipeline_video_object_tracking_sample.py b/samples/snippets/create_training_pipeline_video_object_tracking_sample.py new file mode 100644 index 0000000000..03d3bbeb11 --- /dev/null +++ b/samples/snippets/create_training_pipeline_video_object_tracking_sample.py @@ -0,0 +1,50 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_create_training_pipeline_video_object_tracking_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def create_training_pipeline_video_object_tracking_sample( + project: str, + display_name: str, + dataset_id: str, + model_display_name: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) + training_task_inputs_dict = {"modelType": "CLOUD"} + training_task_inputs = json_format.ParseDict(training_task_inputs_dict, Value()) + + training_pipeline = { + "display_name": display_name, + "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_video_object_tracking_1.0.0.yaml", + "training_task_inputs": training_task_inputs, + "input_data_config": {"dataset_id": dataset_id}, + "model_to_upload": {"display_name": model_display_name}, + } + parent = f"projects/{project}/locations/{location}" + response = client.create_training_pipeline( + parent=parent, training_pipeline=training_pipeline + ) + print("response:", response) + + +# [END aiplatform_create_training_pipeline_video_object_tracking_sample] diff --git a/samples/snippets/create_training_pipeline_video_object_tracking_sample_test.py b/samples/snippets/create_training_pipeline_video_object_tracking_sample_test.py new file mode 100644 index 0000000000..1c1eacb4ef --- /dev/null +++ b/samples/snippets/create_training_pipeline_video_object_tracking_sample_test.py @@ -0,0 +1,81 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 +import pytest +import os + +import helpers + +import create_training_pipeline_video_object_tracking_sample +import cancel_training_pipeline_sample +import delete_training_pipeline_sample + +from google.cloud import aiplatform + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +DATASET_ID = "1138566280794603520" # Permanent 40 horses dataset +DISPLAY_NAME = f"temp_create_training_pipeline_vot_test_{uuid4()}" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + training_pipeline_id = shared_state["training_pipeline_name"].split("/")[-1] + + # Stop the training pipeline + cancel_training_pipeline_sample.cancel_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + pipeline_client = aiplatform.gapic.PipelineServiceClient( + client_options={"api_endpoint": "us-central1-aiplatform.googleapis.com"} + ) + + # Waiting for training pipeline to be in CANCELLED state + helpers.wait_for_job_state( + get_job_method=pipeline_client.get_training_pipeline, + name=shared_state["training_pipeline_name"], + ) + + # Delete the training pipeline + delete_training_pipeline_sample.delete_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=training_pipeline_id + ) + + +# Training AutoML Vision Model +def test_ucaip_generated_create_training_pipeline_video_object_tracking_sample( + capsys, shared_state +): + + create_training_pipeline_video_object_tracking_sample.create_training_pipeline_video_object_tracking_sample( + project=PROJECT_ID, + display_name=DISPLAY_NAME, + dataset_id=DATASET_ID, + model_display_name=f"Temp Model for {DISPLAY_NAME}", + ) + + out, _ = capsys.readouterr() + assert "response:" in out + + # Save resource name of the newly created training pipeline + shared_state["training_pipeline_name"] = helpers.get_name(out) diff --git a/samples/snippets/delete_batch_prediction_job_sample.py b/samples/snippets/delete_batch_prediction_job_sample.py new file mode 100644 index 0000000000..51121d2c5e --- /dev/null +++ b/samples/snippets/delete_batch_prediction_job_sample.py @@ -0,0 +1,39 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_delete_batch_prediction_job_sample] +from google.cloud import aiplatform + + +def delete_batch_prediction_job_sample( + project: str, + batch_prediction_job_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + name = client.batch_prediction_job_path( + project=project, location=location, batch_prediction_job=batch_prediction_job_id + ) + response = client.delete_batch_prediction_job(name=name) + print("Long running operation:", response.operation.name) + delete_batch_prediction_job_response = response.result(timeout=timeout) + print("delete_batch_prediction_job_response:", delete_batch_prediction_job_response) + + +# [END aiplatform_delete_batch_prediction_job_sample] diff --git a/samples/snippets/delete_custom_job_sample.py b/samples/snippets/delete_custom_job_sample.py new file mode 100644 index 0000000000..a5bb0f18fe --- /dev/null +++ b/samples/snippets/delete_custom_job_sample.py @@ -0,0 +1,39 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_delete_custom_job_sample] +from google.cloud import aiplatform + + +def delete_custom_job_sample( + project: str, + custom_job_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + name = client.custom_job_path( + project=project, location=location, custom_job=custom_job_id + ) + response = client.delete_custom_job(name=name) + print("Long running operation:", response.operation.name) + delete_custom_job_response = response.result(timeout=timeout) + print("delete_custom_job_response:", delete_custom_job_response) + + +# [END aiplatform_delete_custom_job_sample] diff --git a/samples/snippets/delete_data_labeling_job_sample.py b/samples/snippets/delete_data_labeling_job_sample.py new file mode 100644 index 0000000000..4917886210 --- /dev/null +++ b/samples/snippets/delete_data_labeling_job_sample.py @@ -0,0 +1,39 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_delete_data_labeling_job_sample] +from google.cloud import aiplatform + + +def delete_data_labeling_job_sample( + project: str, + data_labeling_job_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + name = client.data_labeling_job_path( + project=project, location=location, data_labeling_job=data_labeling_job_id + ) + response = client.delete_data_labeling_job(name=name) + print("Long running operation:", response.operation.name) + delete_data_labeling_job_response = response.result(timeout=timeout) + print("delete_data_labeling_job_response:", delete_data_labeling_job_response) + + +# [END aiplatform_delete_data_labeling_job_sample] diff --git a/samples/snippets/delete_dataset_sample.py b/samples/snippets/delete_dataset_sample.py new file mode 100644 index 0000000000..722a49a5cf --- /dev/null +++ b/samples/snippets/delete_dataset_sample.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_delete_dataset_sample] +from google.cloud import aiplatform + + +def delete_dataset_sample( + project: str, + dataset_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + name = client.dataset_path(project=project, location=location, dataset=dataset_id) + response = client.delete_dataset(name=name) + print("Long running operation:", response.operation.name) + delete_dataset_response = response.result(timeout=timeout) + print("delete_dataset_response:", delete_dataset_response) + + +# [END aiplatform_delete_dataset_sample] diff --git a/samples/snippets/delete_endpoint_sample.py b/samples/snippets/delete_endpoint_sample.py new file mode 100644 index 0000000000..b0ce2589e1 --- /dev/null +++ b/samples/snippets/delete_endpoint_sample.py @@ -0,0 +1,39 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_delete_endpoint_sample] +from google.cloud import aiplatform + + +def delete_endpoint_sample( + project: str, + endpoint_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.EndpointServiceClient(client_options=client_options) + name = client.endpoint_path( + project=project, location=location, endpoint=endpoint_id + ) + response = client.delete_endpoint(name=name) + print("Long running operation:", response.operation.name) + delete_endpoint_response = response.result(timeout=timeout) + print("delete_endpoint_response:", delete_endpoint_response) + + +# [END aiplatform_delete_endpoint_sample] diff --git a/samples/snippets/delete_hyperparameter_tuning_job_sample.py b/samples/snippets/delete_hyperparameter_tuning_job_sample.py new file mode 100644 index 0000000000..4bbd651227 --- /dev/null +++ b/samples/snippets/delete_hyperparameter_tuning_job_sample.py @@ -0,0 +1,44 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_delete_hyperparameter_tuning_job_sample] +from google.cloud import aiplatform + + +def delete_hyperparameter_tuning_job_sample( + project: str, + hyperparameter_tuning_job_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + name = client.hyperparameter_tuning_job_path( + project=project, + location=location, + hyperparameter_tuning_job=hyperparameter_tuning_job_id, + ) + response = client.delete_hyperparameter_tuning_job(name=name) + print("Long running operation:", response.operation.name) + delete_hyperparameter_tuning_job_response = response.result(timeout=timeout) + print( + "delete_hyperparameter_tuning_job_response:", + delete_hyperparameter_tuning_job_response, + ) + + +# [END aiplatform_delete_hyperparameter_tuning_job_sample] diff --git a/samples/snippets/delete_model_sample.py b/samples/snippets/delete_model_sample.py new file mode 100644 index 0000000000..7ecbe681db --- /dev/null +++ b/samples/snippets/delete_model_sample.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_delete_model_sample] +from google.cloud import aiplatform + + +def delete_model_sample( + project: str, + model_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + name = client.model_path(project=project, location=location, model=model_id) + response = client.delete_model(name=name) + print("Long running operation:", response.operation.name) + delete_model_response = response.result(timeout=timeout) + print("delete_model_response:", delete_model_response) + + +# [END aiplatform_delete_model_sample] diff --git a/samples/snippets/delete_training_pipeline_sample.py b/samples/snippets/delete_training_pipeline_sample.py new file mode 100644 index 0000000000..08dfec20a4 --- /dev/null +++ b/samples/snippets/delete_training_pipeline_sample.py @@ -0,0 +1,39 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_delete_training_pipeline_sample] +from google.cloud import aiplatform + + +def delete_training_pipeline_sample( + project: str, + training_pipeline_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) + name = client.training_pipeline_path( + project=project, location=location, training_pipeline=training_pipeline_id + ) + response = client.delete_training_pipeline(name=name) + print("Long running operation:", response.operation.name) + delete_training_pipeline_response = response.result(timeout=timeout) + print("delete_training_pipeline_response:", delete_training_pipeline_response) + + +# [END aiplatform_delete_training_pipeline_sample] diff --git a/samples/snippets/deploy_model_sample.py b/samples/snippets/deploy_model_sample.py new file mode 100644 index 0000000000..1cb8bef552 --- /dev/null +++ b/samples/snippets/deploy_model_sample.py @@ -0,0 +1,55 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_deploy_model_sample] +from google.cloud import aiplatform + + +def deploy_model_sample( + project: str, + endpoint_id: str, + model_name: str, + deployed_model_display_name: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 7200, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.EndpointServiceClient(client_options=client_options) + deployed_model = { + # format: 'projects/{project}/locations/{location}/models/{model}' + "model": model_name, + "display_name": deployed_model_display_name, + # AutoML Vision models require `automatic_resources` field + # Other model types may require `dedicated_resources` field instead + "automatic_resources": {"min_replica_count": 1, "max_replica_count": 1}, + } + # key '0' assigns traffic for the newly deployed model + # Traffic percentage values must add up to 100 + # Leave dictionary empty if endpoint should not accept any traffic + traffic_split = {"0": 100} + endpoint = client.endpoint_path( + project=project, location=location, endpoint=endpoint_id + ) + response = client.deploy_model( + endpoint=endpoint, deployed_model=deployed_model, traffic_split=traffic_split + ) + print("Long running operation:", response.operation.name) + deploy_model_response = response.result(timeout=timeout) + print("deploy_model_response:", deploy_model_response) + + +# [END aiplatform_deploy_model_sample] diff --git a/samples/snippets/deploy_model_sample_test.py b/samples/snippets/deploy_model_sample_test.py new file mode 100644 index 0000000000..2960a7f3d3 --- /dev/null +++ b/samples/snippets/deploy_model_sample_test.py @@ -0,0 +1,85 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import aiplatform +import deploy_model_sample, delete_endpoint_sample + +from uuid import uuid4 +import pytest +import os + +import helpers + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +LOCATION = "us-central1" +PARENT = f"projects/{PROJECT_ID}/locations/{LOCATION}" +DISPLAY_NAME = f"temp_deploy_model_test_{uuid4()}" + +# Resource Name of "permanent_50_flowers_new_model" +MODEL_NAME = "projects/580378083368/locations/us-central1/models/4190810559500779520" +CLIENT_OPTIONS = {"api_endpoint": "us-central1-aiplatform.googleapis.com"} + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def setup(shared_state): + + # Create an temporary endpoint and store resource name + shared_state["endpoint_client"] = aiplatform.gapic.EndpointServiceClient( + client_options=CLIENT_OPTIONS + ) + create_endpoint_response = shared_state["endpoint_client"].create_endpoint( + parent=PARENT, endpoint={"display_name": DISPLAY_NAME} + ) + shared_state["endpoint"] = create_endpoint_response.result().name + + +def test_ucaip_generated_deploy_model_sample(capsys, shared_state): + + assert shared_state["endpoint"] is not None + + # Deploy existing image classification model to endpoint + deploy_model_sample.deploy_model_sample( + project=PROJECT_ID, + model_name=MODEL_NAME, + deployed_model_display_name=DISPLAY_NAME, + endpoint_id=shared_state["endpoint"].split("/")[-1], + ) + + # Store deployed model ID for undeploying + out, _ = capsys.readouterr() + assert "deploy_model_response" in out + + shared_state["deployed_model_id"] = helpers.get_name(out=out, key="id") + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + undeploy_model_operation = shared_state["endpoint_client"].undeploy_model( + deployed_model_id=shared_state["deployed_model_id"], + endpoint=shared_state["endpoint"], + ) + undeploy_model_operation.result() + + # Delete the endpoint + delete_endpoint_sample.delete_endpoint_sample( + project=PROJECT_ID, endpoint_id=shared_state["endpoint"].split("/")[-1] + ) diff --git a/samples/snippets/export_model_sample.py b/samples/snippets/export_model_sample.py new file mode 100644 index 0000000000..ec083c778e --- /dev/null +++ b/samples/snippets/export_model_sample.py @@ -0,0 +1,41 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_export_model_sample] +from google.cloud import aiplatform + + +def export_model_sample( + project: str, + model_id: str, + gcs_destination_output_uri_prefix: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + output_config = { + "artifact_destination": {"output_uri_prefix": gcs_destination_output_uri_prefix} + } + name = client.model_path(project=project, location=location, model=model_id) + response = client.export_model(name=name, output_config=output_config) + print("Long running operation:", response.operation.name) + export_model_response = response.result(timeout=timeout) + print("export_model_response:", export_model_response) + + +# [END aiplatform_export_model_sample] diff --git a/samples/snippets/export_model_tabular_classification_sample.py b/samples/snippets/export_model_tabular_classification_sample.py new file mode 100644 index 0000000000..5d1c0d1455 --- /dev/null +++ b/samples/snippets/export_model_tabular_classification_sample.py @@ -0,0 +1,43 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_export_model_tabular_classification_sample] +from google.cloud import aiplatform + + +def export_model_tabular_classification_sample( + project: str, + model_id: str, + gcs_destination_output_uri_prefix: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 300, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + gcs_destination = {"output_uri_prefix": gcs_destination_output_uri_prefix} + output_config = { + "artifact_destination": gcs_destination, + "export_format_id": "tf-saved-model", + } + name = client.model_path(project=project, location=location, model=model_id) + response = client.export_model(name=name, output_config=output_config) + print("Long running operation:", response.operation.name) + export_model_response = response.result(timeout=timeout) + print("export_model_response:", export_model_response) + + +# [END aiplatform_export_model_tabular_classification_sample] diff --git a/samples/snippets/export_model_tabular_classification_sample_test.py b/samples/snippets/export_model_tabular_classification_sample_test.py new file mode 100644 index 0000000000..b0b0c7e011 --- /dev/null +++ b/samples/snippets/export_model_tabular_classification_sample_test.py @@ -0,0 +1,47 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os + +from uuid import uuid4 + +import export_model_tabular_classification_sample +from google.cloud import storage + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +MODEL_ID = "5359002081594179584" # iris 1000 +GCS_BUCKET = "gs://ucaip-samples-test-output" +GCS_PREFIX = f"tmp/export_model_test_{uuid4()}" + + +@pytest.fixture(scope="function", autouse=True) +def teardown(): + yield + + storage_client = storage.Client() + bucket = storage_client.get_bucket("ucaip-samples-test-output") + blobs = bucket.list_blobs(prefix=GCS_PREFIX) + for blob in blobs: + blob.delete() + + +def test_ucaip_generated_export_model_tabular_classification_sample(capsys): + export_model_tabular_classification_sample.export_model_tabular_classification_sample( + project=PROJECT_ID, + model_id=MODEL_ID, + gcs_destination_output_uri_prefix=f"{GCS_BUCKET}/{GCS_PREFIX}", + ) + out, _ = capsys.readouterr() + assert "export_model_response" in out diff --git a/samples/snippets/get_batch_prediction_job_sample.py b/samples/snippets/get_batch_prediction_job_sample.py new file mode 100644 index 0000000000..7e840b4a65 --- /dev/null +++ b/samples/snippets/get_batch_prediction_job_sample.py @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_get_batch_prediction_job_sample] +from google.cloud import aiplatform + + +def get_batch_prediction_job_sample( + project: str, + batch_prediction_job_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.JobServiceClient(client_options=client_options) + name = client.batch_prediction_job_path( + project=project, location=location, batch_prediction_job=batch_prediction_job_id + ) + response = client.get_batch_prediction_job(name=name) + print("response:", response) + + +# [END aiplatform_get_batch_prediction_job_sample] diff --git a/samples/snippets/get_model_evaluation_image_classification_sample.py b/samples/snippets/get_model_evaluation_image_classification_sample.py new file mode 100644 index 0000000000..8271f8ee69 --- /dev/null +++ b/samples/snippets/get_model_evaluation_image_classification_sample.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_get_model_evaluation_image_classification_sample] +from google.cloud import aiplatform + + +def get_model_evaluation_image_classification_sample( + project: str, + model_id: str, + evaluation_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + name = client.model_evaluation_path( + project=project, location=location, model=model_id, evaluation=evaluation_id + ) + response = client.get_model_evaluation(name=name) + print("response:", response) + + +# [END aiplatform_get_model_evaluation_image_classification_sample] diff --git a/samples/snippets/get_model_evaluation_image_object_detection_sample.py b/samples/snippets/get_model_evaluation_image_object_detection_sample.py new file mode 100644 index 0000000000..fb7a4f06b3 --- /dev/null +++ b/samples/snippets/get_model_evaluation_image_object_detection_sample.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_get_model_evaluation_image_object_detection_sample] +from google.cloud import aiplatform + + +def get_model_evaluation_image_object_detection_sample( + project: str, + model_id: str, + evaluation_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + name = client.model_evaluation_path( + project=project, location=location, model=model_id, evaluation=evaluation_id + ) + response = client.get_model_evaluation(name=name) + print("response:", response) + + +# [END aiplatform_get_model_evaluation_image_object_detection_sample] diff --git a/samples/snippets/get_model_evaluation_sample.py b/samples/snippets/get_model_evaluation_sample.py new file mode 100644 index 0000000000..8616e15e62 --- /dev/null +++ b/samples/snippets/get_model_evaluation_sample.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_get_model_evaluation_sample] +from google.cloud import aiplatform + + +def get_model_evaluation_sample( + project: str, + model_id: str, + evaluation_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + name = client.model_evaluation_path( + project=project, location=location, model=model_id, evaluation=evaluation_id + ) + response = client.get_model_evaluation(name=name) + print("response:", response) + + +# [END aiplatform_get_model_evaluation_sample] diff --git a/samples/snippets/get_model_evaluation_sample_test.py b/samples/snippets/get_model_evaluation_sample_test.py new file mode 100644 index 0000000000..bba69215ec --- /dev/null +++ b/samples/snippets/get_model_evaluation_sample_test.py @@ -0,0 +1,30 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os + +import get_model_evaluation_sample + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +MODEL_ID = "5162251072873431040" # permanent_safe_driver_model +EVALUATION_ID = "5615675837586029221" # permanent_safe_driver_model Evaluation + + +def test_ucaip_generated_get_model_evaluation_sample(capsys): + get_model_evaluation_sample.get_model_evaluation_sample( + project=PROJECT_ID, model_id=MODEL_ID, evaluation_id=EVALUATION_ID + ) + out, _ = capsys.readouterr() + assert "metrics_schema_uri" in out diff --git a/samples/snippets/get_model_evaluation_slice_sample.py b/samples/snippets/get_model_evaluation_slice_sample.py new file mode 100644 index 0000000000..f1d0f2f850 --- /dev/null +++ b/samples/snippets/get_model_evaluation_slice_sample.py @@ -0,0 +1,42 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_get_model_evaluation_slice_sample] +from google.cloud import aiplatform + + +def get_model_evaluation_slice_sample( + project: str, + model_id: str, + evaluation_id: str, + slice_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + name = client.model_evaluation_slice_path( + project=project, + location=location, + model=model_id, + evaluation=evaluation_id, + slice=slice_id, + ) + response = client.get_model_evaluation_slice(name=name) + print("response:", response) + + +# [END aiplatform_get_model_evaluation_slice_sample] diff --git a/samples/snippets/get_model_evaluation_slice_sample_test.py b/samples/snippets/get_model_evaluation_slice_sample_test.py new file mode 100644 index 0000000000..f8ed759f6d --- /dev/null +++ b/samples/snippets/get_model_evaluation_slice_sample_test.py @@ -0,0 +1,34 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os + +import get_model_evaluation_slice_sample + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +MODEL_ID = "5162251072873431040" # permanent_safe_driver_model +EVALUATION_ID = "5615675837586029221" # permanent_safe_driver_model Evaluation +SLICE_ID = "4322488217836113260" # permanent_safe_driver_model Eval Slice + + +def test_ucaip_generated_get_model_evaluation_slice_sample(capsys): + get_model_evaluation_slice_sample.get_model_evaluation_slice_sample( + project=PROJECT_ID, + model_id=MODEL_ID, + evaluation_id=EVALUATION_ID, + slice_id=SLICE_ID, + ) + out, _ = capsys.readouterr() + assert "metrics_schema_uri" in out diff --git a/samples/snippets/get_model_evaluation_tabular_classification_sample.py b/samples/snippets/get_model_evaluation_tabular_classification_sample.py new file mode 100644 index 0000000000..ade0345ac5 --- /dev/null +++ b/samples/snippets/get_model_evaluation_tabular_classification_sample.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_get_model_evaluation_tabular_classification_sample] +from google.cloud import aiplatform + + +def get_model_evaluation_tabular_classification_sample( + project: str, + model_id: str, + evaluation_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + name = client.model_evaluation_path( + project=project, location=location, model=model_id, evaluation=evaluation_id + ) + response = client.get_model_evaluation(name=name) + print("response:", response) + + +# [END aiplatform_get_model_evaluation_tabular_classification_sample] diff --git a/samples/snippets/get_model_evaluation_tabular_classification_sample_test.py b/samples/snippets/get_model_evaluation_tabular_classification_sample_test.py new file mode 100644 index 0000000000..dcfe5da069 --- /dev/null +++ b/samples/snippets/get_model_evaluation_tabular_classification_sample_test.py @@ -0,0 +1,30 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os + +import get_model_evaluation_tabular_classification_sample + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +MODEL_ID = "5162251072873431040" # permanent_safe_driver_model +EVALUATION_ID = "5615675837586029221" # permanent_safe_driver_model Evaluation + + +def test_ucaip_generated_get_model_evaluation_tabular_classification_sample(capsys): + get_model_evaluation_tabular_classification_sample.get_model_evaluation_tabular_classification_sample( + project=PROJECT_ID, model_id=MODEL_ID, evaluation_id=EVALUATION_ID + ) + out, _ = capsys.readouterr() + assert "metrics_schema_uri" in out diff --git a/samples/snippets/get_model_evaluation_tabular_regression_sample.py b/samples/snippets/get_model_evaluation_tabular_regression_sample.py new file mode 100644 index 0000000000..65d717fa3a --- /dev/null +++ b/samples/snippets/get_model_evaluation_tabular_regression_sample.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_get_model_evaluation_tabular_regression_sample] +from google.cloud import aiplatform + + +def get_model_evaluation_tabular_regression_sample( + project: str, + model_id: str, + evaluation_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + name = client.model_evaluation_path( + project=project, location=location, model=model_id, evaluation=evaluation_id + ) + response = client.get_model_evaluation(name=name) + print("response:", response) + + +# [END aiplatform_get_model_evaluation_tabular_regression_sample] diff --git a/samples/snippets/get_model_evaluation_tabular_regression_sample_test.py b/samples/snippets/get_model_evaluation_tabular_regression_sample_test.py new file mode 100644 index 0000000000..a0d97b0a77 --- /dev/null +++ b/samples/snippets/get_model_evaluation_tabular_regression_sample_test.py @@ -0,0 +1,30 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os + +import get_model_evaluation_sample + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +MODEL_ID = "3125638878883479552" # bq all +EVALUATION_ID = "2025948722346981108" # bq all evaluation + + +def test_ucaip_generated_get_model_evaluation_sample(capsys): + get_model_evaluation_sample.get_model_evaluation_sample( + project=PROJECT_ID, model_id=MODEL_ID, evaluation_id=EVALUATION_ID + ) + out, _ = capsys.readouterr() + assert "metrics_schema_uri" in out diff --git a/samples/snippets/get_model_evaluation_text_classification_sample.py b/samples/snippets/get_model_evaluation_text_classification_sample.py new file mode 100644 index 0000000000..4fb60d4e18 --- /dev/null +++ b/samples/snippets/get_model_evaluation_text_classification_sample.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_get_model_evaluation_text_classification_sample] +from google.cloud import aiplatform + + +def get_model_evaluation_text_classification_sample( + project: str, + model_id: str, + evaluation_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + name = client.model_evaluation_path( + project=project, location=location, model=model_id, evaluation=evaluation_id + ) + response = client.get_model_evaluation(name=name) + print("response:", response) + + +# [END aiplatform_get_model_evaluation_text_classification_sample] diff --git a/samples/snippets/get_model_evaluation_text_entity_extraction_sample.py b/samples/snippets/get_model_evaluation_text_entity_extraction_sample.py new file mode 100644 index 0000000000..4f93953399 --- /dev/null +++ b/samples/snippets/get_model_evaluation_text_entity_extraction_sample.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_get_model_evaluation_text_entity_extraction_sample] +from google.cloud import aiplatform + + +def get_model_evaluation_text_entity_extraction_sample( + project: str, + model_id: str, + evaluation_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + name = client.model_evaluation_path( + project=project, location=location, model=model_id, evaluation=evaluation_id + ) + response = client.get_model_evaluation(name=name) + print("response:", response) + + +# [END aiplatform_get_model_evaluation_text_entity_extraction_sample] diff --git a/samples/snippets/get_model_evaluation_text_sentiment_analysis_sample.py b/samples/snippets/get_model_evaluation_text_sentiment_analysis_sample.py new file mode 100644 index 0000000000..977d755114 --- /dev/null +++ b/samples/snippets/get_model_evaluation_text_sentiment_analysis_sample.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_get_model_evaluation_text_sentiment_analysis_sample] +from google.cloud import aiplatform + + +def get_model_evaluation_text_sentiment_analysis_sample( + project: str, + model_id: str, + evaluation_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + name = client.model_evaluation_path( + project=project, location=location, model=model_id, evaluation=evaluation_id + ) + response = client.get_model_evaluation(name=name) + print("response:", response) + + +# [END aiplatform_get_model_evaluation_text_sentiment_analysis_sample] diff --git a/samples/snippets/get_model_evaluation_video_classification_sample.py b/samples/snippets/get_model_evaluation_video_classification_sample.py new file mode 100644 index 0000000000..6a0f5c8143 --- /dev/null +++ b/samples/snippets/get_model_evaluation_video_classification_sample.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_get_model_evaluation_video_classification_sample] +from google.cloud import aiplatform + + +def get_model_evaluation_video_classification_sample( + project: str, + model_id: str, + evaluation_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + name = client.model_evaluation_path( + project=project, location=location, model=model_id, evaluation=evaluation_id + ) + response = client.get_model_evaluation(name=name) + print("response:", response) + + +# [END aiplatform_get_model_evaluation_video_classification_sample] diff --git a/samples/snippets/get_model_evaluation_video_classification_sample_test.py b/samples/snippets/get_model_evaluation_video_classification_sample_test.py new file mode 100644 index 0000000000..cfb8a84f29 --- /dev/null +++ b/samples/snippets/get_model_evaluation_video_classification_sample_test.py @@ -0,0 +1,30 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os + +import get_model_evaluation_video_classification_sample + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +MODEL_ID = "667940119734386688" # Permanent sports model +EVALUATION_ID = "789396572185034752" # Permanent sports evaluation + + +def test_ucaip_generated_get_model_evaluation_sample(capsys): + get_model_evaluation_video_classification_sample.get_model_evaluation_video_classification_sample( + project=PROJECT_ID, model_id=MODEL_ID, evaluation_id=EVALUATION_ID + ) + out, _ = capsys.readouterr() + assert "metrics_schema_uri" in out diff --git a/samples/snippets/get_model_evaluation_video_object_tracking_sample.py b/samples/snippets/get_model_evaluation_video_object_tracking_sample.py new file mode 100644 index 0000000000..4fd4b62958 --- /dev/null +++ b/samples/snippets/get_model_evaluation_video_object_tracking_sample.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_get_model_evaluation_video_object_tracking_sample] +from google.cloud import aiplatform + + +def get_model_evaluation_video_object_tracking_sample( + project: str, + model_id: str, + evaluation_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + name = client.model_evaluation_path( + project=project, location=location, model=model_id, evaluation=evaluation_id + ) + response = client.get_model_evaluation(name=name) + print("response:", response) + + +# [END aiplatform_get_model_evaluation_video_object_tracking_sample] diff --git a/samples/snippets/get_model_evaluation_video_object_tracking_sample_test.py b/samples/snippets/get_model_evaluation_video_object_tracking_sample_test.py new file mode 100644 index 0000000000..4777889ed7 --- /dev/null +++ b/samples/snippets/get_model_evaluation_video_object_tracking_sample_test.py @@ -0,0 +1,30 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os + +import get_model_evaluation_video_object_tracking_sample + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +MODEL_ID = "20547673299877888" # Permanent horses model +EVALUATION_ID = "1165447141070471168" # Permanent horses evaluation + + +def test_ucaip_generated_get_model_evaluation_sample(capsys): + get_model_evaluation_video_object_tracking_sample.get_model_evaluation_video_object_tracking_sample( + project=PROJECT_ID, model_id=MODEL_ID, evaluation_id=EVALUATION_ID + ) + out, _ = capsys.readouterr() + assert "metrics_schema_uri" in out diff --git a/samples/snippets/get_model_sample.py b/samples/snippets/get_model_sample.py new file mode 100644 index 0000000000..6d56f90304 --- /dev/null +++ b/samples/snippets/get_model_sample.py @@ -0,0 +1,34 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_get_model_sample] +from google.cloud import aiplatform + + +def get_model_sample( + project: str, + model_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + name = client.model_path(project=project, location=location, model=model_id) + response = client.get_model(name=name) + print("response:", response) + + +# [END aiplatform_get_model_sample] diff --git a/samples/snippets/get_model_sample_test.py b/samples/snippets/get_model_sample_test.py new file mode 100644 index 0000000000..baf045205e --- /dev/null +++ b/samples/snippets/get_model_sample_test.py @@ -0,0 +1,28 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os + +import get_model_sample + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +MODEL_ID = "1478306577684365312" # permanent_50_flowers_model +KNOWN_MODEL = f"/locations/us-central1/models/{MODEL_ID}" + + +def test_ucaip_generated_get_model_sample(capsys): + get_model_sample.get_model_sample(project=PROJECT_ID, model_id=MODEL_ID) + out, _ = capsys.readouterr() + assert KNOWN_MODEL in out diff --git a/samples/snippets/get_training_pipeline_sample.py b/samples/snippets/get_training_pipeline_sample.py new file mode 100644 index 0000000000..52317e1e95 --- /dev/null +++ b/samples/snippets/get_training_pipeline_sample.py @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_get_training_pipeline_sample] +from google.cloud import aiplatform + + +def get_training_pipeline_sample( + project: str, + training_pipeline_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PipelineServiceClient(client_options=client_options) + name = client.training_pipeline_path( + project=project, location=location, training_pipeline=training_pipeline_id + ) + response = client.get_training_pipeline(name=name) + print("response:", response) + + +# [END aiplatform_get_training_pipeline_sample] diff --git a/samples/snippets/get_training_pipeline_sample_test.py b/samples/snippets/get_training_pipeline_sample_test.py new file mode 100644 index 0000000000..2d3a29f481 --- /dev/null +++ b/samples/snippets/get_training_pipeline_sample_test.py @@ -0,0 +1,33 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os + +import get_training_pipeline_sample + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") + +# Permanent training pipeline of 50 flowers dataset +TRAINING_PIPELINE_ID = "1419759782528548864" +TRAINING_PIPELINE_DISPLAY_NAME = "permanent_50_flowers_pipeline" + + +def test_ucaip_generated_get_training_pipeline_sample(capsys): + get_training_pipeline_sample.get_training_pipeline_sample( + project=PROJECT_ID, training_pipeline_id=TRAINING_PIPELINE_ID + ) + + out, _ = capsys.readouterr() + assert TRAINING_PIPELINE_DISPLAY_NAME in out diff --git a/samples/snippets/helpers.py b/samples/snippets/helpers.py new file mode 100644 index 0000000000..15a9f31e5c --- /dev/null +++ b/samples/snippets/helpers.py @@ -0,0 +1,58 @@ +import re +import time + +from typing import Callable + + +def get_name(out, key="name"): + pattern = re.compile(fr'{key}:\s*"([\-a-zA-Z0-9/]+)"') + name = re.search(pattern, out).group(1) + + return name + + +def get_state(out): + pattern = re.compile(r"state:\s*([._a-zA-Z0-9/]+)") + state = re.search(pattern, out).group(1) + + return state + + +def wait_for_job_state( + get_job_method: Callable[[str], "proto.Message"], + name: str, + expected_state: str = "CANCELLED", + timeout: int = 90, + freq: float = 1.5, +) -> None: + """ Waits until the Job state of provided resource name is a particular state. + + Args: + get_job_method: Callable[[str], "proto.Message"] + Required. The GAPIC getter method to poll. Takes 'name' parameter + and has a 'state' attribute in its response. + name (str): + Required. Complete uCAIP resource name to pass to get_job_method + expected_state (str): + The state at which this method will stop waiting. + Default is "CANCELLED". + timeout (int): + Maximum number of seconds to wait for expected_state. If the job + state is not expected_state within timeout, a TimeoutError will be raised. + Default is 90 seconds. + freq (float): + Number of seconds between calls to get_job_method. + Default is 1.5 seconds. + """ + + for _ in range(int(timeout / freq)): + response = get_job_method(name=name) + if expected_state in str(response.state): + return None + time.sleep(freq) + + raise TimeoutError( + f"Job state did not become {expected_state} within {timeout} seconds" + "\nTry increasing the timeout in sample test" + f"\nLast recorded state: {response.state}" + ) diff --git a/samples/snippets/import_data_image_classification_single_label_sample.py b/samples/snippets/import_data_image_classification_single_label_sample.py new file mode 100644 index 0000000000..d4289820d5 --- /dev/null +++ b/samples/snippets/import_data_image_classification_single_label_sample.py @@ -0,0 +1,44 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_import_data_image_classification_single_label_sample] +from google.cloud import aiplatform + + +def import_data_image_classification_single_label_sample( + project: str, + dataset_id: str, + gcs_source_uri: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 1800, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + import_configs = [ + { + "gcs_source": {"uris": [gcs_source_uri]}, + "import_schema_uri": "gs://google-cloud-aiplatform/schema/dataset/ioformat/image_classification_single_label_io_format_1.0.0.yaml", + } + ] + name = client.dataset_path(project=project, location=location, dataset=dataset_id) + response = client.import_data(name=name, import_configs=import_configs) + print("Long running operation:", response.operation.name) + import_data_response = response.result(timeout=timeout) + print("import_data_response:", import_data_response) + + +# [END aiplatform_import_data_image_classification_single_label_sample] diff --git a/samples/snippets/import_data_image_object_detection_sample.py b/samples/snippets/import_data_image_object_detection_sample.py new file mode 100644 index 0000000000..ce6297268b --- /dev/null +++ b/samples/snippets/import_data_image_object_detection_sample.py @@ -0,0 +1,44 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_import_data_image_object_detection_sample] +from google.cloud import aiplatform + + +def import_data_image_object_detection_sample( + project: str, + dataset_id: str, + gcs_source_uri: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 1800, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + import_configs = [ + { + "gcs_source": {"uris": [gcs_source_uri]}, + "import_schema_uri": "gs://google-cloud-aiplatform/schema/dataset/ioformat/image_bounding_box_io_format_1.0.0.yaml", + } + ] + name = client.dataset_path(project=project, location=location, dataset=dataset_id) + response = client.import_data(name=name, import_configs=import_configs) + print("Long running operation:", response.operation.name) + import_data_response = response.result(timeout=timeout) + print("import_data_response:", import_data_response) + + +# [END aiplatform_import_data_image_object_detection_sample] diff --git a/samples/snippets/import_data_sample.py b/samples/snippets/import_data_sample.py new file mode 100644 index 0000000000..f712bc59ad --- /dev/null +++ b/samples/snippets/import_data_sample.py @@ -0,0 +1,46 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_import_data_sample] +from google.cloud import aiplatform + + +def import_data_sample( + project: str, + dataset_id: str, + gcs_source_uri: str, + import_schema_uri: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 1800, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + # Here we use only one import config with one source + import_configs = [ + { + "gcs_source": {"uris": [gcs_source_uri]}, + "import_schema_uri": import_schema_uri, + } + ] + name = client.dataset_path(project=project, location=location, dataset=dataset_id) + response = client.import_data(name=name, import_configs=import_configs) + print("Long running operation:", response.operation.name) + import_data_response = response.result(timeout=timeout) + print("import_data_response:", import_data_response) + + +# [END aiplatform_import_data_sample] diff --git a/samples/snippets/import_data_text_classification_single_label_sample.py b/samples/snippets/import_data_text_classification_single_label_sample.py new file mode 100644 index 0000000000..0579531271 --- /dev/null +++ b/samples/snippets/import_data_text_classification_single_label_sample.py @@ -0,0 +1,44 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_import_data_text_classification_single_label_sample] +from google.cloud import aiplatform + + +def import_data_text_classification_single_label_sample( + project: str, + dataset_id: str, + gcs_source_uri: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 1800, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + import_configs = [ + { + "gcs_source": {"uris": [gcs_source_uri]}, + "import_schema_uri": "gs://google-cloud-aiplatform/schema/dataset/ioformat/text_classification_single_label_io_format_1.0.0.yaml", + } + ] + name = client.dataset_path(project=project, location=location, dataset=dataset_id) + response = client.import_data(name=name, import_configs=import_configs) + print("Long running operation:", response.operation.name) + import_data_response = response.result(timeout=timeout) + print("import_data_response:", import_data_response) + + +# [END aiplatform_import_data_text_classification_single_label_sample] diff --git a/samples/snippets/import_data_text_classification_single_label_sample_test.py b/samples/snippets/import_data_text_classification_single_label_sample_test.py new file mode 100644 index 0000000000..6b7dbdd195 --- /dev/null +++ b/samples/snippets/import_data_text_classification_single_label_sample_test.py @@ -0,0 +1,39 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +import os +from unittest.mock import patch, mock_open, MagicMock + +import import_data_text_classification_single_label_sample + +# Test to assert that the import data function was called. We assert that the function was called +# rather than wait for this LRO to complete +def test_ucaip_generated_import_data_text_classification_single_label_sample(): + response = MagicMock() + response.next_page_token = b"" + rpc = MagicMock(return_value=response) + + mock_channel = MagicMock() + mock_channel.unary_unary = MagicMock(return_value=rpc) + + with patch( + "google.api_core.grpc_helpers.create_channel", return_value=mock_channel + ), patch("time.sleep"), patch("builtins.open", mock_open(read_data=b"")): + import_data_text_classification_single_label_sample.import_data_text_classification_single_label_sample( + gcs_source_uri="GCS_SOURCE_URI", project="PROJECT", dataset_id="DATASET_ID" + ) + + rpc.assert_called() diff --git a/samples/snippets/import_data_text_entity_extraction_sample.py b/samples/snippets/import_data_text_entity_extraction_sample.py new file mode 100644 index 0000000000..f322f918b1 --- /dev/null +++ b/samples/snippets/import_data_text_entity_extraction_sample.py @@ -0,0 +1,44 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_import_data_text_entity_extraction_sample] +from google.cloud import aiplatform + + +def import_data_text_entity_extraction_sample( + project: str, + dataset_id: str, + gcs_source_uri: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 1800, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + import_configs = [ + { + "gcs_source": {"uris": [gcs_source_uri]}, + "import_schema_uri": "gs://google-cloud-aiplatform/schema/dataset/ioformat/text_extraction_io_format_1.0.0.yaml", + } + ] + name = client.dataset_path(project=project, location=location, dataset=dataset_id) + response = client.import_data(name=name, import_configs=import_configs) + print("Long running operation:", response.operation.name) + import_data_response = response.result(timeout=timeout) + print("import_data_response:", import_data_response) + + +# [END aiplatform_import_data_text_entity_extraction_sample] diff --git a/samples/snippets/import_data_text_entity_extraction_sample_test.py b/samples/snippets/import_data_text_entity_extraction_sample_test.py new file mode 100644 index 0000000000..d74c6a322c --- /dev/null +++ b/samples/snippets/import_data_text_entity_extraction_sample_test.py @@ -0,0 +1,62 @@ +import pytest +import os + +from uuid import uuid4 +from google.cloud import aiplatform + +import import_data_text_entity_extraction_sample +import delete_dataset_sample + +print( + f"uCAIP Library Source:\t{aiplatform.__file__}" +) # Package source location sanity check +print( + f"uCAIP Import Source:\t{import_data_text_entity_extraction_sample.__file__}" +) # Package source location sanity check + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +LOCATION = "us-central1" +GCS_SOURCE = "gs://ucaip-test-us-central1/dataset/ucaip_ten_dataset.jsonl" +METADATA_SCHEMA_URI = ( + "gs://google-cloud-aiplatform/schema/dataset/metadata/text_1.0.0.yaml" +) + + +@pytest.fixture(scope="function", autouse=True) +def dataset_name(): + client_options = {"api_endpoint": "us-central1-aiplatform.googleapis.com"} + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + + dataset = aiplatform.gapic.Dataset( + display_name=f"temp_import_dataset_test_{uuid4()}", + metadata_schema_uri=METADATA_SCHEMA_URI, + ) + + operation = client.create_dataset( + parent=f"projects/{PROJECT_ID}/locations/{LOCATION}", dataset=dataset + ) + + created_dataset = operation.result() + + yield created_dataset.name + + dataset_id = created_dataset.name.split("/")[-1] + + # Delete the created dataset + delete_dataset_sample.delete_dataset_sample( + project=PROJECT_ID, dataset_id=dataset_id + ) + + +def test_ucaip_generated_import_data_text_entity_extraction_sample( + capsys, dataset_name +): + dataset_id = dataset_name.split("/")[-1] + + import_data_text_entity_extraction_sample.import_data_text_entity_extraction_sample( + gcs_source_uri=GCS_SOURCE, project=PROJECT_ID, dataset_id=dataset_id + ) + + out, _ = capsys.readouterr() + + assert "import_data_response" in out diff --git a/samples/snippets/import_data_text_sentiment_analysis_sample.py b/samples/snippets/import_data_text_sentiment_analysis_sample.py new file mode 100644 index 0000000000..2d9ee850fb --- /dev/null +++ b/samples/snippets/import_data_text_sentiment_analysis_sample.py @@ -0,0 +1,44 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_import_data_text_sentiment_analysis_sample] +from google.cloud import aiplatform + + +def import_data_text_sentiment_analysis_sample( + project: str, + dataset_id: str, + gcs_source_uri: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 1800, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + import_configs = [ + { + "gcs_source": {"uris": [gcs_source_uri]}, + "import_schema_uri": "gs://google-cloud-aiplatform/schema/dataset/ioformat/text_sentiment_io_format_1.0.0.yaml", + } + ] + name = client.dataset_path(project=project, location=location, dataset=dataset_id) + response = client.import_data(name=name, import_configs=import_configs) + print("Long running operation:", response.operation.name) + import_data_response = response.result(timeout=timeout) + print("import_data_response:", import_data_response) + + +# [END aiplatform_import_data_text_sentiment_analysis_sample] diff --git a/samples/snippets/import_data_text_sentiment_analysis_sample_test.py b/samples/snippets/import_data_text_sentiment_analysis_sample_test.py new file mode 100644 index 0000000000..3fd9b20538 --- /dev/null +++ b/samples/snippets/import_data_text_sentiment_analysis_sample_test.py @@ -0,0 +1,62 @@ +import pytest +import os + +from uuid import uuid4 +from google.cloud import aiplatform + +import import_data_text_sentiment_analysis_sample +import delete_dataset_sample + +print( + f"uCAIP Library Source:\t{aiplatform.__file__}" +) # Package source location sanity check +print( + f"uCAIP Import Source:\t{import_data_text_sentiment_analysis_sample.__file__}" +) # Package source location sanity check + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +LOCATION = "us-central1" +GCS_SOURCE = "gs://ucaip-test-us-central1/dataset/ucaip_tst_dataset_10.csv" +METADATA_SCHEMA_URI = ( + "gs://google-cloud-aiplatform/schema/dataset/metadata/text_1.0.0.yaml" +) + + +@pytest.fixture(scope="function", autouse=True) +def dataset_name(): + client_options = {"api_endpoint": "us-central1-aiplatform.googleapis.com"} + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + + dataset = aiplatform.gapic.Dataset( + display_name=f"temp_import_dataset_test_{uuid4()}", + metadata_schema_uri=METADATA_SCHEMA_URI, + ) + + operation = client.create_dataset( + parent=f"projects/{PROJECT_ID}/locations/{LOCATION}", dataset=dataset + ) + + created_dataset = operation.result(timeout=120) + + yield created_dataset.name + + dataset_id = created_dataset.name.split("/")[-1] + + # Delete the created dataset + delete_dataset_sample.delete_dataset_sample( + project=PROJECT_ID, dataset_id=dataset_id + ) + + +def test_ucaip_generated_import_data_text_sentiment_analysis_sample( + capsys, dataset_name +): + dataset_id = dataset_name.split("/")[-1] + + import_data_text_sentiment_analysis_sample.import_data_text_sentiment_analysis_sample( + gcs_source_uri=GCS_SOURCE, project=PROJECT_ID, dataset_id=dataset_id + ) + + out, _ = capsys.readouterr() + + assert "import_data_response" in out diff --git a/samples/snippets/import_data_video_classification_sample.py b/samples/snippets/import_data_video_classification_sample.py new file mode 100644 index 0000000000..6162d6b662 --- /dev/null +++ b/samples/snippets/import_data_video_classification_sample.py @@ -0,0 +1,44 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_import_data_video_classification_sample] +from google.cloud import aiplatform + + +def import_data_video_classification_sample( + project: str, + dataset_id: str, + gcs_source_uri: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 1800, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + import_configs = [ + { + "gcs_source": {"uris": [gcs_source_uri]}, + "import_schema_uri": "gs://google-cloud-aiplatform/schema/dataset/ioformat/video_classification_io_format_1.0.0.yaml", + } + ] + name = client.dataset_path(project=project, location=location, dataset=dataset_id) + response = client.import_data(name=name, import_configs=import_configs) + print("Long running operation:", response.operation.name) + import_data_response = response.result(timeout=timeout) + print("import_data_response:", import_data_response) + + +# [END aiplatform_import_data_video_classification_sample] diff --git a/samples/snippets/import_data_video_classification_sample_test.py b/samples/snippets/import_data_video_classification_sample_test.py new file mode 100644 index 0000000000..0a9a1b08c9 --- /dev/null +++ b/samples/snippets/import_data_video_classification_sample_test.py @@ -0,0 +1,69 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +import os + +from uuid import uuid4 +from google.cloud import aiplatform + +import import_data_video_classification_sample +import delete_dataset_sample + + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +LOCATION = "us-central1" +GCS_SOURCE = "gs://ucaip-sample-resources/video_classification_train.jsonl" +METADATA_SCHEMA_URI = ( + "gs://google-cloud-aiplatform/schema/dataset/metadata/video_1.0.0.yaml" +) + + +@pytest.fixture(scope="function", autouse=True) +def dataset_name(): + client_options = {"api_endpoint": "us-central1-aiplatform.googleapis.com"} + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + + dataset = aiplatform.gapic.Dataset( + display_name=f"temp_import_dataset_test_{uuid4()}", + metadata_schema_uri=METADATA_SCHEMA_URI, + ) + + operation = client.create_dataset( + parent=f"projects/{PROJECT_ID}/locations/{LOCATION}", dataset=dataset + ) + + created_dataset = operation.result(timeout=120) + + yield created_dataset.name + + dataset_id = created_dataset.name.split("/")[-1] + + # Delete the created dataset + delete_dataset_sample.delete_dataset_sample( + project=PROJECT_ID, dataset_id=dataset_id + ) + + +def test_ucaip_generated_import_data_video_classification_sample_single_label_image( + capsys, dataset_name +): + dataset_id = dataset_name.split("/")[-1] + + import_data_video_classification_sample.import_data_video_classification_sample( + project=PROJECT_ID, dataset_id=dataset_id, gcs_source_uri=GCS_SOURCE, + ) + out, _ = capsys.readouterr() + assert "import_data_response" in out diff --git a/samples/snippets/import_data_video_object_tracking_sample.py b/samples/snippets/import_data_video_object_tracking_sample.py new file mode 100644 index 0000000000..65dd09bd9e --- /dev/null +++ b/samples/snippets/import_data_video_object_tracking_sample.py @@ -0,0 +1,44 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_import_data_video_object_tracking_sample] +from google.cloud import aiplatform + + +def import_data_video_object_tracking_sample( + project: str, + dataset_id: str, + gcs_source_uri: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 1800, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + import_configs = [ + { + "gcs_source": {"uris": [gcs_source_uri]}, + "import_schema_uri": "gs://google-cloud-aiplatform/schema/dataset/ioformat/video_object_tracking_io_format_1.0.0.yaml", + } + ] + name = client.dataset_path(project=project, location=location, dataset=dataset_id) + response = client.import_data(name=name, import_configs=import_configs) + print("Long running operation:", response.operation.name) + import_data_response = response.result(timeout=timeout) + print("import_data_response:", import_data_response) + + +# [END aiplatform_import_data_video_object_tracking_sample] diff --git a/samples/snippets/import_data_video_object_tracking_sample_test.py b/samples/snippets/import_data_video_object_tracking_sample_test.py new file mode 100644 index 0000000000..457bf869b5 --- /dev/null +++ b/samples/snippets/import_data_video_object_tracking_sample_test.py @@ -0,0 +1,69 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +import os + +from uuid import uuid4 +from google.cloud import aiplatform + +import import_data_video_object_tracking_sample +import delete_dataset_sample + + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +LOCATION = "us-central1" +GCS_SOURCE = "gs://ucaip-sample-resources/video_object_tracking_train.jsonl" +METADATA_SCHEMA_URI = ( + "gs://google-cloud-aiplatform/schema/dataset/metadata/video_1.0.0.yaml" +) + + +@pytest.fixture(scope="function", autouse=True) +def dataset_name(): + client_options = {"api_endpoint": "us-central1-aiplatform.googleapis.com"} + client = aiplatform.gapic.DatasetServiceClient(client_options=client_options) + + dataset = aiplatform.gapic.Dataset( + display_name=f"temp_import_dataset_test_{uuid4()}", + metadata_schema_uri=METADATA_SCHEMA_URI, + ) + + operation = client.create_dataset( + parent=f"projects/{PROJECT_ID}/locations/{LOCATION}", dataset=dataset + ) + + created_dataset = operation.result(timeout=120) + + yield created_dataset.name + + dataset_id = created_dataset.name.split("/")[-1] + + # Delete the created dataset + delete_dataset_sample.delete_dataset_sample( + project=PROJECT_ID, dataset_id=dataset_id + ) + + +def test_ucaip_generated_import_data_video_object_tracking_sample_single_label_image( + capsys, dataset_name +): + dataset_id = dataset_name.split("/")[-1] + + import_data_video_object_tracking_sample.import_data_video_object_tracking_sample( + project=PROJECT_ID, dataset_id=dataset_id, gcs_source_uri=GCS_SOURCE, + ) + out, _ = capsys.readouterr() + assert "import_data_response" in out diff --git a/samples/snippets/list_model_evaluation_slices_sample.py b/samples/snippets/list_model_evaluation_slices_sample.py new file mode 100644 index 0000000000..5ae1f2212d --- /dev/null +++ b/samples/snippets/list_model_evaluation_slices_sample.py @@ -0,0 +1,38 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_list_model_evaluation_slices_sample] +from google.cloud import aiplatform + + +def list_model_evaluation_slices_sample( + project: str, + model_id: str, + evaluation_id: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + parent = client.model_evaluation_path( + project=project, location=location, model=model_id, evaluation=evaluation_id + ) + response = client.list_model_evaluation_slices(parent=parent) + for model_evaluation_slice in response: + print("model_evaluation_slice:", model_evaluation_slice) + + +# [END aiplatform_list_model_evaluation_slices_sample] diff --git a/samples/snippets/list_model_evaluation_slices_sample_test.py b/samples/snippets/list_model_evaluation_slices_sample_test.py new file mode 100644 index 0000000000..9e7e3823c0 --- /dev/null +++ b/samples/snippets/list_model_evaluation_slices_sample_test.py @@ -0,0 +1,32 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os + +import list_model_evaluation_slices_sample + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +MODEL_ID = "5162251072873431040" +EVALUATION_ID = "5615675837586029221" + +KNOWN_EVALUATION_SLICE = "/locations/us-central1/models/5162251072873431040/evaluations/5615675837586029221/slices/4322488217836113260" + + +def test_ucaip_generated_get_model_evaluation_slices_sample(capsys): + list_model_evaluation_slices_sample.list_model_evaluation_slices_sample( + project=PROJECT_ID, model_id=MODEL_ID, evaluation_id=EVALUATION_ID + ) + out, _ = capsys.readouterr() + assert KNOWN_EVALUATION_SLICE in out diff --git a/samples/noxfile.py b/samples/snippets/noxfile.py similarity index 93% rename from samples/noxfile.py rename to samples/snippets/noxfile.py index ba6624f0c8..5660f08be4 100644 --- a/samples/noxfile.py +++ b/samples/snippets/noxfile.py @@ -37,24 +37,22 @@ TEST_CONFIG = { # You can opt out from the test for specific Python versions. - 'ignored_versions': ["2.7"], - + "ignored_versions": ["2.7"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. - 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - 'envs': {}, + "envs": {}, } try: # Ensure we can import noxfile_config in the project's directory. - sys.path.append('.') + sys.path.append(".") from noxfile_config import TEST_CONFIG_OVERRIDE except ImportError as e: print("No user noxfile_config found: detail: {}".format(e)) @@ -69,12 +67,12 @@ def get_pytest_env_vars(): ret = {} # Override the GCLOUD_PROJECT and the alias. - env_key = TEST_CONFIG['gcloud_project_env'] + env_key = TEST_CONFIG["gcloud_project_env"] # This should error out if not set. - ret['GOOGLE_CLOUD_PROJECT'] = os.environ[env_key] + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] # Apply user supplied envs. - ret.update(TEST_CONFIG['envs']) + ret.update(TEST_CONFIG["envs"]) return ret @@ -83,7 +81,7 @@ def get_pytest_env_vars(): ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"] # Any default versions that should be ignored. -IGNORED_VERSIONS = TEST_CONFIG['ignored_versions'] +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) @@ -138,7 +136,7 @@ def lint(session): args = FLAKE8_COMMON_ARGS + [ "--application-import-names", ",".join(local_names), - "." + ".", ] session.run("flake8", *args) @@ -182,9 +180,9 @@ def py(session): if session.python in TESTED_VERSIONS: _session_tests(session) else: - session.skip("SKIPPED: {} tests are disabled for this sample.".format( - session.python - )) + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) # @@ -221,4 +219,4 @@ def readmegen(session, path): in_file = os.path.join(dir_, "README.rst.in") session.run( "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file - ) \ No newline at end of file + ) diff --git a/samples/noxfile_config.py b/samples/snippets/noxfile_config.py similarity index 85% rename from samples/noxfile_config.py rename to samples/snippets/noxfile_config.py index 46ed3b4c08..82a6331eee 100644 --- a/samples/noxfile_config.py +++ b/samples/snippets/noxfile_config.py @@ -19,16 +19,16 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - 'ignored_versions': ["2.7"], - + "ignored_versions": ["2.7"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. # 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', - 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - + "gcloud_project_env": "BUILD_SPECIFIC_GCLOUD_PROJECT", # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - 'envs': {}, -} \ No newline at end of file + "envs": { + "DATA_LABELING_API_ENDPOINT": "us-central1-autopush-aiplatform.sandbox.googleapis.com" + }, +} diff --git a/samples/snippets/predict_custom_trained_model_sample.py b/samples/snippets/predict_custom_trained_model_sample.py new file mode 100644 index 0000000000..308b0956fd --- /dev/null +++ b/samples/snippets/predict_custom_trained_model_sample.py @@ -0,0 +1,52 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_predict_custom_trained_model_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value +from typing import Dict + + +def predict_custom_trained_model_sample( + project: str, + endpoint_id: str, + instance_dict: Dict, + location: str = "us-central1", + api_endpoint: str = "us-central1-prediction-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PredictionServiceClient(client_options=client_options) + # The format of each instance should conform to the deployed model's prediction input schema. + instance = json_format.ParseDict(instance_dict, Value()) + instances = [instance] + parameters_dict = {} + parameters = json_format.ParseDict(parameters_dict, Value()) + endpoint = client.endpoint_path( + project=project, location=location, endpoint=endpoint_id + ) + response = client.predict( + endpoint=endpoint, instances=instances, parameters=parameters + ) + print("response") + print(" deployed_model_id:", response.deployed_model_id) + # The predictions are a google.protobuf.Value representation of the model's predictions. + predictions = response.predictions + for prediction in predictions: + print(" prediction:", dict(prediction)) + + +# [END aiplatform_predict_custom_trained_model_sample] diff --git a/samples/snippets/predict_custom_trained_model_sample_test.py b/samples/snippets/predict_custom_trained_model_sample_test.py new file mode 100644 index 0000000000..2bb19c4c95 --- /dev/null +++ b/samples/snippets/predict_custom_trained_model_sample_test.py @@ -0,0 +1,41 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os +import base64 +import pathlib + +import predict_custom_trained_model_sample + + +ENDPOINT_ID = "6119547468666372096" # permanent_custom_flowers_model +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") + +PATH_TO_IMG = pathlib.Path(__file__).parent.absolute() / "resources/daisy.jpg" + + +def test_ucaip_generated_predict_custom_trained_model_sample(capsys): + with open(PATH_TO_IMG, "rb") as f: + file_content = f.read() + encoded_content = base64.b64encode(file_content).decode("utf-8") + + instance_dict = {"image_bytes": {"b64": encoded_content}, "key": "0"} + + predict_custom_trained_model_sample.predict_custom_trained_model_sample( + instance_dict=instance_dict, project=PROJECT_ID, endpoint_id=ENDPOINT_ID + ) + + out, _ = capsys.readouterr() + assert "number_value: 1.0" in out diff --git a/samples/snippets/predict_image_classification_sample.py b/samples/snippets/predict_image_classification_sample.py new file mode 100644 index 0000000000..d90dd3732b --- /dev/null +++ b/samples/snippets/predict_image_classification_sample.py @@ -0,0 +1,59 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_predict_image_classification_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value +import base64 + + +def predict_image_classification_sample( + project: str, + endpoint_id: str, + filename: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-prediction-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PredictionServiceClient(client_options=client_options) + with open(filename, "rb") as f: + file_content = f.read() + + # The format of each instance should conform to the deployed model's prediction input schema. + encoded_content = base64.b64encode(file_content).decode("utf-8") + instance_dict = {"content": encoded_content} + + instance = json_format.ParseDict(instance_dict, Value()) + instances = [instance] + # See gs://google-cloud-aiplatform/schema/predict/params/image_classification_1.0.0.yaml for the format of the parameters. + parameters_dict = {"confidence_threshold": 0.5, "max_predictions": 5} + parameters = json_format.ParseDict(parameters_dict, Value()) + endpoint = client.endpoint_path( + project=project, location=location, endpoint=endpoint_id + ) + response = client.predict( + endpoint=endpoint, instances=instances, parameters=parameters + ) + print("response") + print(" deployed_model_id:", response.deployed_model_id) + # See gs://google-cloud-aiplatform/schema/predict/prediction/classification.yaml for the format of the predictions. + predictions = response.predictions + for prediction in predictions: + print(" prediction:", dict(prediction)) + + +# [END aiplatform_predict_image_classification_sample] diff --git a/samples/snippets/predict_image_classification_sample_test.py b/samples/snippets/predict_image_classification_sample_test.py new file mode 100644 index 0000000000..828700c318 --- /dev/null +++ b/samples/snippets/predict_image_classification_sample_test.py @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os +import base64 +import pathlib + +import predict_image_classification_sample + + +ENDPOINT_ID = "71213169107795968" # permanent_50_flowers_endpoint +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") + +PATH_TO_IMG = pathlib.Path(__file__).parent.absolute() / "resources/daisy.jpg" + + +def test_ucaip_generated_predict_image_classification_sample(capsys): + + predict_image_classification_sample.predict_image_classification_sample( + filename=PATH_TO_IMG, project=PROJECT_ID, endpoint_id=ENDPOINT_ID + ) + + out, _ = capsys.readouterr() + assert 'string_value: "daisy"' in out diff --git a/samples/snippets/predict_image_object_detection_sample.py b/samples/snippets/predict_image_object_detection_sample.py new file mode 100644 index 0000000000..4d3137830f --- /dev/null +++ b/samples/snippets/predict_image_object_detection_sample.py @@ -0,0 +1,59 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_predict_image_object_detection_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value +import base64 + + +def predict_image_object_detection_sample( + project: str, + endpoint_id: str, + filename: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-prediction-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PredictionServiceClient(client_options=client_options) + with open(filename, "rb") as f: + file_content = f.read() + + # The format of each instance should conform to the deployed model's prediction input schema. + encoded_content = base64.b64encode(file_content).decode("utf-8") + instance_dict = {"content": encoded_content} + + instance = json_format.ParseDict(instance_dict, Value()) + instances = [instance] + # See gs://google-cloud-aiplatform/schema/predict/params/image_object_detection_1.0.0.yaml for the format of the parameters. + parameters_dict = {"confidence_threshold": 0.5, "max_predictions": 5} + parameters = json_format.ParseDict(parameters_dict, Value()) + endpoint = client.endpoint_path( + project=project, location=location, endpoint=endpoint_id + ) + response = client.predict( + endpoint=endpoint, instances=instances, parameters=parameters + ) + print("response") + print(" deployed_model_id:", response.deployed_model_id) + # See gs://google-cloud-aiplatform/schema/predict/prediction/image_object_detection.yaml for the format of the predictions. + predictions = response.predictions + for prediction in predictions: + print(" prediction:", dict(prediction)) + + +# [END aiplatform_predict_image_object_detection_sample] diff --git a/samples/snippets/predict_image_object_detection_sample_test.py b/samples/snippets/predict_image_object_detection_sample_test.py new file mode 100644 index 0000000000..4cb4c5bdc2 --- /dev/null +++ b/samples/snippets/predict_image_object_detection_sample_test.py @@ -0,0 +1,35 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os +import base64 +import pathlib + +import predict_image_object_detection_sample + +ENDPOINT_ID = "2791387344039575552" # permanent_salad_img_obj_detection_endpoint +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") + +PATH_TO_IMG = pathlib.Path(__file__).parent.absolute() / "resources/caprese_salad.jpg" + + +def test_ucaip_generated_predict_image_object_detection_sample(capsys): + + predict_image_object_detection_sample.predict_image_object_detection_sample( + filename=PATH_TO_IMG, project=PROJECT_ID, endpoint_id=ENDPOINT_ID + ) + + out, _ = capsys.readouterr() + assert 'string_value: "Salad"' in out diff --git a/samples/snippets/predict_sample.py b/samples/snippets/predict_sample.py new file mode 100644 index 0000000000..d3a4a21f51 --- /dev/null +++ b/samples/snippets/predict_sample.py @@ -0,0 +1,52 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_predict_tutorial] +# [START aiplatform_predict_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value +from typing import Dict + + +def predict_sample( + project: str, + endpoint_id: str, + instance_dict: Dict, + location: str = "us-central1", + api_endpoint: str = "us-central1-prediction-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PredictionServiceClient(client_options=client_options) + instance = json_format.ParseDict(instance_dict, Value()) + instances = [instance] + parameters_dict = {} + parameters = json_format.ParseDict(parameters_dict, Value()) + endpoint = client.endpoint_path( + project=project, location=location, endpoint=endpoint_id + ) + response = client.predict( + endpoint=endpoint, instances=instances, parameters=parameters + ) + print("response") + print(" deployed_model_id:", response.deployed_model_id) + predictions = response.predictions + for prediction in predictions: + print(" prediction:", dict(prediction)) + + +# [END aiplatform_predict_sample] +# [END aiplatform_predict_tutorial] diff --git a/samples/snippets/predict_tabular_classification_sample.py b/samples/snippets/predict_tabular_classification_sample.py new file mode 100644 index 0000000000..fb09770789 --- /dev/null +++ b/samples/snippets/predict_tabular_classification_sample.py @@ -0,0 +1,53 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_predict_tabular_classification_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value +from typing import Dict + + +def predict_tabular_classification_sample( + project: str, + endpoint_id: str, + instance_dict: Dict, + location: str = "us-central1", + api_endpoint: str = "us-central1-prediction-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PredictionServiceClient(client_options=client_options) + # for more info on the instance schema, please use get_model_sample.py + # and look at the yaml found in instance_schema_uri + instance = json_format.ParseDict(instance_dict, Value()) + instances = [instance] + parameters_dict = {} + parameters = json_format.ParseDict(parameters_dict, Value()) + endpoint = client.endpoint_path( + project=project, location=location, endpoint=endpoint_id + ) + response = client.predict( + endpoint=endpoint, instances=instances, parameters=parameters + ) + print("response") + print(" deployed_model_id:", response.deployed_model_id) + # See gs://google-cloud-aiplatform/schema/predict/prediction/tables_classification.yaml for the format of the predictions. + predictions = response.predictions + for prediction in predictions: + print(" prediction:", dict(prediction)) + + +# [END aiplatform_predict_tabular_classification_sample] diff --git a/samples/snippets/predict_tabular_classification_sample_test.py b/samples/snippets/predict_tabular_classification_sample_test.py new file mode 100644 index 0000000000..db3410a315 --- /dev/null +++ b/samples/snippets/predict_tabular_classification_sample_test.py @@ -0,0 +1,38 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os + +import predict_tabular_classification_sample + +ENDPOINT_ID = "4966625964059525120" # iris 1000 +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") + +INSTANCE = { + "petal_length": "1.4", + "petal_width": "1.3", + "sepal_length": "5.1", + "sepal_width": "2.8", +} + + +def test_ucaip_generated_predict_tabular_classification_sample(capsys): + + predict_tabular_classification_sample.predict_tabular_classification_sample( + instance_dict=INSTANCE, project=PROJECT_ID, endpoint_id=ENDPOINT_ID + ) + + out, _ = capsys.readouterr() + assert 'string_value: "setosa"' in out diff --git a/samples/snippets/predict_tabular_regression_sample.py b/samples/snippets/predict_tabular_regression_sample.py new file mode 100644 index 0000000000..667a10ecd4 --- /dev/null +++ b/samples/snippets/predict_tabular_regression_sample.py @@ -0,0 +1,53 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_predict_tabular_regression_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value +from typing import Dict + + +def predict_tabular_regression_sample( + project: str, + endpoint_id: str, + instance_dict: Dict, + location: str = "us-central1", + api_endpoint: str = "us-central1-prediction-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PredictionServiceClient(client_options=client_options) + # for more info on the instance schema, please use get_model_sample.py + # and look at the yaml found in instance_schema_uri + instance = json_format.ParseDict(instance_dict, Value()) + instances = [instance] + parameters_dict = {} + parameters = json_format.ParseDict(parameters_dict, Value()) + endpoint = client.endpoint_path( + project=project, location=location, endpoint=endpoint_id + ) + response = client.predict( + endpoint=endpoint, instances=instances, parameters=parameters + ) + print("response") + print(" deployed_model_id:", response.deployed_model_id) + # See gs://google-cloud-aiplatform/schema/predict/prediction/tables_regression.yaml for the format of the predictions. + predictions = response.predictions + for prediction in predictions: + print(" prediction:", dict(prediction)) + + +# [END aiplatform_predict_tabular_regression_sample] diff --git a/samples/snippets/predict_tabular_regression_sample_test.py b/samples/snippets/predict_tabular_regression_sample_test.py new file mode 100644 index 0000000000..af9bac6b33 --- /dev/null +++ b/samples/snippets/predict_tabular_regression_sample_test.py @@ -0,0 +1,57 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os + +import predict_tabular_regression_sample + +ENDPOINT_ID = "1014154341088493568" # bq all +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") + +INSTANCE = { + "BOOLEAN_2unique_NULLABLE": False, + "DATETIME_1unique_NULLABLE": "2019-01-01 00:00:00", + "DATE_1unique_NULLABLE": "2019-01-01", + "FLOAT_5000unique_NULLABLE": 1611, + "FLOAT_5000unique_REPEATED": [2320, 1192], + "INTEGER_5000unique_NULLABLE": "8", + "NUMERIC_5000unique_NULLABLE": 16, + "STRING_5000unique_NULLABLE": "str-2", + "STRUCT_NULLABLE": { + "BOOLEAN_2unique_NULLABLE": False, + "DATE_1unique_NULLABLE": "2019-01-01", + "DATETIME_1unique_NULLABLE": "2019-01-01 00:00:00", + "FLOAT_5000unique_NULLABLE": 1308, + "FLOAT_5000unique_REPEATED": [2323, 1178], + "FLOAT_5000unique_REQUIRED": 3089, + "INTEGER_5000unique_NULLABLE": "1777", + "NUMERIC_5000unique_NULLABLE": 3323, + "TIME_1unique_NULLABLE": "23:59:59.999999", + "STRING_5000unique_NULLABLE": "str-49", + "TIMESTAMP_1unique_NULLABLE": "1546387199999999", + }, + "TIMESTAMP_1unique_NULLABLE": "1546387199999999", + "TIME_1unique_NULLABLE": "23:59:59.999999", +} + + +def test_ucaip_generated_predict_tabular_regression_sample(capsys): + + predict_tabular_regression_sample.predict_tabular_regression_sample( + instance_dict=INSTANCE, project=PROJECT_ID, endpoint_id=ENDPOINT_ID + ) + + out, _ = capsys.readouterr() + assert "2982.0" in out diff --git a/samples/snippets/predict_text_classification_single_label_sample.py b/samples/snippets/predict_text_classification_single_label_sample.py new file mode 100644 index 0000000000..b0f5bac9fb --- /dev/null +++ b/samples/snippets/predict_text_classification_single_label_sample.py @@ -0,0 +1,51 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_predict_text_classification_single_label_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def predict_text_classification_single_label_sample( + project: str, + endpoint_id: str, + content: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-prediction-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PredictionServiceClient(client_options=client_options) + instance_dict = {"content": content} + instance = json_format.ParseDict(instance_dict, Value()) + instances = [instance] + parameters_dict = {} + parameters = json_format.ParseDict(parameters_dict, Value()) + endpoint = client.endpoint_path( + project=project, location=location, endpoint=endpoint_id + ) + response = client.predict( + endpoint=endpoint, instances=instances, parameters=parameters + ) + print("response") + print(" deployed_model_id:", response.deployed_model_id) + # See gs://google-cloud-aiplatform/schema/predict/prediction/text_classification.yaml for the format of the predictions. + predictions = response.predictions + for prediction in predictions: + print(" prediction:", dict(prediction)) + + +# [END aiplatform_predict_text_classification_single_label_sample] diff --git a/samples/snippets/predict_text_classification_single_label_sample_test.py b/samples/snippets/predict_text_classification_single_label_sample_test.py new file mode 100644 index 0000000000..d4c6d88a9d --- /dev/null +++ b/samples/snippets/predict_text_classification_single_label_sample_test.py @@ -0,0 +1,33 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os + +import predict_text_classification_single_label_sample + +ENDPOINT_ID = "65372563341049856" # text_classification_single_label +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") + +content = "The Chicago Bears is the best football team on Earth." + + +def test_ucaip_generated_predict_text_classification_single_label_sample(capsys): + + predict_text_classification_single_label_sample.predict_text_classification_single_label_sample( + content=content, project=PROJECT_ID, endpoint_id=ENDPOINT_ID + ) + + out, _ = capsys.readouterr() + assert "prediction" in out diff --git a/samples/snippets/predict_text_entity_extraction_sample.py b/samples/snippets/predict_text_entity_extraction_sample.py new file mode 100644 index 0000000000..a78566ea06 --- /dev/null +++ b/samples/snippets/predict_text_entity_extraction_sample.py @@ -0,0 +1,52 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_predict_text_entity_extraction_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def predict_text_entity_extraction_sample( + project: str, + endpoint_id: str, + content: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-prediction-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PredictionServiceClient(client_options=client_options) + # The format of each instance should conform to the deployed model's prediction input schema + instance_dict = {"content": content} + instance = json_format.ParseDict(instance_dict, Value()) + instances = [instance] + parameters_dict = {} + parameters = json_format.ParseDict(parameters_dict, Value()) + endpoint = client.endpoint_path( + project=project, location=location, endpoint=endpoint_id + ) + response = client.predict( + endpoint=endpoint, instances=instances, parameters=parameters + ) + print("response") + print(" deployed_model_id:", response.deployed_model_id) + # See gs://google-cloud-aiplatform/schema/predict/prediction/text_extraction.yaml for the format of the predictions. + predictions = response.predictions + for prediction in predictions: + print(" prediction:", dict(prediction)) + + +# [END aiplatform_predict_text_entity_extraction_sample] diff --git a/samples/snippets/predict_text_entity_extraction_sample_test.py b/samples/snippets/predict_text_entity_extraction_sample_test.py new file mode 100644 index 0000000000..4207b42396 --- /dev/null +++ b/samples/snippets/predict_text_entity_extraction_sample_test.py @@ -0,0 +1,33 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os + +import predict_text_entity_extraction_sample + +ENDPOINT_ID = "6207156555167563776" # text_entity_extraction endpoint +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") + +CONTENT = "I really love working at Google!" + + +def test_ucaip_generated_predict_text_entity_extraction_sample(capsys): + + predict_text_entity_extraction_sample.predict_text_entity_extraction_sample( + project=PROJECT_ID, endpoint_id=ENDPOINT_ID, content=CONTENT + ) + + out, _ = capsys.readouterr() + assert "confidences" in out diff --git a/samples/snippets/predict_text_sentiment_analysis_sample.py b/samples/snippets/predict_text_sentiment_analysis_sample.py new file mode 100644 index 0000000000..d899a263fd --- /dev/null +++ b/samples/snippets/predict_text_sentiment_analysis_sample.py @@ -0,0 +1,51 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_predict_text_sentiment_analysis_sample] +from google.cloud import aiplatform +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Value + + +def predict_text_sentiment_analysis_sample( + project: str, + endpoint_id: str, + content: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-prediction-aiplatform.googleapis.com", +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.PredictionServiceClient(client_options=client_options) + instance_dict = {"content": content} + instance = json_format.ParseDict(instance_dict, Value()) + instances = [instance] + parameters_dict = {} + parameters = json_format.ParseDict(parameters_dict, Value()) + endpoint = client.endpoint_path( + project=project, location=location, endpoint=endpoint_id + ) + response = client.predict( + endpoint=endpoint, instances=instances, parameters=parameters + ) + print("response") + print(" deployed_model_id:", response.deployed_model_id) + # See gs://google-cloud-aiplatform/schema/predict/prediction/text_sentiment.yaml for the format of the predictions. + predictions = response.predictions + for prediction in predictions: + print(" prediction:", dict(prediction)) + + +# [END aiplatform_predict_text_sentiment_analysis_sample] diff --git a/samples/snippets/predict_text_sentiment_analysis_sample_test.py b/samples/snippets/predict_text_sentiment_analysis_sample_test.py new file mode 100644 index 0000000000..e66595b944 --- /dev/null +++ b/samples/snippets/predict_text_sentiment_analysis_sample_test.py @@ -0,0 +1,33 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os + +import predict_text_sentiment_analysis_sample + +ENDPOINT_ID = "7811563922418302976" # sentiment analysis endpoint +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") + +content = "The Chicago Bears is a great football team!" + + +def test_ucaip_generated_predict_text_sentiment_analysis_sample(capsys): + + predict_text_sentiment_analysis_sample.predict_text_sentiment_analysis_sample( + content=content, project=PROJECT_ID, endpoint_id=ENDPOINT_ID + ) + + out, _ = capsys.readouterr() + assert "sentiment" in out diff --git a/samples/requirements-tests.txt b/samples/snippets/requirements-tests.txt similarity index 100% rename from samples/requirements-tests.txt rename to samples/snippets/requirements-tests.txt diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt new file mode 100644 index 0000000000..c448d3d434 --- /dev/null +++ b/samples/snippets/requirements.txt @@ -0,0 +1,3 @@ +pytest==6.0.1 +google-cloud-storage>=1.26.0, <2.0.0dev +google-cloud-aiplatform==0.3.0 diff --git a/samples/snippets/resources/caprese_salad.jpg b/samples/snippets/resources/caprese_salad.jpg new file mode 100644 index 0000000000..fbd7e6575c Binary files /dev/null and b/samples/snippets/resources/caprese_salad.jpg differ diff --git a/samples/snippets/resources/daisy.jpg b/samples/snippets/resources/daisy.jpg new file mode 100644 index 0000000000..ae01cae918 Binary files /dev/null and b/samples/snippets/resources/daisy.jpg differ diff --git a/samples/snippets/upload_model_sample.py b/samples/snippets/upload_model_sample.py new file mode 100644 index 0000000000..9bb099ec17 --- /dev/null +++ b/samples/snippets/upload_model_sample.py @@ -0,0 +1,58 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START aiplatform_upload_model_sample] +from google.cloud import aiplatform + + +def upload_model_sample( + project: str, + display_name: str, + metadata_schema_uri: str, + image_uri: str, + artifact_uri: str, + location: str = "us-central1", + api_endpoint: str = "us-central1-aiplatform.googleapis.com", + timeout: int = 1800, +): + client_options = {"api_endpoint": api_endpoint} + # Initialize client that will be used to create and send requests. + # This client only needs to be created once, and can be reused for multiple requests. + client = aiplatform.gapic.ModelServiceClient(client_options=client_options) + model = { + "display_name": display_name, + "metadata_schema_uri": metadata_schema_uri, + # The artifact_uri should be the path to a GCS directory containing + # saved model artifacts. The bucket must be accessible for the + # project's AI Platform service account and in the same region as + # the api endpoint. + "artifact_uri": artifact_uri, + "container_spec": { + "image_uri": image_uri, + "command": [], + "args": [], + "env": [], + "ports": [], + "predict_route": "", + "health_route": "", + }, + } + parent = f"projects/{project}/locations/{location}" + response = client.upload_model(parent=parent, model=model) + print("Long running operation:", response.operation.name) + upload_model_response = response.result(timeout=timeout) + print("upload_model_response:", upload_model_response) + + +# [END aiplatform_upload_model_sample] diff --git a/samples/snippets/upload_model_sample_test.py b/samples/snippets/upload_model_sample_test.py new file mode 100644 index 0000000000..3814ddd4a7 --- /dev/null +++ b/samples/snippets/upload_model_sample_test.py @@ -0,0 +1,56 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import uuid4 +import pytest +import os + +import helpers + +import upload_model_sample, delete_model_sample + +PROJECT_ID = os.getenv("BUILD_SPECIFIC_GCLOUD_PROJECT") +IMAGE_URI = "gcr.io/cloud-ml-service-public/cloud-ml-online-prediction-model-server-cpu:v1_15py3cmle_op_images_20200229_0210_RC00" +ARTIFACT_URI = "gs://ucaip-samples-us-central1/model/explain/" +DISPLAY_NAME = f"temp_upload_model_test_{uuid4()}" + + +@pytest.fixture +def shared_state(): + state = {} + yield state + + +@pytest.fixture(scope="function", autouse=True) +def teardown(shared_state): + yield + + model_id = shared_state["model_name"].split("/")[-1] + + delete_model_sample.delete_model_sample(project=PROJECT_ID, model_id=model_id) + + +def test_ucaip_generated_upload_model_sample(capsys, shared_state): + + upload_model_sample.upload_model_sample( + display_name=DISPLAY_NAME, + metadata_schema_uri="", + image_uri=IMAGE_URI, + artifact_uri=ARTIFACT_URI, + project=PROJECT_ID, + ) + + out, _ = capsys.readouterr() + + shared_state["model_name"] = helpers.get_name(out, key="model")