From 359b6e93cd271cd7004bb97d3182ba79b6a5ae85 Mon Sep 17 00:00:00 2001 From: ivanmkc Date: Tue, 20 Apr 2021 18:51:09 -0400 Subject: [PATCH 1/9] Added create_and_import_dataset_tabular_gcs_sample.py --- ...e_and_import_dataset_tabular_gcs_sample.py | 44 +++++++++++++++++++ ...ate_and_import_dataset_tabular_gcs_test.py | 39 ++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 samples/model-builder/create_and_import_dataset_tabular_gcs_sample.py create mode 100644 samples/model-builder/create_and_import_dataset_tabular_gcs_test.py diff --git a/samples/model-builder/create_and_import_dataset_tabular_gcs_sample.py b/samples/model-builder/create_and_import_dataset_tabular_gcs_sample.py new file mode 100644 index 0000000000..dd6f09a799 --- /dev/null +++ b/samples/model-builder/create_and_import_dataset_tabular_gcs_sample.py @@ -0,0 +1,44 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Union + +from google.cloud import aiplatform + + +# [START aiplatform_sdk_create_and_import_dataset_tabular_gcs_sample] +def create_and_import_dataset_tabular_gcs_sample( + project: str, + location: str, + display_name: str, + src_uris: Union[str, List[str]], + sync: bool, +): + aiplatform.init(project=project, location=location) + + ds = aiplatform.TabularDataset.create( + display_name=display_name, + gcs_source=src_uris, + sync=sync, + ) + + if not sync: + ds.wait() + + print(ds.display_name) + print(ds.resource_name) + return ds + + +# [END aiplatform_sdk_create_and_import_dataset_tabular_gcs_sample] diff --git a/samples/model-builder/create_and_import_dataset_tabular_gcs_test.py b/samples/model-builder/create_and_import_dataset_tabular_gcs_test.py new file mode 100644 index 0000000000..e3c1d92d70 --- /dev/null +++ b/samples/model-builder/create_and_import_dataset_tabular_gcs_test.py @@ -0,0 +1,39 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from google.cloud.aiplatform import schema + +import create_and_import_dataset_tabular_gcs_sample +import test_constants as constants + + +def test_create_and_import_dataset_tabular_gcs_sample(mock_sdk_init, mock_create_tabular_dataset): + + create_and_import_dataset_tabular_gcs_sample.create_and_import_dataset_tabular_gcs_sample( + project=constants.PROJECT, + location=constants.LOCATION, + src_uris=constants.GCS_SOURCES, + display_name=constants.DISPLAY_NAME, + sync=True, + ) + + mock_sdk_init.assert_called_once_with( + project=constants.PROJECT, location=constants.LOCATION + ) + mock_create_tabular_dataset.assert_called_once_with( + display_name=constants.DISPLAY_NAME, + gcs_source=constants.GCS_SOURCES, + sync=True, + ) From 0ca06d5ba7d1e8a36ecc6401a7827e5ea23bfc88 Mon Sep 17 00:00:00 2001 From: ivanmkc Date: Tue, 20 Apr 2021 18:58:33 -0400 Subject: [PATCH 2/9] Added create_and_import_dataset_tabular_bigquery_sample.py --- ..._import_dataset_tabular_bigquery_sample.py | 43 +++++++++++++++++++ ...nd_import_dataset_tabular_bigquery_test.py | 39 +++++++++++++++++ ...e_and_import_dataset_tabular_gcs_sample.py | 3 +- samples/model-builder/test_constants.py | 1 + 4 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 samples/model-builder/create_and_import_dataset_tabular_bigquery_sample.py create mode 100644 samples/model-builder/create_and_import_dataset_tabular_bigquery_test.py diff --git a/samples/model-builder/create_and_import_dataset_tabular_bigquery_sample.py b/samples/model-builder/create_and_import_dataset_tabular_bigquery_sample.py new file mode 100644 index 0000000000..8493188355 --- /dev/null +++ b/samples/model-builder/create_and_import_dataset_tabular_bigquery_sample.py @@ -0,0 +1,43 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Union + +from google.cloud import aiplatform + + +# [START aiplatform_sdk_create_and_import_dataset_tabular_bigquery_sample] +def create_and_import_dataset_tabular_bigquery_sample( + project: str, + location: str, + display_name: str, + src_uris: Union[str, List[str]], + sync: bool, +): + aiplatform.init(project=project, location=location) + + ds = aiplatform.TabularDataset.create( + display_name=display_name, + bq_source=src_uris, + sync=sync, + ) + + ds.wait() + + print(ds.display_name) + print(ds.resource_name) + return ds + + +# [END aiplatform_sdk_create_and_import_dataset_tabular_bigquery_sample] diff --git a/samples/model-builder/create_and_import_dataset_tabular_bigquery_test.py b/samples/model-builder/create_and_import_dataset_tabular_bigquery_test.py new file mode 100644 index 0000000000..82b9fc2eb6 --- /dev/null +++ b/samples/model-builder/create_and_import_dataset_tabular_bigquery_test.py @@ -0,0 +1,39 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from google.cloud.aiplatform import schema + +import create_and_import_dataset_tabular_bigquery_sample +import test_constants as constants + + +def test_create_and_import_dataset_tabular_bigquery_sample(mock_sdk_init, mock_create_tabular_dataset): + + create_and_import_dataset_tabular_bigquery_sample.create_and_import_dataset_tabular_bigquery_sample( + project=constants.PROJECT, + location=constants.LOCATION, + src_uris=constants.BIGQUERY_SOURCE, + display_name=constants.DISPLAY_NAME, + sync=True, + ) + + mock_sdk_init.assert_called_once_with( + project=constants.PROJECT, location=constants.LOCATION + ) + mock_create_tabular_dataset.assert_called_once_with( + display_name=constants.DISPLAY_NAME, + bq_source=constants.BIGQUERY_SOURCE, + sync=True, + ) diff --git a/samples/model-builder/create_and_import_dataset_tabular_gcs_sample.py b/samples/model-builder/create_and_import_dataset_tabular_gcs_sample.py index dd6f09a799..409398b5f4 100644 --- a/samples/model-builder/create_and_import_dataset_tabular_gcs_sample.py +++ b/samples/model-builder/create_and_import_dataset_tabular_gcs_sample.py @@ -33,8 +33,7 @@ def create_and_import_dataset_tabular_gcs_sample( sync=sync, ) - if not sync: - ds.wait() + ds.wait() print(ds.display_name) print(ds.resource_name) diff --git a/samples/model-builder/test_constants.py b/samples/model-builder/test_constants.py index 50dfa968b4..4cf4b51bc1 100644 --- a/samples/model-builder/test_constants.py +++ b/samples/model-builder/test_constants.py @@ -40,6 +40,7 @@ TRAINING_JOB_NAME = f"{PARENT}/trainingJobs/{RESOURCE_ID}" GCS_SOURCES = ["gs://bucket1/source1.jsonl", "gs://bucket7/source4.jsonl"] +BIGQUERY_SOURCE = "bq://bigquery-public-data.ml_datasets.iris" GCS_DESTINATION = "gs://bucket3/output-dir/" TRAINING_FRACTION_SPLIT = 0.7 From 94fd07b82322af3f3aa6cdb640a249c8e0bd4c19 Mon Sep 17 00:00:00 2001 From: ivanmkc Date: Tue, 20 Apr 2021 20:12:30 -0400 Subject: [PATCH 3/9] Added create_training_pipeline_custom_job_sample.py and tweaked other tests --- samples/model-builder/conftest.py | 13 +++++ ..._import_dataset_tabular_bigquery_sample.py | 2 +- ...nd_import_dataset_tabular_bigquery_test.py | 1 - ...e_and_import_dataset_tabular_gcs_sample.py | 2 +- ...ate_and_import_dataset_tabular_gcs_test.py | 1 - ...ate_training_pipeline_custom_job_sample.py | 57 +++++++++++++++++++ ...reate_training_pipeline_custom_job_test.py | 57 +++++++++++++++++++ samples/model-builder/test_constants.py | 3 + 8 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 samples/model-builder/create_training_pipeline_custom_job_sample.py create mode 100644 samples/model-builder/create_training_pipeline_custom_job_test.py diff --git a/samples/model-builder/conftest.py b/samples/model-builder/conftest.py index 580c6a962d..b3ce6d149c 100644 --- a/samples/model-builder/conftest.py +++ b/samples/model-builder/conftest.py @@ -151,6 +151,19 @@ def mock_run_automl_image_training_job(): with patch.object(aiplatform.training_jobs.AutoMLImageTrainingJob, "run") as mock: yield mock +@pytest.fixture +def mock_init_custom_training_job(): + with patch.object( + aiplatform.training_jobs.CustomTrainingJob, "__init__" + ) as mock: + mock.return_value = None + yield mock + + +@pytest.fixture +def mock_run_custom_training_job(): + with patch.object(aiplatform.training_jobs.CustomTrainingJob, "run") as mock: + yield mock """ ---------------------------------------------------------------------------- diff --git a/samples/model-builder/create_and_import_dataset_tabular_bigquery_sample.py b/samples/model-builder/create_and_import_dataset_tabular_bigquery_sample.py index 8493188355..4be34ff12b 100644 --- a/samples/model-builder/create_and_import_dataset_tabular_bigquery_sample.py +++ b/samples/model-builder/create_and_import_dataset_tabular_bigquery_sample.py @@ -23,7 +23,7 @@ def create_and_import_dataset_tabular_bigquery_sample( location: str, display_name: str, src_uris: Union[str, List[str]], - sync: bool, + sync: bool = True, ): aiplatform.init(project=project, location=location) diff --git a/samples/model-builder/create_and_import_dataset_tabular_bigquery_test.py b/samples/model-builder/create_and_import_dataset_tabular_bigquery_test.py index 82b9fc2eb6..51118fa56c 100644 --- a/samples/model-builder/create_and_import_dataset_tabular_bigquery_test.py +++ b/samples/model-builder/create_and_import_dataset_tabular_bigquery_test.py @@ -26,7 +26,6 @@ def test_create_and_import_dataset_tabular_bigquery_sample(mock_sdk_init, mock_c location=constants.LOCATION, src_uris=constants.BIGQUERY_SOURCE, display_name=constants.DISPLAY_NAME, - sync=True, ) mock_sdk_init.assert_called_once_with( diff --git a/samples/model-builder/create_and_import_dataset_tabular_gcs_sample.py b/samples/model-builder/create_and_import_dataset_tabular_gcs_sample.py index 409398b5f4..62deb4b617 100644 --- a/samples/model-builder/create_and_import_dataset_tabular_gcs_sample.py +++ b/samples/model-builder/create_and_import_dataset_tabular_gcs_sample.py @@ -23,7 +23,7 @@ def create_and_import_dataset_tabular_gcs_sample( location: str, display_name: str, src_uris: Union[str, List[str]], - sync: bool, + sync: bool = True, ): aiplatform.init(project=project, location=location) diff --git a/samples/model-builder/create_and_import_dataset_tabular_gcs_test.py b/samples/model-builder/create_and_import_dataset_tabular_gcs_test.py index e3c1d92d70..e60a3b11e2 100644 --- a/samples/model-builder/create_and_import_dataset_tabular_gcs_test.py +++ b/samples/model-builder/create_and_import_dataset_tabular_gcs_test.py @@ -26,7 +26,6 @@ def test_create_and_import_dataset_tabular_gcs_sample(mock_sdk_init, mock_create location=constants.LOCATION, src_uris=constants.GCS_SOURCES, display_name=constants.DISPLAY_NAME, - sync=True, ) mock_sdk_init.assert_called_once_with( diff --git a/samples/model-builder/create_training_pipeline_custom_job_sample.py b/samples/model-builder/create_training_pipeline_custom_job_sample.py new file mode 100644 index 0000000000..123d76c3fb --- /dev/null +++ b/samples/model-builder/create_training_pipeline_custom_job_sample.py @@ -0,0 +1,57 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import aiplatform + + +# [START aiplatform_sdk_create_training_pipeline_custom_job_sample] +def create_training_pipeline_custom_job_sample( + project: str, + display_name: str, + script_path: str, + container_uri: str, + dataset_id: int, + location: str = "us-central1", + model_display_name: str = None, + training_fraction_split: float = 0.8, + validation_fraction_split: float = 0.1, + test_fraction_split: float = 0.1, + sync: bool = True, +): + aiplatform.init(project=project, location=location) + + job = aiplatform.CustomTrainingJob(display_name=display_name, + script_path=script_path, + container_uri=container_uri) + + my_image_ds = aiplatform.ImageDataset(dataset_id) + + model = job.run( + dataset=my_image_ds, + model_display_name=model_display_name, + training_fraction_split=training_fraction_split, + validation_fraction_split=validation_fraction_split, + test_fraction_split=test_fraction_split, + sync=sync, + ) + + model.wait() + + print(model.display_name) + print(model.resource_name) + print(model.uri) + return model + + +# [END aiplatform_sdk_create_training_pipeline_custom_job_sample] diff --git a/samples/model-builder/create_training_pipeline_custom_job_test.py b/samples/model-builder/create_training_pipeline_custom_job_test.py new file mode 100644 index 0000000000..2dbfeaf9cb --- /dev/null +++ b/samples/model-builder/create_training_pipeline_custom_job_test.py @@ -0,0 +1,57 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import create_training_pipeline_custom_job_sample +import test_constants as constants + + +def test_create_training_pipeline_custom_job_sample( + mock_sdk_init, + mock_image_dataset, + mock_init_custom_training_job, + mock_run_custom_training_job, + mock_get_image_dataset, +): + + create_training_pipeline_custom_job_sample.create_training_pipeline_custom_job_sample( + project=constants.PROJECT, + display_name=constants.DISPLAY_NAME, + script_path=constants.SCRIPT_PATH, + container_uri=constants.CONTAINER_URI, + dataset_id=constants.RESOURCE_ID, + model_display_name=constants.DISPLAY_NAME_2, + training_fraction_split=constants.TRAINING_FRACTION_SPLIT, + validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT, + test_fraction_split=constants.TEST_FRACTION_SPLIT, + ) + + mock_get_image_dataset.assert_called_once_with(constants.RESOURCE_ID) + + mock_sdk_init.assert_called_once_with( + project=constants.PROJECT, location=constants.LOCATION + ) + mock_init_custom_training_job.assert_called_once_with( + display_name=constants.DISPLAY_NAME, + script_path=constants.SCRIPT_PATH, + container_uri=constants.CONTAINER_URI, + ) + mock_run_custom_training_job.assert_called_once_with( + dataset=mock_image_dataset, + model_display_name=constants.DISPLAY_NAME_2, + training_fraction_split=constants.TRAINING_FRACTION_SPLIT, + validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT, + test_fraction_split=constants.TEST_FRACTION_SPLIT, + sync=True, + ) diff --git a/samples/model-builder/test_constants.py b/samples/model-builder/test_constants.py index 4cf4b51bc1..ef9098da78 100644 --- a/samples/model-builder/test_constants.py +++ b/samples/model-builder/test_constants.py @@ -52,3 +52,6 @@ ENCRYPTION_SPEC_KEY_NAME = f"{PARENT}/keyRings/{RESOURCE_ID}/cryptoKeys/{RESOURCE_ID_2}" PREDICTION_TEXT_INSTANCE = "This is some text for testing NLP prediction output" + +SCRIPT_PATH = "task.py" +CONTAINER_URI = "gcr.io/my_project/my_image:latest" \ No newline at end of file From 68eff565dccd0aa8549a975a1f45d21fbb65dcc0 Mon Sep 17 00:00:00 2001 From: ivanmkc Date: Tue, 20 Apr 2021 20:26:30 -0400 Subject: [PATCH 4/9] Added create_training_pipeline_custom_training_managed_dataset_sample and fixed unmanaged sample --- ...ate_training_pipeline_custom_job_sample.py | 7 +-- ...reate_training_pipeline_custom_job_test.py | 8 +-- ..._custom_training_managed_dataset_sample.py | 57 +++++++++++++++++++ ...ne_custom_training_managed_dataset_test.py | 57 +++++++++++++++++++ samples/model-builder/test_constants.py | 3 +- 5 files changed, 121 insertions(+), 11 deletions(-) create mode 100644 samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py create mode 100644 samples/model-builder/create_training_pipeline_custom_training_managed_dataset_test.py diff --git a/samples/model-builder/create_training_pipeline_custom_job_sample.py b/samples/model-builder/create_training_pipeline_custom_job_sample.py index 123d76c3fb..33cdb60aad 100644 --- a/samples/model-builder/create_training_pipeline_custom_job_sample.py +++ b/samples/model-builder/create_training_pipeline_custom_job_sample.py @@ -13,15 +13,16 @@ # limitations under the License. from google.cloud import aiplatform +from typing import List # [START aiplatform_sdk_create_training_pipeline_custom_job_sample] def create_training_pipeline_custom_job_sample( project: str, display_name: str, + args: List[str], script_path: str, container_uri: str, - dataset_id: int, location: str = "us-central1", model_display_name: str = None, training_fraction_split: float = 0.8, @@ -35,11 +36,9 @@ def create_training_pipeline_custom_job_sample( script_path=script_path, container_uri=container_uri) - my_image_ds = aiplatform.ImageDataset(dataset_id) - model = job.run( - dataset=my_image_ds, model_display_name=model_display_name, + args=args, training_fraction_split=training_fraction_split, validation_fraction_split=validation_fraction_split, test_fraction_split=test_fraction_split, diff --git a/samples/model-builder/create_training_pipeline_custom_job_test.py b/samples/model-builder/create_training_pipeline_custom_job_test.py index 2dbfeaf9cb..a47a14ad90 100644 --- a/samples/model-builder/create_training_pipeline_custom_job_test.py +++ b/samples/model-builder/create_training_pipeline_custom_job_test.py @@ -19,26 +19,22 @@ def test_create_training_pipeline_custom_job_sample( mock_sdk_init, - mock_image_dataset, mock_init_custom_training_job, mock_run_custom_training_job, - mock_get_image_dataset, ): create_training_pipeline_custom_job_sample.create_training_pipeline_custom_job_sample( project=constants.PROJECT, display_name=constants.DISPLAY_NAME, + args=constants.ARGS, script_path=constants.SCRIPT_PATH, container_uri=constants.CONTAINER_URI, - dataset_id=constants.RESOURCE_ID, model_display_name=constants.DISPLAY_NAME_2, training_fraction_split=constants.TRAINING_FRACTION_SPLIT, validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT, test_fraction_split=constants.TEST_FRACTION_SPLIT, ) - mock_get_image_dataset.assert_called_once_with(constants.RESOURCE_ID) - mock_sdk_init.assert_called_once_with( project=constants.PROJECT, location=constants.LOCATION ) @@ -48,8 +44,8 @@ def test_create_training_pipeline_custom_job_sample( container_uri=constants.CONTAINER_URI, ) mock_run_custom_training_job.assert_called_once_with( - dataset=mock_image_dataset, model_display_name=constants.DISPLAY_NAME_2, + args=constants.ARGS, training_fraction_split=constants.TRAINING_FRACTION_SPLIT, validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT, test_fraction_split=constants.TEST_FRACTION_SPLIT, diff --git a/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py new file mode 100644 index 0000000000..1757356fa9 --- /dev/null +++ b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py @@ -0,0 +1,57 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import aiplatform + + +# [START aiplatform_sdk_create_training_pipeline_custom_job_sample] +def create_training_pipeline_custom_training_managed_dataset_sample( + project: str, + display_name: str, + script_path: str, + container_uri: str, + dataset_id: int, + location: str = "us-central1", + model_display_name: str = None, + training_fraction_split: float = 0.8, + validation_fraction_split: float = 0.1, + test_fraction_split: float = 0.1, + sync: bool = True, +): + aiplatform.init(project=project, location=location) + + job = aiplatform.CustomTrainingJob(display_name=display_name, + script_path=script_path, + container_uri=container_uri) + + my_image_ds = aiplatform.ImageDataset(dataset_id) + + model = job.run( + dataset=my_image_ds, + model_display_name=model_display_name, + training_fraction_split=training_fraction_split, + validation_fraction_split=validation_fraction_split, + test_fraction_split=test_fraction_split, + sync=sync, + ) + + model.wait() + + print(model.display_name) + print(model.resource_name) + print(model.uri) + return model + + +# [END aiplatform_sdk_create_training_pipeline_custom_job_sample] diff --git a/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_test.py b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_test.py new file mode 100644 index 0000000000..59cddc4212 --- /dev/null +++ b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_test.py @@ -0,0 +1,57 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import create_training_pipeline_custom_training_managed_dataset_sample +import test_constants as constants + + +def test_create_training_pipeline_custom_job_sample( + mock_sdk_init, + mock_image_dataset, + mock_init_custom_training_job, + mock_run_custom_training_job, + mock_get_image_dataset, +): + + create_training_pipeline_custom_training_managed_dataset_sample.create_training_pipeline_custom_training_managed_dataset_sample( + project=constants.PROJECT, + display_name=constants.DISPLAY_NAME, + script_path=constants.SCRIPT_PATH, + container_uri=constants.CONTAINER_URI, + dataset_id=constants.RESOURCE_ID, + model_display_name=constants.DISPLAY_NAME_2, + training_fraction_split=constants.TRAINING_FRACTION_SPLIT, + validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT, + test_fraction_split=constants.TEST_FRACTION_SPLIT, + ) + + mock_get_image_dataset.assert_called_once_with(constants.RESOURCE_ID) + + mock_sdk_init.assert_called_once_with( + project=constants.PROJECT, location=constants.LOCATION + ) + mock_init_custom_training_job.assert_called_once_with( + display_name=constants.DISPLAY_NAME, + script_path=constants.SCRIPT_PATH, + container_uri=constants.CONTAINER_URI, + ) + mock_run_custom_training_job.assert_called_once_with( + dataset=mock_image_dataset, + model_display_name=constants.DISPLAY_NAME_2, + training_fraction_split=constants.TRAINING_FRACTION_SPLIT, + validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT, + test_fraction_split=constants.TEST_FRACTION_SPLIT, + sync=True, + ) diff --git a/samples/model-builder/test_constants.py b/samples/model-builder/test_constants.py index ef9098da78..67430bf46c 100644 --- a/samples/model-builder/test_constants.py +++ b/samples/model-builder/test_constants.py @@ -54,4 +54,5 @@ PREDICTION_TEXT_INSTANCE = "This is some text for testing NLP prediction output" SCRIPT_PATH = "task.py" -CONTAINER_URI = "gcr.io/my_project/my_image:latest" \ No newline at end of file +CONTAINER_URI = "gcr.io/my_project/my_image:latest" +ARGS = ["--tfds", "tf_flowers:3.*.*"] \ No newline at end of file From cca3652ab0e3f012637247b3e0f9fb014402a20f Mon Sep 17 00:00:00 2001 From: ivanmkc Date: Wed, 21 Apr 2021 13:27:25 -0400 Subject: [PATCH 5/9] Fixed args --- .../create_training_pipeline_custom_job_sample.py | 10 ++++++---- .../create_training_pipeline_custom_job_test.py | 2 ++ ..._pipeline_custom_training_managed_dataset_sample.py | 10 +++++++--- ...ng_pipeline_custom_training_managed_dataset_test.py | 4 ++++ ...te_training_pipeline_image_classification_sample.py | 4 ++-- 5 files changed, 21 insertions(+), 9 deletions(-) diff --git a/samples/model-builder/create_training_pipeline_custom_job_sample.py b/samples/model-builder/create_training_pipeline_custom_job_sample.py index 33cdb60aad..d082cd2ad1 100644 --- a/samples/model-builder/create_training_pipeline_custom_job_sample.py +++ b/samples/model-builder/create_training_pipeline_custom_job_sample.py @@ -13,18 +13,19 @@ # limitations under the License. from google.cloud import aiplatform -from typing import List +from typing import List, Optional, Union # [START aiplatform_sdk_create_training_pipeline_custom_job_sample] def create_training_pipeline_custom_job_sample( project: str, display_name: str, - args: List[str], script_path: str, container_uri: str, + model_serving_container_image_uri: str, + args: Optional[List[Union[str, float, int]]] = None, location: str = "us-central1", - model_display_name: str = None, + model_display_name: Optional[str] = None, training_fraction_split: float = 0.8, validation_fraction_split: float = 0.1, test_fraction_split: float = 0.1, @@ -34,7 +35,8 @@ def create_training_pipeline_custom_job_sample( job = aiplatform.CustomTrainingJob(display_name=display_name, script_path=script_path, - container_uri=container_uri) + container_uri=container_uri, + model_serving_container_image_uri=model_serving_container_image_uri) model = job.run( model_display_name=model_display_name, diff --git a/samples/model-builder/create_training_pipeline_custom_job_test.py b/samples/model-builder/create_training_pipeline_custom_job_test.py index a47a14ad90..49ff4eb5f1 100644 --- a/samples/model-builder/create_training_pipeline_custom_job_test.py +++ b/samples/model-builder/create_training_pipeline_custom_job_test.py @@ -29,6 +29,7 @@ def test_create_training_pipeline_custom_job_sample( args=constants.ARGS, script_path=constants.SCRIPT_PATH, container_uri=constants.CONTAINER_URI, + model_serving_container_image_uri=constants.CONTAINER_URI, model_display_name=constants.DISPLAY_NAME_2, training_fraction_split=constants.TRAINING_FRACTION_SPLIT, validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT, @@ -42,6 +43,7 @@ def test_create_training_pipeline_custom_job_sample( display_name=constants.DISPLAY_NAME, script_path=constants.SCRIPT_PATH, container_uri=constants.CONTAINER_URI, + model_serving_container_image_uri=constants.CONTAINER_URI, ) mock_run_custom_training_job.assert_called_once_with( model_display_name=constants.DISPLAY_NAME_2, diff --git a/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py index 1757356fa9..6d4cd10514 100644 --- a/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py +++ b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py @@ -13,7 +13,7 @@ # limitations under the License. from google.cloud import aiplatform - +from typing import List, Optional, Union # [START aiplatform_sdk_create_training_pipeline_custom_job_sample] def create_training_pipeline_custom_training_managed_dataset_sample( @@ -21,9 +21,11 @@ def create_training_pipeline_custom_training_managed_dataset_sample( display_name: str, script_path: str, container_uri: str, + model_serving_container_image_uri: str, dataset_id: int, + args: Optional[List[Union[str, float, int]]] = None, location: str = "us-central1", - model_display_name: str = None, + model_display_name: Optional[str] = None, training_fraction_split: float = 0.8, validation_fraction_split: float = 0.1, test_fraction_split: float = 0.1, @@ -33,7 +35,8 @@ def create_training_pipeline_custom_training_managed_dataset_sample( job = aiplatform.CustomTrainingJob(display_name=display_name, script_path=script_path, - container_uri=container_uri) + container_uri=container_uri, + model_serving_container_image_uri=model_serving_container_image_uri) my_image_ds = aiplatform.ImageDataset(dataset_id) @@ -43,6 +46,7 @@ def create_training_pipeline_custom_training_managed_dataset_sample( training_fraction_split=training_fraction_split, validation_fraction_split=validation_fraction_split, test_fraction_split=test_fraction_split, + args=args, sync=sync, ) diff --git a/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_test.py b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_test.py index 59cddc4212..9d32b23efd 100644 --- a/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_test.py +++ b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_test.py @@ -28,8 +28,10 @@ def test_create_training_pipeline_custom_job_sample( create_training_pipeline_custom_training_managed_dataset_sample.create_training_pipeline_custom_training_managed_dataset_sample( project=constants.PROJECT, display_name=constants.DISPLAY_NAME, + args=constants.ARGS, script_path=constants.SCRIPT_PATH, container_uri=constants.CONTAINER_URI, + model_serving_container_image_uri=constants.CONTAINER_URI, dataset_id=constants.RESOURCE_ID, model_display_name=constants.DISPLAY_NAME_2, training_fraction_split=constants.TRAINING_FRACTION_SPLIT, @@ -46,10 +48,12 @@ def test_create_training_pipeline_custom_job_sample( display_name=constants.DISPLAY_NAME, script_path=constants.SCRIPT_PATH, container_uri=constants.CONTAINER_URI, + model_serving_container_image_uri=constants.CONTAINER_URI, ) mock_run_custom_training_job.assert_called_once_with( dataset=mock_image_dataset, model_display_name=constants.DISPLAY_NAME_2, + args=constants.ARGS, training_fraction_split=constants.TRAINING_FRACTION_SPLIT, validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT, test_fraction_split=constants.TEST_FRACTION_SPLIT, diff --git a/samples/model-builder/create_training_pipeline_image_classification_sample.py b/samples/model-builder/create_training_pipeline_image_classification_sample.py index 050d40af82..3a3ed0b3aa 100644 --- a/samples/model-builder/create_training_pipeline_image_classification_sample.py +++ b/samples/model-builder/create_training_pipeline_image_classification_sample.py @@ -13,7 +13,7 @@ # limitations under the License. from google.cloud import aiplatform - +from typing import Optional # [START aiplatform_sdk_create_training_pipeline_image_classification_sample] def create_training_pipeline_image_classification_sample( @@ -21,7 +21,7 @@ def create_training_pipeline_image_classification_sample( display_name: str, dataset_id: int, location: str = "us-central1", - model_display_name: str = None, + model_display_name: Optional[str] = None, training_fraction_split: float = 0.8, validation_fraction_split: float = 0.1, test_fraction_split: float = 0.1, From 75505d1f34138e9d2ae9e4330a1648e3f279fb6b Mon Sep 17 00:00:00 2001 From: ivanmkc Date: Wed, 21 Apr 2021 13:28:58 -0400 Subject: [PATCH 6/9] Deleted duplicated samples --- ..._import_dataset_tabular_bigquery_sample.py | 43 ------------------- ...nd_import_dataset_tabular_bigquery_test.py | 38 ---------------- ...e_and_import_dataset_tabular_gcs_sample.py | 43 ------------------- ...ate_and_import_dataset_tabular_gcs_test.py | 38 ---------------- 4 files changed, 162 deletions(-) delete mode 100644 samples/model-builder/create_and_import_dataset_tabular_bigquery_sample.py delete mode 100644 samples/model-builder/create_and_import_dataset_tabular_bigquery_test.py delete mode 100644 samples/model-builder/create_and_import_dataset_tabular_gcs_sample.py delete mode 100644 samples/model-builder/create_and_import_dataset_tabular_gcs_test.py diff --git a/samples/model-builder/create_and_import_dataset_tabular_bigquery_sample.py b/samples/model-builder/create_and_import_dataset_tabular_bigquery_sample.py deleted file mode 100644 index 4be34ff12b..0000000000 --- a/samples/model-builder/create_and_import_dataset_tabular_bigquery_sample.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Union - -from google.cloud import aiplatform - - -# [START aiplatform_sdk_create_and_import_dataset_tabular_bigquery_sample] -def create_and_import_dataset_tabular_bigquery_sample( - project: str, - location: str, - display_name: str, - src_uris: Union[str, List[str]], - sync: bool = True, -): - aiplatform.init(project=project, location=location) - - ds = aiplatform.TabularDataset.create( - display_name=display_name, - bq_source=src_uris, - sync=sync, - ) - - ds.wait() - - print(ds.display_name) - print(ds.resource_name) - return ds - - -# [END aiplatform_sdk_create_and_import_dataset_tabular_bigquery_sample] diff --git a/samples/model-builder/create_and_import_dataset_tabular_bigquery_test.py b/samples/model-builder/create_and_import_dataset_tabular_bigquery_test.py deleted file mode 100644 index 51118fa56c..0000000000 --- a/samples/model-builder/create_and_import_dataset_tabular_bigquery_test.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from google.cloud.aiplatform import schema - -import create_and_import_dataset_tabular_bigquery_sample -import test_constants as constants - - -def test_create_and_import_dataset_tabular_bigquery_sample(mock_sdk_init, mock_create_tabular_dataset): - - create_and_import_dataset_tabular_bigquery_sample.create_and_import_dataset_tabular_bigquery_sample( - project=constants.PROJECT, - location=constants.LOCATION, - src_uris=constants.BIGQUERY_SOURCE, - display_name=constants.DISPLAY_NAME, - ) - - mock_sdk_init.assert_called_once_with( - project=constants.PROJECT, location=constants.LOCATION - ) - mock_create_tabular_dataset.assert_called_once_with( - display_name=constants.DISPLAY_NAME, - bq_source=constants.BIGQUERY_SOURCE, - sync=True, - ) diff --git a/samples/model-builder/create_and_import_dataset_tabular_gcs_sample.py b/samples/model-builder/create_and_import_dataset_tabular_gcs_sample.py deleted file mode 100644 index 62deb4b617..0000000000 --- a/samples/model-builder/create_and_import_dataset_tabular_gcs_sample.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Union - -from google.cloud import aiplatform - - -# [START aiplatform_sdk_create_and_import_dataset_tabular_gcs_sample] -def create_and_import_dataset_tabular_gcs_sample( - project: str, - location: str, - display_name: str, - src_uris: Union[str, List[str]], - sync: bool = True, -): - aiplatform.init(project=project, location=location) - - ds = aiplatform.TabularDataset.create( - display_name=display_name, - gcs_source=src_uris, - sync=sync, - ) - - ds.wait() - - print(ds.display_name) - print(ds.resource_name) - return ds - - -# [END aiplatform_sdk_create_and_import_dataset_tabular_gcs_sample] diff --git a/samples/model-builder/create_and_import_dataset_tabular_gcs_test.py b/samples/model-builder/create_and_import_dataset_tabular_gcs_test.py deleted file mode 100644 index e60a3b11e2..0000000000 --- a/samples/model-builder/create_and_import_dataset_tabular_gcs_test.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from google.cloud.aiplatform import schema - -import create_and_import_dataset_tabular_gcs_sample -import test_constants as constants - - -def test_create_and_import_dataset_tabular_gcs_sample(mock_sdk_init, mock_create_tabular_dataset): - - create_and_import_dataset_tabular_gcs_sample.create_and_import_dataset_tabular_gcs_sample( - project=constants.PROJECT, - location=constants.LOCATION, - src_uris=constants.GCS_SOURCES, - display_name=constants.DISPLAY_NAME, - ) - - mock_sdk_init.assert_called_once_with( - project=constants.PROJECT, location=constants.LOCATION - ) - mock_create_tabular_dataset.assert_called_once_with( - display_name=constants.DISPLAY_NAME, - gcs_source=constants.GCS_SOURCES, - sync=True, - ) From fec3e6c96e960f853094aaf1ecbd3446fd2b83d7 Mon Sep 17 00:00:00 2001 From: ivanmkc Date: Wed, 21 Apr 2021 14:31:06 -0400 Subject: [PATCH 7/9] Added more args to samples --- ...eate_training_pipeline_custom_job_sample.py | 14 +++++++++++--- ...create_training_pipeline_custom_job_test.py | 9 +++++++++ ...e_custom_training_managed_dataset_sample.py | 18 +++++++++++++----- ...ine_custom_training_managed_dataset_test.py | 9 +++++++++ ...ing_pipeline_image_classification_sample.py | 4 ++-- ...ipeline_image_classification_sample_test.py | 1 + samples/model-builder/test_constants.py | 6 +++++- 7 files changed, 50 insertions(+), 11 deletions(-) diff --git a/samples/model-builder/create_training_pipeline_custom_job_sample.py b/samples/model-builder/create_training_pipeline_custom_job_sample.py index d082cd2ad1..830ea736b6 100644 --- a/samples/model-builder/create_training_pipeline_custom_job_sample.py +++ b/samples/model-builder/create_training_pipeline_custom_job_sample.py @@ -19,13 +19,17 @@ # [START aiplatform_sdk_create_training_pipeline_custom_job_sample] def create_training_pipeline_custom_job_sample( project: str, + location: str, display_name: str, script_path: str, container_uri: str, - model_serving_container_image_uri: str, - args: Optional[List[Union[str, float, int]]] = None, - location: str = "us-central1", + model_serving_container_image_uri: str, model_display_name: Optional[str] = None, + args: Optional[List[Union[str, float, int]]] = None, + replica_count: int = 0, + machine_type: str = "n1-standard-4", + accelerator_type: str = "ACCELERATOR_TYPE_UNSPECIFIED", + accelerator_count: int = 0, training_fraction_split: float = 0.8, validation_fraction_split: float = 0.1, test_fraction_split: float = 0.1, @@ -41,6 +45,10 @@ def create_training_pipeline_custom_job_sample( model = job.run( model_display_name=model_display_name, args=args, + replica_count=replica_count, + machine_type=machine_type, + accelerator_type=accelerator_type, + accelerator_count=accelerator_count, training_fraction_split=training_fraction_split, validation_fraction_split=validation_fraction_split, test_fraction_split=test_fraction_split, diff --git a/samples/model-builder/create_training_pipeline_custom_job_test.py b/samples/model-builder/create_training_pipeline_custom_job_test.py index 49ff4eb5f1..3378b782dc 100644 --- a/samples/model-builder/create_training_pipeline_custom_job_test.py +++ b/samples/model-builder/create_training_pipeline_custom_job_test.py @@ -25,12 +25,17 @@ def test_create_training_pipeline_custom_job_sample( create_training_pipeline_custom_job_sample.create_training_pipeline_custom_job_sample( project=constants.PROJECT, + location=constants.LOCATION, display_name=constants.DISPLAY_NAME, args=constants.ARGS, script_path=constants.SCRIPT_PATH, container_uri=constants.CONTAINER_URI, model_serving_container_image_uri=constants.CONTAINER_URI, model_display_name=constants.DISPLAY_NAME_2, + replica_count=constants.REPLICA_COUNT, + machine_type=constants.MACHINE_TYPE, + accelerator_type=constants.ACCELERATOR_TYPE, + accelerator_count=constants.ACCELERATOR_COUNT, training_fraction_split=constants.TRAINING_FRACTION_SPLIT, validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT, test_fraction_split=constants.TEST_FRACTION_SPLIT, @@ -47,6 +52,10 @@ def test_create_training_pipeline_custom_job_sample( ) mock_run_custom_training_job.assert_called_once_with( model_display_name=constants.DISPLAY_NAME_2, + replica_count=constants.REPLICA_COUNT, + machine_type=constants.MACHINE_TYPE, + accelerator_type=constants.ACCELERATOR_TYPE, + accelerator_count=constants.ACCELERATOR_COUNT, args=constants.ARGS, training_fraction_split=constants.TRAINING_FRACTION_SPLIT, validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT, diff --git a/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py index 6d4cd10514..89b6f5bcb2 100644 --- a/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py +++ b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py @@ -18,14 +18,18 @@ # [START aiplatform_sdk_create_training_pipeline_custom_job_sample] def create_training_pipeline_custom_training_managed_dataset_sample( project: str, + location: str, display_name: str, script_path: str, container_uri: str, model_serving_container_image_uri: str, - dataset_id: int, - args: Optional[List[Union[str, float, int]]] = None, - location: str = "us-central1", + dataset_id: int, model_display_name: Optional[str] = None, + args: Optional[List[Union[str, float, int]]] = None, + replica_count: int = 0, + machine_type: str = "n1-standard-4", + accelerator_type: str = "ACCELERATOR_TYPE_UNSPECIFIED", + accelerator_count: int = 0, training_fraction_split: float = 0.8, validation_fraction_split: float = 0.1, test_fraction_split: float = 0.1, @@ -43,10 +47,14 @@ def create_training_pipeline_custom_training_managed_dataset_sample( model = job.run( dataset=my_image_ds, model_display_name=model_display_name, + args=args, + replica_count=replica_count, + machine_type=machine_type, + accelerator_type=accelerator_type, + accelerator_count=accelerator_count, training_fraction_split=training_fraction_split, validation_fraction_split=validation_fraction_split, - test_fraction_split=test_fraction_split, - args=args, + test_fraction_split=test_fraction_split, sync=sync, ) diff --git a/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_test.py b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_test.py index 9d32b23efd..a97e84ed43 100644 --- a/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_test.py +++ b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_test.py @@ -27,6 +27,7 @@ def test_create_training_pipeline_custom_job_sample( create_training_pipeline_custom_training_managed_dataset_sample.create_training_pipeline_custom_training_managed_dataset_sample( project=constants.PROJECT, + location=constants.LOCATION, display_name=constants.DISPLAY_NAME, args=constants.ARGS, script_path=constants.SCRIPT_PATH, @@ -34,6 +35,10 @@ def test_create_training_pipeline_custom_job_sample( model_serving_container_image_uri=constants.CONTAINER_URI, dataset_id=constants.RESOURCE_ID, model_display_name=constants.DISPLAY_NAME_2, + replica_count=constants.REPLICA_COUNT, + machine_type=constants.MACHINE_TYPE, + accelerator_type=constants.ACCELERATOR_TYPE, + accelerator_count=constants.ACCELERATOR_COUNT, training_fraction_split=constants.TRAINING_FRACTION_SPLIT, validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT, test_fraction_split=constants.TEST_FRACTION_SPLIT, @@ -54,6 +59,10 @@ def test_create_training_pipeline_custom_job_sample( dataset=mock_image_dataset, model_display_name=constants.DISPLAY_NAME_2, args=constants.ARGS, + replica_count=constants.REPLICA_COUNT, + machine_type=constants.MACHINE_TYPE, + accelerator_type=constants.ACCELERATOR_TYPE, + accelerator_count=constants.ACCELERATOR_COUNT, training_fraction_split=constants.TRAINING_FRACTION_SPLIT, validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT, test_fraction_split=constants.TEST_FRACTION_SPLIT, diff --git a/samples/model-builder/create_training_pipeline_image_classification_sample.py b/samples/model-builder/create_training_pipeline_image_classification_sample.py index 3a3ed0b3aa..dbf629dd59 100644 --- a/samples/model-builder/create_training_pipeline_image_classification_sample.py +++ b/samples/model-builder/create_training_pipeline_image_classification_sample.py @@ -18,9 +18,9 @@ # [START aiplatform_sdk_create_training_pipeline_image_classification_sample] def create_training_pipeline_image_classification_sample( project: str, + location: str, display_name: str, - dataset_id: int, - location: str = "us-central1", + dataset_id: int, model_display_name: Optional[str] = None, training_fraction_split: float = 0.8, validation_fraction_split: float = 0.1, diff --git a/samples/model-builder/create_training_pipeline_image_classification_sample_test.py b/samples/model-builder/create_training_pipeline_image_classification_sample_test.py index c49e0e5f05..cb91898938 100644 --- a/samples/model-builder/create_training_pipeline_image_classification_sample_test.py +++ b/samples/model-builder/create_training_pipeline_image_classification_sample_test.py @@ -27,6 +27,7 @@ def test_create_training_pipeline_image_classification_sample( create_training_pipeline_image_classification_sample.create_training_pipeline_image_classification_sample( project=constants.PROJECT, + location=constants.LOCATION, display_name=constants.DISPLAY_NAME, dataset_id=constants.RESOURCE_ID, model_display_name=constants.DISPLAY_NAME_2, diff --git a/samples/model-builder/test_constants.py b/samples/model-builder/test_constants.py index 67430bf46c..7d7561a7b0 100644 --- a/samples/model-builder/test_constants.py +++ b/samples/model-builder/test_constants.py @@ -55,4 +55,8 @@ SCRIPT_PATH = "task.py" CONTAINER_URI = "gcr.io/my_project/my_image:latest" -ARGS = ["--tfds", "tf_flowers:3.*.*"] \ No newline at end of file +ARGS = ["--tfds", "tf_flowers:3.*.*"] +REPLICA_COUNT = 0 +MACHINE_TYPE = "n1-standard-4" +ACCELERATOR_TYPE = "ACCELERATOR_TYPE_UNSPECIFIED" +ACCELERATOR_COUNT = 0 \ No newline at end of file From f2b0a0b5ed277208c32b614102b292b2f60a3668 Mon Sep 17 00:00:00 2001 From: ivanmkc Date: Fri, 23 Apr 2021 17:29:50 -0400 Subject: [PATCH 8/9] Ran black --- .../create_training_pipeline_custom_job_sample.py | 11 +++++++---- .../create_training_pipeline_custom_job_test.py | 4 +--- ...pipeline_custom_training_managed_dataset_sample.py | 10 ++++++---- ...g_pipeline_custom_training_managed_dataset_test.py | 2 +- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/samples/model-builder/create_training_pipeline_custom_job_sample.py b/samples/model-builder/create_training_pipeline_custom_job_sample.py index 830ea736b6..cea8d25cde 100644 --- a/samples/model-builder/create_training_pipeline_custom_job_sample.py +++ b/samples/model-builder/create_training_pipeline_custom_job_sample.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.cloud import aiplatform from typing import List, Optional, Union +from google.cloud import aiplatform + # [START aiplatform_sdk_create_training_pipeline_custom_job_sample] def create_training_pipeline_custom_job_sample( @@ -23,7 +24,7 @@ def create_training_pipeline_custom_job_sample( display_name: str, script_path: str, container_uri: str, - model_serving_container_image_uri: str, + model_serving_container_image_uri: str, model_display_name: Optional[str] = None, args: Optional[List[Union[str, float, int]]] = None, replica_count: int = 0, @@ -37,10 +38,12 @@ def create_training_pipeline_custom_job_sample( ): aiplatform.init(project=project, location=location) - job = aiplatform.CustomTrainingJob(display_name=display_name, + job = aiplatform.CustomTrainingJob( + display_name=display_name, script_path=script_path, container_uri=container_uri, - model_serving_container_image_uri=model_serving_container_image_uri) + model_serving_container_image_uri=model_serving_container_image_uri, + ) model = job.run( model_display_name=model_display_name, diff --git a/samples/model-builder/create_training_pipeline_custom_job_test.py b/samples/model-builder/create_training_pipeline_custom_job_test.py index 3378b782dc..f01fd94a9a 100644 --- a/samples/model-builder/create_training_pipeline_custom_job_test.py +++ b/samples/model-builder/create_training_pipeline_custom_job_test.py @@ -18,9 +18,7 @@ def test_create_training_pipeline_custom_job_sample( - mock_sdk_init, - mock_init_custom_training_job, - mock_run_custom_training_job, + mock_sdk_init, mock_init_custom_training_job, mock_run_custom_training_job, ): create_training_pipeline_custom_job_sample.create_training_pipeline_custom_job_sample( diff --git a/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py index 89b6f5bcb2..bcea3a2bf8 100644 --- a/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py +++ b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py @@ -23,7 +23,7 @@ def create_training_pipeline_custom_training_managed_dataset_sample( script_path: str, container_uri: str, model_serving_container_image_uri: str, - dataset_id: int, + dataset_id: int, model_display_name: Optional[str] = None, args: Optional[List[Union[str, float, int]]] = None, replica_count: int = 0, @@ -37,10 +37,12 @@ def create_training_pipeline_custom_training_managed_dataset_sample( ): aiplatform.init(project=project, location=location) - job = aiplatform.CustomTrainingJob(display_name=display_name, + job = aiplatform.CustomTrainingJob( + display_name=display_name, script_path=script_path, container_uri=container_uri, - model_serving_container_image_uri=model_serving_container_image_uri) + model_serving_container_image_uri=model_serving_container_image_uri, + ) my_image_ds = aiplatform.ImageDataset(dataset_id) @@ -54,7 +56,7 @@ def create_training_pipeline_custom_training_managed_dataset_sample( accelerator_count=accelerator_count, training_fraction_split=training_fraction_split, validation_fraction_split=validation_fraction_split, - test_fraction_split=test_fraction_split, + test_fraction_split=test_fraction_split, sync=sync, ) diff --git a/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_test.py b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_test.py index a97e84ed43..4197f658b1 100644 --- a/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_test.py +++ b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_test.py @@ -62,7 +62,7 @@ def test_create_training_pipeline_custom_job_sample( replica_count=constants.REPLICA_COUNT, machine_type=constants.MACHINE_TYPE, accelerator_type=constants.ACCELERATOR_TYPE, - accelerator_count=constants.ACCELERATOR_COUNT, + accelerator_count=constants.ACCELERATOR_COUNT, training_fraction_split=constants.TRAINING_FRACTION_SPLIT, validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT, test_fraction_split=constants.TEST_FRACTION_SPLIT, From 0c06fdd197a2fa45893f141305427ad1668e517f Mon Sep 17 00:00:00 2001 From: ivanmkc Date: Mon, 26 Apr 2021 20:31:23 -0400 Subject: [PATCH 9/9] Ran linter --- samples/model-builder/conftest.py | 6 +++--- ...ining_pipeline_custom_training_managed_dataset_sample.py | 4 +++- .../create_training_pipeline_image_classification_sample.py | 6 ++++-- samples/model-builder/test_constants.py | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/samples/model-builder/conftest.py b/samples/model-builder/conftest.py index b3ce6d149c..01e944b19c 100644 --- a/samples/model-builder/conftest.py +++ b/samples/model-builder/conftest.py @@ -151,11 +151,10 @@ def mock_run_automl_image_training_job(): with patch.object(aiplatform.training_jobs.AutoMLImageTrainingJob, "run") as mock: yield mock + @pytest.fixture def mock_init_custom_training_job(): - with patch.object( - aiplatform.training_jobs.CustomTrainingJob, "__init__" - ) as mock: + with patch.object(aiplatform.training_jobs.CustomTrainingJob, "__init__") as mock: mock.return_value = None yield mock @@ -165,6 +164,7 @@ def mock_run_custom_training_job(): with patch.object(aiplatform.training_jobs.CustomTrainingJob, "run") as mock: yield mock + """ ---------------------------------------------------------------------------- Model Fixtures diff --git a/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py index bcea3a2bf8..7d7dc6357c 100644 --- a/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py +++ b/samples/model-builder/create_training_pipeline_custom_training_managed_dataset_sample.py @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.cloud import aiplatform from typing import List, Optional, Union +from google.cloud import aiplatform + + # [START aiplatform_sdk_create_training_pipeline_custom_job_sample] def create_training_pipeline_custom_training_managed_dataset_sample( project: str, diff --git a/samples/model-builder/create_training_pipeline_image_classification_sample.py b/samples/model-builder/create_training_pipeline_image_classification_sample.py index dbf629dd59..3786894a05 100644 --- a/samples/model-builder/create_training_pipeline_image_classification_sample.py +++ b/samples/model-builder/create_training_pipeline_image_classification_sample.py @@ -12,15 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.cloud import aiplatform from typing import Optional +from google.cloud import aiplatform + + # [START aiplatform_sdk_create_training_pipeline_image_classification_sample] def create_training_pipeline_image_classification_sample( project: str, location: str, display_name: str, - dataset_id: int, + dataset_id: int, model_display_name: Optional[str] = None, training_fraction_split: float = 0.8, validation_fraction_split: float = 0.1, diff --git a/samples/model-builder/test_constants.py b/samples/model-builder/test_constants.py index 7d7561a7b0..1a27b2c530 100644 --- a/samples/model-builder/test_constants.py +++ b/samples/model-builder/test_constants.py @@ -59,4 +59,4 @@ REPLICA_COUNT = 0 MACHINE_TYPE = "n1-standard-4" ACCELERATOR_TYPE = "ACCELERATOR_TYPE_UNSPECIFIED" -ACCELERATOR_COUNT = 0 \ No newline at end of file +ACCELERATOR_COUNT = 0