From 24d6920a530995a3a5c6ad6b25d0b867f7aebe27 Mon Sep 17 00:00:00 2001 From: Morgan Du Date: Mon, 3 May 2021 13:30:40 -0700 Subject: [PATCH] Feat: add mbsdk video dataset samples (#307) * fix: rebase and resolve conflict * fix: lint --- samples/model-builder/conftest.py | 14 ++++-- .../create_and_import_dataset_video_sample.py | 44 ++++++++++++++++++ ...te_and_import_dataset_video_sample_test.py | 42 +++++++++++++++++ ...rt_data_video_action_recognition_sample.py | 45 ++++++++++++++++++ ...ta_video_action_recognition_sample_test.py | 44 ++++++++++++++++++ ...import_data_video_classification_sample.py | 46 +++++++++++++++++++ ...t_data_video_classification_sample_test.py | 42 +++++++++++++++++ ...mport_data_video_object_tracking_sample.py | 45 ++++++++++++++++++ ..._data_video_object_tracking_sample_test.py | 44 ++++++++++++++++++ 9 files changed, 361 insertions(+), 5 deletions(-) create mode 100644 samples/model-builder/create_and_import_dataset_video_sample.py create mode 100644 samples/model-builder/create_and_import_dataset_video_sample_test.py create mode 100644 samples/model-builder/import_data_video_action_recognition_sample.py create mode 100644 samples/model-builder/import_data_video_action_recognition_sample_test.py create mode 100644 samples/model-builder/import_data_video_classification_sample.py create mode 100644 samples/model-builder/import_data_video_classification_sample_test.py create mode 100644 samples/model-builder/import_data_video_object_tracking_sample.py create mode 100644 samples/model-builder/import_data_video_object_tracking_sample_test.py diff --git a/samples/model-builder/conftest.py b/samples/model-builder/conftest.py index 718f4645ff..112d5c200b 100644 --- a/samples/model-builder/conftest.py +++ b/samples/model-builder/conftest.py @@ -130,11 +130,15 @@ def mock_import_text_dataset(mock_text_dataset): yield mock -""" ----------------------------------------------------------------------------- -TrainingJob Fixtures ----------------------------------------------------------------------------- -""" +@pytest.fixture +def mock_import_video_data(mock_video_dataset): + with patch.object(mock_video_dataset, "import_data") as mock: + yield mock + + +# ---------------------------------------------------------------------------- +# TrainingJob Fixtures +# ---------------------------------------------------------------------------- @pytest.fixture diff --git a/samples/model-builder/create_and_import_dataset_video_sample.py b/samples/model-builder/create_and_import_dataset_video_sample.py new file mode 100644 index 0000000000..60cfd65f7a --- /dev/null +++ b/samples/model-builder/create_and_import_dataset_video_sample.py @@ -0,0 +1,44 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Union + +from google.cloud import aiplatform + + +# [START aiplatform_sdk_create_and_import_dataset_video_sample] +def create_and_import_dataset_video_sample( + project: str, + location: str, + display_name: str, + src_uris: Union[str, List[str]], + sync: bool = True, +): + aiplatform.init(project=project, location=location) + + ds = aiplatform.VideoDataset.create( + display_name=display_name, + gcs_source=src_uris, + import_schema_uri=aiplatform.schema.dataset.ioformat.video.classification, + sync=sync, + ) + + ds.wait() + + print(ds.display_name) + print(ds.resource_name) + return ds + + +# [END aiplatform_sdk_create_and_import_dataset_video_sample] diff --git a/samples/model-builder/create_and_import_dataset_video_sample_test.py b/samples/model-builder/create_and_import_dataset_video_sample_test.py new file mode 100644 index 0000000000..1ebbc7a3d0 --- /dev/null +++ b/samples/model-builder/create_and_import_dataset_video_sample_test.py @@ -0,0 +1,42 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from google.cloud.aiplatform import schema + +import create_and_import_dataset_video_sample + +import test_constants as constants + + +def test_create_and_import_dataset_video_sample( + mock_sdk_init, mock_create_video_dataset +): + + create_and_import_dataset_video_sample.create_and_import_dataset_video_sample( + project=constants.PROJECT, + location=constants.LOCATION, + src_uris=constants.GCS_SOURCES, + display_name=constants.DISPLAY_NAME, + ) + + mock_sdk_init.assert_called_once_with( + project=constants.PROJECT, location=constants.LOCATION + ) + mock_create_video_dataset.assert_called_once_with( + display_name=constants.DISPLAY_NAME, + gcs_source=constants.GCS_SOURCES, + import_schema_uri=schema.dataset.ioformat.video.classification, + sync=True, + ) diff --git a/samples/model-builder/import_data_video_action_recognition_sample.py b/samples/model-builder/import_data_video_action_recognition_sample.py new file mode 100644 index 0000000000..dbdab59215 --- /dev/null +++ b/samples/model-builder/import_data_video_action_recognition_sample.py @@ -0,0 +1,45 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Union + +from google.cloud import aiplatform + + +# [START aiplatform_sdk_import_data_video_action_recognition_sample] +def import_data_video_action_recognition_sample( + project: str, + location: str, + dataset_name: str, + src_uris: Union[str, List[str]], + sync: bool = True, +): + aiplatform.init(project=project, location=location) + + ds = aiplatform.VideoDataset(dataset_name=dataset_name) + + ds.import_data( + gcs_source=src_uris, + import_schema_uri=aiplatform.schema.dataset.ioformat.video.action_recognition, + sync=sync, + ) + + ds.wait() + + print(ds.display_name) + print(ds.resource_name) + return ds + + +# [END aiplatform_sdk_import_data_video_action_recognition_sample] diff --git a/samples/model-builder/import_data_video_action_recognition_sample_test.py b/samples/model-builder/import_data_video_action_recognition_sample_test.py new file mode 100644 index 0000000000..f8eef996f2 --- /dev/null +++ b/samples/model-builder/import_data_video_action_recognition_sample_test.py @@ -0,0 +1,44 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud.aiplatform import schema + +import pytest + +import import_data_video_action_recognition_sample + +import test_constants as constants + + +@pytest.mark.usefixtures("mock_get_video_dataset") +def test_import_data_video_action_recognition_sample( + mock_sdk_init, mock_import_video_data +): + + import_data_video_action_recognition_sample.import_data_video_action_recognition_sample( + project=constants.PROJECT, + location=constants.LOCATION, + dataset_name=constants.DATASET_NAME, + src_uris=constants.GCS_SOURCES, + ) + + mock_sdk_init.assert_called_once_with( + project=constants.PROJECT, location=constants.LOCATION + ) + + mock_import_video_data.assert_called_once_with( + gcs_source=constants.GCS_SOURCES, + import_schema_uri=schema.dataset.ioformat.video.action_recognition, + sync=True, + ) diff --git a/samples/model-builder/import_data_video_classification_sample.py b/samples/model-builder/import_data_video_classification_sample.py new file mode 100644 index 0000000000..1e7243746c --- /dev/null +++ b/samples/model-builder/import_data_video_classification_sample.py @@ -0,0 +1,46 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Union + +from google.cloud import aiplatform + + +# [START aiplatform_sdk_import_data_video_classification_sample] +def import_data_video_classification_sample( + project: str, + location: str, + dataset_name: str, + src_uris: Union[str, List[str]], + sync: bool = True, +): + aiplatform.init(project=project, location=location) + + ds = aiplatform.VideoDataset(dataset_name=dataset_name) + + print(ds.display_name) + print(ds.resource_name) + + ds.import_data( + gcs_source=src_uris, + import_schema_uri=aiplatform.schema.dataset.ioformat.video.classification, + sync=sync, + ) + + ds.wait() + + return ds + + +# [END aiplatform_sdk_import_data_video_classification_sample] diff --git a/samples/model-builder/import_data_video_classification_sample_test.py b/samples/model-builder/import_data_video_classification_sample_test.py new file mode 100644 index 0000000000..cce5c0abd6 --- /dev/null +++ b/samples/model-builder/import_data_video_classification_sample_test.py @@ -0,0 +1,42 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud.aiplatform import schema + +import pytest + +import import_data_video_classification_sample + +import test_constants as constants + + +@pytest.mark.usefixtures("mock_get_video_dataset") +def test_import_data_video_classification_sample(mock_sdk_init, mock_import_video_data): + + import_data_video_classification_sample.import_data_video_classification_sample( + project=constants.PROJECT, + location=constants.LOCATION, + dataset_name=constants.DATASET_NAME, + src_uris=constants.GCS_SOURCES, + ) + + mock_sdk_init.assert_called_once_with( + project=constants.PROJECT, location=constants.LOCATION + ) + + mock_import_video_data.assert_called_once_with( + gcs_source=constants.GCS_SOURCES, + import_schema_uri=schema.dataset.ioformat.video.classification, + sync=True, + ) diff --git a/samples/model-builder/import_data_video_object_tracking_sample.py b/samples/model-builder/import_data_video_object_tracking_sample.py new file mode 100644 index 0000000000..ae38748b89 --- /dev/null +++ b/samples/model-builder/import_data_video_object_tracking_sample.py @@ -0,0 +1,45 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Union + +from google.cloud import aiplatform + + +# [START aiplatform_sdk_import_data_video_object_tracking_sample] +def import_data_video_object_tracking_sample( + project: str, + location: str, + dataset_name: str, + src_uris: Union[str, List[str]], + sync: bool = True, +): + aiplatform.init(project=project, location=location) + + ds = aiplatform.VideoDataset(dataset_name=dataset_name) + + ds.import_data( + gcs_source=src_uris, + import_schema_uri=aiplatform.schema.dataset.ioformat.video.object_tracking, + sync=sync, + ) + + ds.wait() + + print(ds.display_name) + print(ds.resource_name) + return ds + + +# [END aiplatform_sdk_import_data_video_object_tracking_sample] diff --git a/samples/model-builder/import_data_video_object_tracking_sample_test.py b/samples/model-builder/import_data_video_object_tracking_sample_test.py new file mode 100644 index 0000000000..2f94bc7dac --- /dev/null +++ b/samples/model-builder/import_data_video_object_tracking_sample_test.py @@ -0,0 +1,44 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud.aiplatform import schema + +import pytest + +import import_data_video_object_tracking_sample + +import test_constants as constants + + +@pytest.mark.usefixtures("mock_get_video_dataset") +def test_import_data_video_object_tracking_sample( + mock_sdk_init, mock_import_video_data +): + + import_data_video_object_tracking_sample.import_data_video_object_tracking_sample( + project=constants.PROJECT, + location=constants.LOCATION, + dataset_name=constants.DATASET_NAME, + src_uris=constants.GCS_SOURCES, + ) + + mock_sdk_init.assert_called_once_with( + project=constants.PROJECT, location=constants.LOCATION + ) + + mock_import_video_data.assert_called_once_with( + gcs_source=constants.GCS_SOURCES, + import_schema_uri=schema.dataset.ioformat.video.object_tracking, + sync=True, + )