diff --git a/samples/snippets/conftest.py b/samples/snippets/conftest.py index 6c2fdf9497..810e2af823 100644 --- a/samples/snippets/conftest.py +++ b/samples/snippets/conftest.py @@ -180,9 +180,11 @@ def teardown_training_pipeline(shared_state, pipeline_client): ) # Waiting for training pipeline to be in CANCELLED state + timeout = shared_state["cancel_batch_prediction_job_timeout"] helpers.wait_for_job_state( get_job_method=pipeline_client.get_training_pipeline, name=shared_state["training_pipeline_name"], + timeout=timeout, ) # Delete the training pipeline diff --git a/samples/snippets/helpers.py b/samples/snippets/helpers.py index 17baa0a34d..67bac774ee 100644 --- a/samples/snippets/helpers.py +++ b/samples/snippets/helpers.py @@ -1,5 +1,7 @@ +import collections import re import time +from timeit import default_timer as timer from typing import Callable @@ -63,3 +65,21 @@ def wait_for_job_state( "\nTry increasing the timeout in sample test" f"\nLast recorded state: {response.state}" ) + + +def flaky_test_diagnostic(file_name, test_name, N=20): + + import pytest + + timing_dict = collections.defaultdict(list) + for ri in range(N): + start = timer() + result = pytest.main(['-s', f'{file_name}::{test_name}']) + end = timer() + delta = end-start + if result == pytest.ExitCode.OK: + timing_dict['SUCCESS'].append(delta) + else: + timing_dict['FAILURE'].append(delta) + + return timing_dict diff --git a/samples/snippets/pipeline_service/create_training_pipeline_tabular_regression_sample_test.py b/samples/snippets/pipeline_service/create_training_pipeline_tabular_regression_sample_test.py index 2d602a7050..ac8583f87b 100644 --- a/samples/snippets/pipeline_service/create_training_pipeline_tabular_regression_sample_test.py +++ b/samples/snippets/pipeline_service/create_training_pipeline_tabular_regression_sample_test.py @@ -34,6 +34,9 @@ def teardown(teardown_training_pipeline): def test_ucaip_generated_create_training_pipeline_sample(capsys, shared_state): + # The return of the cancellation can be flaky; max of 20 runs was 215 sec. + shared_state["cancel_batch_prediction_job_timeout"] = 300 + create_training_pipeline_tabular_regression_sample.create_training_pipeline_tabular_regression_sample( project=PROJECT_ID, display_name=DISPLAY_NAME, diff --git a/scripts/run_flaky_test_diagnostic.py b/scripts/run_flaky_test_diagnostic.py new file mode 100644 index 0000000000..394c701f99 --- /dev/null +++ b/scripts/run_flaky_test_diagnostic.py @@ -0,0 +1,22 @@ +import os +import pathlib +import sys + +# Run test in snippets directory: +dir_of_curr_file = os.path.dirname(__file__) +helper_filepath = pathlib.Path(dir_of_curr_file).parent / 'samples' / 'snippets' +sys.path.append(helper_filepath.resolve().as_posix()) +os.chdir(helper_filepath.resolve()) +from helpers import flaky_test_diagnostic + +# Settings: +file_name = 'pipeline_service/create_training_pipeline_tabular_regression_sample_test.py' +test_name = 'test_ucaip_generated_create_training_pipeline_sample' +timing_dict = flaky_test_diagnostic(file_name, test_name, N=1) + +for key, delta_list in timing_dict.items(): + mean_time = sum(delta_list)/len(delta_list) + max_time = max(delta_list) + min_time = min(delta_list) + report_string = f'Result: {key}, mean={mean_time:3.2f}, min={min_time:3.2f}, max={max_time:3.2f}, count={len(delta_list)}' + print(report_string)