diff --git a/perfmetrics/scripts/ls_metrics/listing_benchmark.py b/perfmetrics/scripts/ls_metrics/listing_benchmark.py index 5be8fed808..f579b013a8 100644 --- a/perfmetrics/scripts/ls_metrics/listing_benchmark.py +++ b/perfmetrics/scripts/ls_metrics/listing_benchmark.py @@ -81,18 +81,18 @@ def _parse_results(folders, results_list, message, num_samples) -> dict: # Sorting based on time. results_list[testing_folder.name] = sorted(results_list[testing_folder.name]) - metrics[testing_folder.name]['Mean'] = stat.mean(results_list[testing_folder.name]) - metrics[testing_folder.name]['Median'] = stat.median(results_list[testing_folder.name]) - metrics[testing_folder.name]['Standard Dev'] = stat.stdev(gcs_bucket_results[testing_folder.name]) + metrics[testing_folder.name]['Mean'] = round(stat.mean(results_list[testing_folder.name]), 3) + metrics[testing_folder.name]['Median'] = round(stat.median(results_list[testing_folder.name]), 3) + metrics[testing_folder.name]['Standard Dev'] = round(stat.stdev(results_list[testing_folder.name]), 3) metrics[testing_folder.name]['Quantiles'] = dict() for percentile in range(0, 100, 20): - metrics[testing_folder.name]['Quantiles']['{} %ile'.format(percentile)] = np.percentile(results_list[testing_folder.name], percentile) - metrics[testing_folder.name]['Quantiles']['90 %ile'] = np.percentile(results_list[testing_folder.name], 90) - metrics[testing_folder.name]['Quantiles']['95 %ile'] = np.percentile(results_list[testing_folder.name], 95) - metrics[testing_folder.name]['Quantiles']['98 %ile'] = np.percentile(results_list[testing_folder.name], 98) - metrics[testing_folder.name]['Quantiles']['99 %ile'] = np.percentile(results_list[testing_folder.name], 99) - metrics[testing_folder.name]['Quantiles']['100 %ile'] = np.percentile(results_list[testing_folder.name], 100) + metrics[testing_folder.name]['Quantiles']['{} %ile'.format(percentile)] = round(np.percentile(results_list[testing_folder.name], percentile), 3) + metrics[testing_folder.name]['Quantiles']['90 %ile'] = round(np.percentile(results_list[testing_folder.name], 90), 3) + metrics[testing_folder.name]['Quantiles']['95 %ile'] = round(np.percentile(results_list[testing_folder.name], 95), 3) + metrics[testing_folder.name]['Quantiles']['98 %ile'] = round(np.percentile(results_list[testing_folder.name], 98), 3) + metrics[testing_folder.name]['Quantiles']['99 %ile'] = round(np.percentile(results_list[testing_folder.name], 99), 3) + metrics[testing_folder.name]['Quantiles']['100 %ile'] = round(np.percentile(results_list[testing_folder.name], 100), 3) print(metrics) return metrics @@ -200,20 +200,19 @@ def _create_directory_structure(gcs_bucket_url, persistent_disk_url, directory_s return int(result > 0) -def _list_directory(path) -> (list, int): - """Returns the list containing path of all the contents present in the current directory and also the number of contents. +def _list_directory(path) -> list: + """Returns the list containing path of all the contents present in the current directory. Args: path: Path of the directory. Returns: - A list containing path of all contents present in the input path. Also returns the number of contents. + A list containing path of all contents present in the input path. """ contents = subprocess.check_output('gsutil -m ls {}'.format(path), shell=True) contents_url = contents.decode('utf-8').split('\n')[:-1] - num_contents = len(contents_url) - return contents_url, num_contents + return contents_url def _compare_directory_structure(url, directory_structure) -> bool: @@ -227,7 +226,7 @@ def _compare_directory_structure(url, directory_structure) -> bool: True if GCS bucket contents matches the directory structure. """ - contents_url, num_contents = _list_directory(url) + contents_url = _list_directory(url) # gsutil in some cases return the contents_url list with the current directory in # the first index. We dont want the current directory so we remove it # manually. diff --git a/perfmetrics/scripts/ls_metrics/listing_benchmark_test.py b/perfmetrics/scripts/ls_metrics/listing_benchmark_test.py new file mode 100644 index 0000000000..7a94ff7717 --- /dev/null +++ b/perfmetrics/scripts/ls_metrics/listing_benchmark_test.py @@ -0,0 +1,523 @@ +"""Tests for listing_benchmark.""" + +import unittest +from unittest import mock + +import directory_pb2 as directory_proto +from google.protobuf.json_format import ParseDict +import listing_benchmark +from mock import patch, call + +# (Type 1) - 0 levels deep directory structure. +DIRECTORY_STRUCTURE1 = { + 'name': 'fake_bucket' +} + +# (Type 2) - 1 level deep directory structure with an empty testing folder. +DIRECTORY_STRUCTURE2 = { + 'name': 'fake_bucket', + 'num_folders': 3, + 'num_files': 1, + 'file_size': '1kb', + 'file_name_prefix': 'file', + 'folders': [ + { + 'name': '2KB_3files_0subdir', + 'num_files': 3, + 'file_name_prefix': 'file', + 'file_size': '2kb' + }, + { + 'name': '1KB_2files_0subdir', + 'num_files': 2, + 'file_size': '1kb', + 'file_name_prefix': 'file' + }, + { + 'name': '1KB_0files_0subdir' + } + ] +} + +# (Type 3) - Multilevel deep directory structure with many edge cases embedded. +DIRECTORY_STRUCTURE3 = { + 'name': 'fake_bucket', + 'num_folders': 3, + 'num_files': 0, + 'folders': [ + { + 'name': '1KB_4files_3subdir', + 'num_files': 4, + 'file_name_prefix': 'file', + 'file_size': '1kb', + 'num_folders': 3, + 'folders': [ + { + 'name': 'subdir1', + 'num_files': 1, + 'file_name_prefix': 'file', + 'file_size': '1kb', + 'num_folders': 2, + 'folders': [ + { + 'name': 'subsubdir1', + 'num_files': 2, + 'file_name_prefix': 'file', + 'file_size': '1kb' + }, + { + 'name': 'subsubdir2' + } + ] + }, + { + 'name': 'subdir2', + 'num_files': 1, + 'file_name_prefix': 'file', + 'file_size': '1kb' + }, + { + 'name': 'subdir3', + 'num_files': 1, + 'file_name_prefix': 'file', + 'file_size': '1kb', + 'num_folders': 1, + 'folders': [ + { + 'name': 'subsubdir1', + 'num_files': 1, + 'file_name_prefix': 'file', + 'file_size': '1kb' + } + ] + } + ] + }, + { + 'name': '2KB_3files_1subdir', + 'num_files': 3, + 'file_name_prefix': 'file', + 'file_size': '2kb', + 'num_folders': 1, + 'folders': [ + { + 'name': 'subdir1' + } + ] + }, + { + 'name': '1KB_1files_0subdir', + 'num_files': 1, + 'file_size': '1kb', + 'file_name_prefix': 'file' + } + ] +} + +# List of latencies (msec) of list operation to test _parse_results method. +METRICS1 = [1.234, 0.995, 0.121, 0.222, 0.01709] +METRICS2 = [90.45, 1.95, 0.334, 7.090, 0.001] +METRICS3 = [100, 7, 6, 51, 21] + +# Converting JSON to protobuf. +DIRECTORY_STRUCTURE1 = ParseDict( + DIRECTORY_STRUCTURE1, directory_proto.Directory()) +DIRECTORY_STRUCTURE2 = ParseDict( + DIRECTORY_STRUCTURE2, directory_proto.Directory()) +DIRECTORY_STRUCTURE3 = ParseDict( + DIRECTORY_STRUCTURE3, directory_proto.Directory()) + + +class ListingBenchmarkTest(unittest.TestCase): + + def test_parse_results_single_level_dir(self): + metrics = listing_benchmark._parse_results( + DIRECTORY_STRUCTURE1.folders, {}, 'fake_test', 5) + self.assertEqual(metrics, {}) + + def test_parse_results_double_level_dir(self): + metrics = listing_benchmark._parse_results(DIRECTORY_STRUCTURE2.folders, { + '2KB_3files_0subdir': METRICS1, + '1KB_2files_0subdir': METRICS2, + '1KB_0files_0subdir': METRICS3 + }, 'fake_test', 5) + self.assertEqual(metrics, + { + '2KB_3files_0subdir': + { + 'Test Desc.': 'fake_test', + 'Number of samples': 5, + 'Mean': 0.518, + 'Median': 0.222, + 'Standard Dev': 0.556, + 'Quantiles': + { + '0 %ile': 0.017, + '20 %ile': 0.1, + '40 %ile': 0.182, + '60 %ile': 0.531, + '80 %ile': 1.043, + '90 %ile': 1.138, + '95 %ile': 1.186, + '98 %ile': 1.215, + '99 %ile': 1.224, + '100 %ile': 1.234 + } + }, + '1KB_2files_0subdir': + { + 'Test Desc.': 'fake_test', + 'Number of samples': 5, + 'Mean': 19.965, + 'Median': 1.95, + 'Standard Dev': 39.504, + 'Quantiles': + { + '0 %ile': 0.001, + '20 %ile': 0.267, + '40 %ile': 1.304, + '60 %ile': 4.006, + '80 %ile': 23.762, + '90 %ile': 57.106, + '95 %ile': 73.778, + '98 %ile': 83.781, + '99 %ile': 87.116, + '100 %ile': 90.45 + } + }, + '1KB_0files_0subdir': + { + 'Test Desc.': 'fake_test', + 'Number of samples': 5, + 'Mean': 37, + 'Median': 21, + 'Standard Dev': 39.63, + 'Quantiles': + { + '0 %ile': 6.0, + '20 %ile': 6.8, + '40 %ile': 15.4, + '60 %ile': 33.0, + '80 %ile': 60.8, + '90 %ile': 80.4, + '95 %ile': 90.2, + '98 %ile': 96.08, + '99 %ile': 98.04, + '100 %ile': 100.0 + } + } + } + ) + + @patch('listing_benchmark.subprocess.call', return_value=1) + @patch('listing_benchmark.time.time', return_value=1) + def test_record_time_of_operation_same_time(self, mock_time, mock_subprocess_call): + result_list = listing_benchmark._record_time_of_operation('ls', 'fakepath/', 5) + self.assertEqual(mock_subprocess_call.call_count, 5) + self.assertEqual(result_list, [0, 0, 0, 0, 0]) + + @patch('listing_benchmark.subprocess.call', return_value=1) + @patch('listing_benchmark.time.time') + def test_record_time_of_operation_different_time(self, mock_time, mock_subprocess_call): + mock_time.side_effect = [1, 2, 3, 5] + result_list = listing_benchmark._record_time_of_operation('ls', 'fakepath/', 2) + self.assertEqual(mock_subprocess_call.call_count, 2) + self.assertEqual(result_list, [1000, 2000]) + + @patch('listing_benchmark._record_time_of_operation') + def test_perform_testing_single_level_dir(self, mock_record_time_of_operation): + mock_record_time_of_operation.return_value = [1, 1, 1] + gcs_bucket_results, persistent_disk_results = listing_benchmark._perform_testing( + DIRECTORY_STRUCTURE1.folders, 'fake_bucket', 'fake_disk', 3, 'ls -R') + self.assertEqual(gcs_bucket_results, persistent_disk_results) + self.assertFalse(mock_record_time_of_operation.called) + self.assertEqual(gcs_bucket_results, {}) + + @patch('listing_benchmark._record_time_of_operation') + def test_perform_testing_double_level_dir(self, mock_record_time_of_operation): + mock_record_time_of_operation.return_value = [1, 1, 1] + gcs_bucket_results, persistent_disk_results = listing_benchmark._perform_testing( + DIRECTORY_STRUCTURE2.folders, 'fake_bucket', 'fake_disk', 3, 'ls -R') + self.assertEqual(gcs_bucket_results, persistent_disk_results) + self.assertTrue(mock_record_time_of_operation.called) + self.assertEqual(gcs_bucket_results, { + '2KB_3files_0subdir': [1, 1, 1], + '1KB_2files_0subdir': [1, 1, 1], + '1KB_0files_0subdir': [1, 1, 1] + }) + + @patch('listing_benchmark._record_time_of_operation', return_value=[1]) + def test_perform_testing_multi_level_dir(self, mock_record_time_of_operation): + mock_record_time_of_operation.return_value = [1, 1] + gcs_bucket_results, persistent_disk_results = listing_benchmark._perform_testing( + DIRECTORY_STRUCTURE3.folders, 'fake_bucket', 'fake_disk', 2, 'ls -R') + self.assertEqual(gcs_bucket_results, persistent_disk_results) + self.assertTrue(mock_record_time_of_operation.called) + self.assertEqual(gcs_bucket_results, { + '1KB_4files_3subdir': [1, 1], + '2KB_3files_1subdir': [1, 1], + '1KB_1files_0subdir': [1, 1] + }) + + @patch('listing_benchmark.subprocess.call', return_value=0) + @patch('listing_benchmark.generate_files.generate_files_and_upload_to_gcs_bucket', return_value=0) + def test_create_directory_structure_single_level_dir(self, mock_generate_files, mock_subprocess_call): + exit_code = listing_benchmark._create_directory_structure( + 'fake_bucket_url/', 'fake_disk_url/', DIRECTORY_STRUCTURE1, True) + self.assertEqual(exit_code, 0) + self.assertEqual(mock_subprocess_call.call_count, 1) + self.assertEqual(mock_generate_files.call_count, 0) + self.assertEqual(mock_subprocess_call.call_args_list, [ + call('mkdir fake_disk_url/', shell=True) + ]) + self.assertEqual(mock_generate_files.call_args_list, []) + + @patch('listing_benchmark.subprocess.call', return_value=0) + @patch('listing_benchmark.generate_files.generate_files_and_upload_to_gcs_bucket', return_value=0) + def test_create_directory_structure_double_level_dir(self, mock_generate_files, mock_subprocess_call): + exit_code = listing_benchmark._create_directory_structure( + 'fake_bucket_url/', 'fake_disk_url/', DIRECTORY_STRUCTURE2, True) + self.assertEqual(exit_code, 0) + self.assertEqual(mock_subprocess_call.call_count, 4) + self.assertEqual(mock_generate_files.call_count, 3) + self.assertEqual(mock_subprocess_call.call_args_list, [ + call('mkdir fake_disk_url/', shell=True), + call('mkdir fake_disk_url/2KB_3files_0subdir/', shell=True), + call('mkdir fake_disk_url/1KB_2files_0subdir/', shell=True), + call('mkdir fake_disk_url/1KB_0files_0subdir/', shell=True) + ]) + self.assertEqual(mock_generate_files.call_args_list, [ + call('fake_bucket_url/', 1, 'kb', 1, 'file', 'fake_disk_url/', True), + call('fake_bucket_url/2KB_3files_0subdir/', 3, 'kb', 2, 'file', + 'fake_disk_url/2KB_3files_0subdir/', True), + call('fake_bucket_url/1KB_2files_0subdir/', 2, 'kb', 1, 'file', + 'fake_disk_url/1KB_2files_0subdir/', True) + ]) + + @patch('listing_benchmark.subprocess.call', return_value=0) + @patch('listing_benchmark.generate_files.generate_files_and_upload_to_gcs_bucket', return_value=0) + def test_create_directory_structure_multi_level_dir(self, mock_generate_files, mock_subprocess_call): + exit_code = listing_benchmark._create_directory_structure( + 'fake_bucket_url/', 'fake_disk_url/', DIRECTORY_STRUCTURE3, True) + self.assertEqual(exit_code, 0) + self.assertEqual(mock_subprocess_call.call_count, 11) + self.assertEqual(mock_generate_files.call_count, 8) + self.assertEqual(mock_subprocess_call.call_args_list, [ + call('mkdir fake_disk_url/', shell=True), + call('mkdir fake_disk_url/1KB_4files_3subdir/', shell=True), + call('mkdir fake_disk_url/1KB_4files_3subdir/subdir1/', shell=True), + call('mkdir fake_disk_url/1KB_4files_3subdir/subdir1/subsubdir1/', shell=True), + call('mkdir fake_disk_url/1KB_4files_3subdir/subdir1/subsubdir2/', shell=True), + call('mkdir fake_disk_url/1KB_4files_3subdir/subdir2/', shell=True), + call('mkdir fake_disk_url/1KB_4files_3subdir/subdir3/', shell=True), + call('mkdir fake_disk_url/1KB_4files_3subdir/subdir3/subsubdir1/', shell=True), + call('mkdir fake_disk_url/2KB_3files_1subdir/', shell=True), + call('mkdir fake_disk_url/2KB_3files_1subdir/subdir1/', shell=True), + call('mkdir fake_disk_url/1KB_1files_0subdir/', shell=True) + ]) + self.assertEqual(mock_generate_files.call_args_list, [ + call('fake_bucket_url/1KB_4files_3subdir/', 4, 'kb', 1, + 'file', 'fake_disk_url/1KB_4files_3subdir/', True), + call('fake_bucket_url/1KB_4files_3subdir/subdir1/', 1, 'kb', 1, + 'file', 'fake_disk_url/1KB_4files_3subdir/subdir1/', True), + call('fake_bucket_url/1KB_4files_3subdir/subdir1/subsubdir1/', 2, 'kb', 1, + 'file', 'fake_disk_url/1KB_4files_3subdir/subdir1/subsubdir1/', True), + call('fake_bucket_url/1KB_4files_3subdir/subdir2/', 1, 'kb', 1, + 'file', 'fake_disk_url/1KB_4files_3subdir/subdir2/', True), + call('fake_bucket_url/1KB_4files_3subdir/subdir3/', 1, 'kb', 1, + 'file', 'fake_disk_url/1KB_4files_3subdir/subdir3/', True), + call('fake_bucket_url/1KB_4files_3subdir/subdir3/subsubdir1/', 1, 'kb', 1, + 'file', 'fake_disk_url/1KB_4files_3subdir/subdir3/subsubdir1/', True), + call('fake_bucket_url/2KB_3files_1subdir/', 3, 'kb', 2, + 'file', 'fake_disk_url/2KB_3files_1subdir/', True), + call('fake_bucket_url/1KB_1files_0subdir/', 1, 'kb', 1, + 'file', 'fake_disk_url/1KB_1files_0subdir/', True) + ]) + + @patch('listing_benchmark.subprocess.call', return_value=0) + @patch('listing_benchmark.generate_files.generate_files_and_upload_to_gcs_bucket', return_value=1) + def test_create_directory_structure_error_multi_level_dir(self, mock_generate_files, mock_subprocess_call): + exit_code = listing_benchmark._create_directory_structure( + 'fake_bucket_url/', 'fake_disk_url/', DIRECTORY_STRUCTURE3, True) + self.assertGreater(exit_code, 0) + self.assertEqual(mock_subprocess_call.call_count, 4) + self.assertEqual(mock_generate_files.call_count, 3) + self.assertEqual(mock_subprocess_call.call_args_list, [ + call('mkdir fake_disk_url/', shell=True), + call('mkdir fake_disk_url/1KB_4files_3subdir/', shell=True), + call('mkdir fake_disk_url/2KB_3files_1subdir/', shell=True), + call('mkdir fake_disk_url/1KB_1files_0subdir/', shell=True) + ]) + self.assertEqual(mock_generate_files.call_args_list, [ + call('fake_bucket_url/1KB_4files_3subdir/', 4, 'kb', 1, + 'file', 'fake_disk_url/1KB_4files_3subdir/', True), + call('fake_bucket_url/2KB_3files_1subdir/', 3, 'kb', 2, + 'file', 'fake_disk_url/2KB_3files_1subdir/', True), + call('fake_bucket_url/1KB_1files_0subdir/', 1, 'kb', 1, + 'file', 'fake_disk_url/1KB_1files_0subdir/', True) + ]) + + @patch('listing_benchmark._list_directory') + def test_compare_directory_structure_true_single_level_dir(self, mock_list): + mock_list.side_effect = [['fake_bucket/']] + result = listing_benchmark._compare_directory_structure('fake_bucket/', DIRECTORY_STRUCTURE1) + self.assertTrue(result) + + @patch('listing_benchmark._list_directory') + def test_compare_directory_structure_true_double_level_dir(self, mock_list): + mock_list.side_effect = [ + ['fake_bucket/', 'fake_bucket/file', 'fake_bucket/2KB_3files_0subdir/', + 'fake_bucket/1KB_2files_0subdir/', 'fake_bucket/1KB_0files_0subdir/'], + ['fake_bucket/2KB_3files_0subdir/file_1', 'fake_bucket/2KB_3files_0subdir/file_2', + 'fake_bucket/2KB_3files_0subdir/file_3'], + ['fake_bucket/1KB_2files_0subdir/file_1', + 'fake_bucket/1KB_2files_0subdir/file_2'], + [] + ] + result = listing_benchmark._compare_directory_structure( + 'fake_bucket/', DIRECTORY_STRUCTURE2) + self.assertTrue(result) + + @patch('listing_benchmark._list_directory') + def test_compare_directory_structure_false_file_double_level_dir_test1(self, mock_list): + mock_list.side_effect = [ + ['fake_bucket/', 'fake_bucket/file', 'fake_bucket/2KB_3files_0subdir/', + 'fake_bucket/1KB_2files_0subdir/', 'fake_bucket/1KB_0files_0subdir/'], + ['fake_bucket/2KB_3files_0subdir/file_1', 'fake_bucket/2KB_3files_0subdir/file_2', + 'fake_bucket/2KB_3files_0subdir/file_3'], + ['fake_bucket/1KB_2files_0subdir/file_1', + 'fake_bucket/1KB_2files_0subdir/file_2'], + ['fake_bucket/1KB_0files_0subdir/file_1'] + ] + result = listing_benchmark._compare_directory_structure( + 'fake_bucket/', DIRECTORY_STRUCTURE2) + self.assertFalse(result) + + @patch('listing_benchmark._list_directory') + def test_compare_directory_structure_false_folder_double_level_dir(self, mock_list): + mock_list.side_effect = [ + ['fake_bucket/', 'fake_bucket/file', 'fake_bucket/2KB_3files_0subdir/', + 'fake_bucket/1KB_2files_0subdir/', 'fake_bucket/1KB_0files_0subdir/'], + ['fake_bucket/2KB_3files_0subdir/dummy_folder/', 'fake_bucket/2KB_3files_0subdir/file_1', + 'fake_bucket/2KB_3files_0subdir/file_2', 'fake_bucket/2KB_3files_0subdir/file_3'], + ['fake_bucket/1KB_2files_0subdir/file_1', + 'fake_bucket/1KB_2files_0subdir/file_2'], + [] + ] + result = listing_benchmark._compare_directory_structure( + 'fake_bucket/', DIRECTORY_STRUCTURE2) + self.assertFalse(result) + + @patch('listing_benchmark._list_directory') + def test_compare_directory_structure_false_file_double_level_dir_test2(self, mock_list): + mock_list.side_effect = [ + ['fake_bucket/', 'fake_bucket/2KB_3files_0subdir/', + 'fake_bucket/1KB_2files_0subdir/', 'fake_bucket/1KB_0files_0subdir/'], + ['fake_bucket/2KB_3files_0subdir/file_1', 'fake_bucket/2KB_3files_0subdir/file_2', + 'fake_bucket/2KB_3files_0subdir/file_3'], + ['fake_bucket/1KB_2files_0subdir/file_1', + 'fake_bucket/1KB_2files_0subdir/file_2'], + [] + ] + result = listing_benchmark._compare_directory_structure( + 'fake_bucket/', DIRECTORY_STRUCTURE2) + self.assertFalse(result) + + @patch('listing_benchmark._list_directory') + def test_compare_directory_structure_true_multi_level_dir(self, mock_list): + mock_list.side_effect = [ + ['fake_bucket/', 'fake_bucket/1KB_4files_3subdir/', + 'fake_bucket/2KB_3files_1subdir/', 'fake_bucket/1KB_1files_0subdir/'], + ['fake_bucket/1KB_4files_3subdir/file_1', 'fake_bucket/1KB_4files_3subdir/file_2', + 'fake_bucket/1KB_4files_3subdir/file_3', 'fake_bucket/1KB_4files_3subdir/file_4', + 'fake_bucket/1KB_4files_3subdir/subdir1/', 'fake_bucket/1KB_4files_3subdir/subdir2/', + 'fake_bucket/1KB_4files_3subdir/subdir3/'], + ['fake_bucket/1KB_4files_3subdir/subdir1/file_1', + 'fake_bucket/1KB_4files_3subdir/subdir1/subsubdir1/', + 'fake_bucket/1KB_4files_3subdir/subdir1/subsubdir2/'], + ['fake_bucket/1KB_4files_3subdir/subdir1/subsubdir1/file_1', + 'fake_bucket/1KB_4files_3subdir/subdir1/subsubdir1/file_2'], + [], + ['fake_bucket/1KB_4files_3subdir/subdir2/file_1'], + ['fake_bucket/1KB_4files_3subdir/subdir3/file_1', + 'fake_bucket/1KB_4files_3subdir/subdir3/subsubdir1/'], + ['fake_bucket/1KB_4files_3subdir/subdir3/subsubdir1/file_1'], + ['fake_bucket/2KB_3files_1subdir/file_1', 'fake_bucket/2KB_3files_1subdir/file_2', + 'fake_bucket/2KB_3files_1subdir/file_3', 'fake_bucket/2KB_3files_1subdir/subdir1/'], + [], + ['fake_bucket/1KB_1files_0subdir/file_1'] + ] + result = listing_benchmark._compare_directory_structure( + 'fake_bucket/', DIRECTORY_STRUCTURE3) + self.assertTrue(result) + + @patch('listing_benchmark._list_directory') + def test_compare_directory_structure_false_file_folder_multi_level_dir(self, mock_list): + mock_list.side_effect = [ + ['fake_bucket/', 'fake_bucket/file1', 'fake_bucket/1KB_4files_3subdir/', + 'fake_bucket/2KB_3files_1subdir/', 'fake_bucket/1KB_1files_0subdir/'], + ['fake_bucket/1KB_4files_3subdir/file_1', 'fake_bucket/1KB_4files_3subdir/file_2', + 'fake_bucket/1KB_4files_3subdir/file_3', 'fake_bucket/1KB_4files_3subdir/file_4', + 'fake_bucket/1KB_4files_3subdir/subdir1/', 'fake_bucket/1KB_4files_3subdir/subdir2/', + 'fake_bucket/1KB_4files_3subdir/subdir3/'], + ['fake_bucket/1KB_4files_3subdir/subdir1/file_1', + 'fake_bucket/1KB_4files_3subdir/subdir1/subsubdir1/', + 'fake_bucket/1KB_4files_3subdir/subdir1/subsubdir2/'], + ['fake_bucket/1KB_4files_3subdir/subdir1/subsubdir1/file_1', + 'fake_bucket/1KB_4files_3subdir/subdir1/subsubdir1/file_2'], + [], + ['fake_bucket/1KB_4files_3subdir/subdir2/file_1'], + ['fake_bucket/1KB_4files_3subdir/subdir3/file_1', + 'fake_bucket/1KB_4files_3subdir/subdir3/subsubdir1/'], + ['fake_bucket/1KB_4files_3subdir/subdir3/subsubdir1/file_1'], + ['fake_bucket/2KB_3files_1subdir/file_1', 'fake_bucket/2KB_3files_1subdir/file_2', + 'fake_bucket/2KB_3files_1subdir/file_3', 'fake_bucket/2KB_3files_1subdir/subdir1/'], + ['fake_bucket/2KB_3files_1subdir/subdir1/file_1', + 'fake_bucket/2KB_3files_1subdir/subdir1/dummy_folder/'], + ['fake_bucket/1KB_1files_0subdir/file_1'] + ] + result = listing_benchmark._compare_directory_structure( + 'fake_bucket/', DIRECTORY_STRUCTURE3) + self.assertFalse(result) + + @patch('listing_benchmark.subprocess.call', return_value=0) + def test_unmount_gcs_bucket(self, mock_subprocess_call): + listing_benchmark._unmount_gcs_bucket('fake_bucket') + self.assertEqual(mock_subprocess_call.call_count, 2) + self.assertEqual(mock_subprocess_call.call_args_list[0], call( + 'umount -l fake_bucket', shell=True)) + self.assertEqual(mock_subprocess_call.call_args_list[1], call( + 'rm -rf fake_bucket', shell=True)) + + @patch('listing_benchmark.subprocess.call', return_value=1) + def test_unmount_gcs_bucket_error(self, mock_subprocess_call): + listing_benchmark._unmount_gcs_bucket('fake_bucket') + self.assertEqual(mock_subprocess_call.call_count, 2) + self.assertEqual(mock_subprocess_call.call_args_list[0], call( + 'umount -l fake_bucket', shell=True)) + self.assertEqual( + mock_subprocess_call.call_args_list[1], call('bash', shell=True)) + + @patch('listing_benchmark.subprocess.call', return_value=0) + def test_mount_gcs_bucket(self, mock_subprocess_call): + directory_name = listing_benchmark._mount_gcs_bucket('fake_bucket') + self.assertEqual(directory_name, 'fake_bucket') + self.assertEqual(mock_subprocess_call.call_count, 2) + self.assertEqual(mock_subprocess_call.call_args_list, [ + call('mkdir fake_bucket', shell=True), + call('gcsfuse --implicit-dirs --disable-http2 --max-conns-per-host 100 fake_bucket fake_bucket', shell=True) + ]) + + @patch('listing_benchmark.subprocess.call', return_value=1) + def test_mount_gcs_bucket_error(self, mock_subprocess_call): + listing_benchmark._mount_gcs_bucket('fake_bucket') + self.assertEqual(mock_subprocess_call.call_count, 3) + self.assertEqual(mock_subprocess_call.call_args_list, [ + call('mkdir fake_bucket', shell=True), + call('gcsfuse --implicit-dirs --disable-http2 --max-conns-per-host 100 fake_bucket fake_bucket', shell=True), + call('bash', shell=True) + ]) + + +if __name__ == '__main__': + unittest.main()