#!/usr/bin/env python # Copyright 2017 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import contextlib import os.path import random import shutil import sys import tempfile import threading import unittest import gtest_parallel from gtest_parallel_mocks import LoggerMock from gtest_parallel_mocks import SubprocessMock from gtest_parallel_mocks import TestTimesMock from gtest_parallel_mocks import TestResultsMock from gtest_parallel_mocks import TaskManagerMock from gtest_parallel_mocks import TaskMockFactory from gtest_parallel_mocks import TaskMock @contextlib.contextmanager def guard_temp_dir(): try: temp_dir = tempfile.mkdtemp() yield temp_dir finally: shutil.rmtree(temp_dir) @contextlib.contextmanager def guard_temp_subdir(temp_dir, *path): assert path, 'Path should not be empty' try: temp_subdir = os.path.join(temp_dir, *path) os.makedirs(temp_subdir) yield temp_subdir finally: shutil.rmtree(os.path.join(temp_dir, path[0])) @contextlib.contextmanager def guard_patch_module(import_name, new_val): def patch(module, names, val): if len(names) == 1: old = getattr(module, names[0]) setattr(module, names[0], val) return old else: return patch(getattr(module, names[0]), names[1:], val) try: old_val = patch(gtest_parallel, import_name.split('.'), new_val) yield old_val finally: patch(gtest_parallel, import_name.split('.'), old_val) class TestTaskManager(unittest.TestCase): def setUp(self): self.passing_task = (('fake_binary', 'Fake.PassingTest'), { 'runtime_ms': [10], 'exit_code': [0], 'last_execution_time': [10], }) self.failing_task = (('fake_binary', 'Fake.FailingTest'), { 'runtime_ms': [20, 30, 40], 'exit_code': [1, 1, 1], 'last_execution_time': [None, None, None], }) self.fails_once_then_succeeds = (('another_binary', 'Fake.Test.FailOnce'), { 'runtime_ms': [21, 22], 'exit_code': [1, 0], 'last_execution_time': [None, 22], }) self.fails_twice_then_succeeds = (('yet_another_binary', 'Fake.Test.FailTwice'), { 'runtime_ms': [23, 25, 24], 'exit_code': [1, 1, 0], 'last_execution_time': [None, None, 24], }) def execute_tasks(self, tasks, retries, expected_exit_code): repeat = 1 times = TestTimesMock(self) logger = LoggerMock(self) test_results = TestResultsMock(self) task_mock_factory = TaskMockFactory(dict(tasks)) task_manager = gtest_parallel.TaskManager(times, logger, test_results, task_mock_factory, retries, repeat) for test_id, expected in tasks: task = task_mock_factory.get_task(test_id) task_manager.run_task(task) expected['execution_number'] = list(range(len(expected['exit_code']))) logger.assertRecorded(test_id, expected, retries + 1) times.assertRecorded(test_id, expected, retries + 1) test_results.assertRecorded(test_id, expected, retries + 1) self.assertEqual(len(task_manager.started), 0) self.assertListEqual( sorted(task.task_id for task in task_manager.passed), sorted(task.task_id for task in task_mock_factory.passed)) self.assertListEqual( sorted(task.task_id for task in task_manager.failed), sorted(task.task_id for task in task_mock_factory.failed)) self.assertEqual(task_manager.global_exit_code, expected_exit_code) def test_passing_task_succeeds(self): self.execute_tasks(tasks=[self.passing_task], retries=0, expected_exit_code=0) def test_failing_task_fails(self): self.execute_tasks(tasks=[self.failing_task], retries=0, expected_exit_code=1) def test_failing_task_fails_even_with_retries(self): self.execute_tasks(tasks=[self.failing_task], retries=2, expected_exit_code=1) def test_executing_passing_and_failing_fails(self): # Executing both a faling test and a passing one should make gtest-parallel # fail, no matter if the failing task is run first or last. self.execute_tasks(tasks=[self.failing_task, self.passing_task], retries=2, expected_exit_code=1) self.execute_tasks(tasks=[self.passing_task, self.failing_task], retries=2, expected_exit_code=1) def test_task_succeeds_with_one_retry(self): # Executes test and retries once. The first run should fail and the second # succeed, so gtest-parallel should succeed. self.execute_tasks(tasks=[self.fails_once_then_succeeds], retries=1, expected_exit_code=0) def test_task_fails_with_one_retry(self): # Executes test and retries once, not enough for the test to start passing, # so gtest-parallel should return an error. self.execute_tasks(tasks=[self.fails_twice_then_succeeds], retries=1, expected_exit_code=1) def test_runner_succeeds_when_all_tasks_eventually_succeeds(self): # Executes the test and retries twice. One test should pass in the first # attempt, another should take two runs, and the last one should take three # runs. All tests should succeed, so gtest-parallel should succeed too. self.execute_tasks(tasks=[ self.passing_task, self.fails_once_then_succeeds, self.fails_twice_then_succeeds ], retries=2, expected_exit_code=0) class TestSaveFilePath(unittest.TestCase): class StreamMock(object): def write(*args): # Suppress any output. pass def test_get_save_file_path_unix(self): with guard_temp_dir() as temp_dir, \ guard_patch_module('os.path.expanduser', lambda p: temp_dir), \ guard_patch_module('sys.stderr', TestSaveFilePath.StreamMock()), \ guard_patch_module('sys.platform', 'darwin'): with guard_patch_module('os.environ', {}), \ guard_temp_subdir(temp_dir, '.cache'): self.assertEqual(os.path.join(temp_dir, '.cache', 'gtest-parallel'), gtest_parallel.get_save_file_path()) with guard_patch_module('os.environ', {'XDG_CACHE_HOME': temp_dir}): self.assertEqual(os.path.join(temp_dir, 'gtest-parallel'), gtest_parallel.get_save_file_path()) with guard_patch_module('os.environ', {'XDG_CACHE_HOME': os.path.realpath(__file__)}): self.assertEqual(os.path.join(temp_dir, '.gtest-parallel-times'), gtest_parallel.get_save_file_path()) def test_get_save_file_path_win32(self): with guard_temp_dir() as temp_dir, \ guard_patch_module('os.path.expanduser', lambda p: temp_dir), \ guard_patch_module('sys.stderr', TestSaveFilePath.StreamMock()), \ guard_patch_module('sys.platform', 'win32'): with guard_patch_module('os.environ', {}), \ guard_temp_subdir(temp_dir, 'AppData', 'Local'): self.assertEqual( os.path.join(temp_dir, 'AppData', 'Local', 'gtest-parallel'), gtest_parallel.get_save_file_path()) with guard_patch_module('os.environ', {'LOCALAPPDATA': temp_dir}): self.assertEqual(os.path.join(temp_dir, 'gtest-parallel'), gtest_parallel.get_save_file_path()) with guard_patch_module('os.environ', {'LOCALAPPDATA': os.path.realpath(__file__)}): self.assertEqual(os.path.join(temp_dir, '.gtest-parallel-times'), gtest_parallel.get_save_file_path()) class TestSerializeTestCases(unittest.TestCase): def _execute_tasks(self, max_number_of_test_cases, max_number_of_tests_per_test_case, max_number_of_repeats, max_number_of_workers, serialize_test_cases): tasks = [] for test_case in range(max_number_of_test_cases): for test_name in range(max_number_of_tests_per_test_case): # All arguments for gtest_parallel.Task except for test_name are fake. test_name = 'TestCase{}.test{}'.format(test_case, test_name) for execution_number in range(random.randint(1, max_number_of_repeats)): tasks.append( gtest_parallel.Task('path/to/binary', test_name, ['path/to/binary', '--gtest_filter=*'], execution_number + 1, None, 'path/to/output')) expected_tasks_number = len(tasks) task_manager = TaskManagerMock() gtest_parallel.execute_tasks(tasks, max_number_of_workers, task_manager, None, serialize_test_cases) self.assertEqual(serialize_test_cases, not task_manager.had_running_parallel_groups) self.assertEqual(expected_tasks_number, task_manager.total_tasks_run) def test_running_parallel_test_cases_without_repeats(self): self._execute_tasks(max_number_of_test_cases=4, max_number_of_tests_per_test_case=32, max_number_of_repeats=1, max_number_of_workers=16, serialize_test_cases=True) def test_running_parallel_test_cases_with_repeats(self): self._execute_tasks(max_number_of_test_cases=4, max_number_of_tests_per_test_case=32, max_number_of_repeats=4, max_number_of_workers=16, serialize_test_cases=True) def test_running_parallel_tests(self): self._execute_tasks(max_number_of_test_cases=4, max_number_of_tests_per_test_case=128, max_number_of_repeats=1, max_number_of_workers=16, serialize_test_cases=False) class TestTestTimes(unittest.TestCase): def test_race_in_test_times_load_save(self): max_number_of_workers = 8 max_number_of_read_write_cycles = 64 test_times_file_name = 'test_times.pickle' def start_worker(save_file): def test_times_worker(): thread_id = threading.current_thread().ident path_to_binary = 'path/to/binary' + hex(thread_id) for cnt in range(max_number_of_read_write_cycles): times = gtest_parallel.TestTimes(save_file) threads_test_times = [ binary for (binary, _) in times._TestTimes__times.keys() if binary.startswith(path_to_binary) ] self.assertEqual(cnt, len(threads_test_times)) times.record_test_time('{}-{}'.format(path_to_binary, cnt), 'TestFoo.testBar', 1000) times.write_to_file(save_file) self.assertEqual( 1000, times.get_test_time('{}-{}'.format(path_to_binary, cnt), 'TestFoo.testBar')) self.assertIsNone( times.get_test_time('{}-{}'.format(path_to_binary, cnt), 'baz')) t = threading.Thread(target=test_times_worker) t.start() return t with guard_temp_dir() as temp_dir: try: workers = [ start_worker(os.path.join(temp_dir, test_times_file_name)) for _ in range(max_number_of_workers) ] finally: for worker in workers: worker.join() class TestTimeoutTestCases(unittest.TestCase): def test_task_timeout(self): timeout = 1 task = gtest_parallel.Task('test_binary', 'test_name', ['test_command'], 1, None, 'output_dir') tasks = [task] task_manager = TaskManagerMock() gtest_parallel.execute_tasks(tasks, 1, task_manager, timeout, True) self.assertEqual(1, task_manager.total_tasks_run) self.assertEqual(None, task.exit_code) self.assertEqual(1000, task.runtime_ms) class TestTask(unittest.TestCase): def test_log_file_names(self): def root(): return 'C:\\' if sys.platform == 'win32' else '/' self.assertEqual(os.path.join('.', 'bin-Test_case-100.log'), gtest_parallel.Task._logname('.', 'bin', 'Test.case', 100)) self.assertEqual( os.path.join('..', 'a', 'b', 'bin-Test_case_2-1.log'), gtest_parallel.Task._logname(os.path.join('..', 'a', 'b'), os.path.join('..', 'bin'), 'Test.case/2', 1)) self.assertEqual( os.path.join('..', 'a', 'b', 'bin-Test_case_2-5.log'), gtest_parallel.Task._logname(os.path.join('..', 'a', 'b'), os.path.join(root(), 'c', 'd', 'bin'), 'Test.case/2', 5)) self.assertEqual( os.path.join(root(), 'a', 'b', 'bin-Instantiation_Test_case_2-3.log'), gtest_parallel.Task._logname(os.path.join(root(), 'a', 'b'), os.path.join('..', 'c', 'bin'), 'Instantiation/Test.case/2', 3)) self.assertEqual( os.path.join(root(), 'a', 'b', 'bin-Test_case-1.log'), gtest_parallel.Task._logname(os.path.join(root(), 'a', 'b'), os.path.join(root(), 'c', 'd', 'bin'), 'Test.case', 1)) def test_logs_to_temporary_files_without_output_dir(self): log_file = gtest_parallel.Task._logname(None, None, None, None) self.assertEqual(tempfile.gettempdir(), os.path.dirname(log_file)) os.remove(log_file) def _execute_run_test(self, run_test_body, interrupt_test): def popen_mock(*_args, **_kwargs): return None class SigHandlerMock(object): class ProcessWasInterrupted(Exception): pass def wait(*_args): if interrupt_test: raise SigHandlerMock.ProcessWasInterrupted() return 42 with guard_temp_dir() as temp_dir, \ guard_patch_module('subprocess.Popen', popen_mock), \ guard_patch_module('sigint_handler', SigHandlerMock()), \ guard_patch_module('thread.exit', lambda: None): run_test_body(temp_dir) def test_run_normal_task(self): def run_test(temp_dir): task = gtest_parallel.Task('fake/binary', 'test', ['fake/binary'], 1, None, temp_dir) self.assertFalse(os.path.isfile(task.log_file)) task.run() self.assertTrue(os.path.isfile(task.log_file)) self.assertEqual(42, task.exit_code) self._execute_run_test(run_test, False) def test_run_interrupted_task_with_transient_log(self): def run_test(_): task = gtest_parallel.Task('fake/binary', 'test', ['fake/binary'], 1, None, None) self.assertTrue(os.path.isfile(task.log_file)) task.run() self.assertTrue(os.path.isfile(task.log_file)) self.assertIsNone(task.exit_code) self._execute_run_test(run_test, True) class TestFilterFormat(unittest.TestCase): def _execute_test(self, test_body, drop_output): class StdoutMock(object): def isatty(*_args): return False def write(*args): pass def flush(*args): pass with guard_temp_dir() as temp_dir, \ guard_patch_module('sys.stdout', StdoutMock()): logger = gtest_parallel.FilterFormat(None if drop_output else temp_dir) logger.log_tasks(42) test_body(logger) logger.flush() def test_no_output_dir(self): def run_test(logger): passed = [ TaskMock( ('fake/binary', 'FakeTest'), 0, { 'runtime_ms': [10], 'exit_code': [0], 'last_execution_time': [10], 'log_file': [os.path.join(tempfile.gettempdir(), 'fake.log')] }) ] open(passed[0].log_file, 'w').close() self.assertTrue(os.path.isfile(passed[0].log_file)) logger.log_exit(passed[0]) self.assertFalse(os.path.isfile(passed[0].log_file)) logger.print_tests('', passed, True) logger.move_to(None, passed) logger.summarize(passed, [], []) self._execute_test(run_test, True) def test_with_output_dir(self): def run_test(logger): failed = [ TaskMock( ('fake/binary', 'FakeTest'), 0, { 'runtime_ms': [10], 'exit_code': [1], 'last_execution_time': [10], 'log_file': [os.path.join(logger.output_dir, 'fake.log')] }) ] open(failed[0].log_file, 'w').close() self.assertTrue(os.path.isfile(failed[0].log_file)) logger.log_exit(failed[0]) self.assertTrue(os.path.isfile(failed[0].log_file)) logger.print_tests('', failed, True) logger.move_to('failed', failed) self.assertFalse(os.path.isfile(failed[0].log_file)) self.assertTrue( os.path.isfile(os.path.join(logger.output_dir, 'failed', 'fake.log'))) logger.summarize([], failed, []) self._execute_test(run_test, False) class TestFindTests(unittest.TestCase): ONE_DISABLED_ONE_ENABLED_TEST = { "fake_unittests": { "FakeTest": { "Test1": None, "DISABLED_Test2": None, } } } ONE_FAILED_ONE_PASSED_TEST = { "fake_unittests": { "FakeTest": { # Failed (and new) tests have no recorded runtime. "FailedTest": None, "Test": 1, } } } ONE_TEST = { "fake_unittests": { "FakeTest": { "TestSomething": None, } } } MULTIPLE_BINARIES_MULTIPLE_TESTS_ONE_FAILURE = { "fake_unittests": { "FakeTest": { "TestSomething": None, "TestSomethingElse": 2, }, "SomeOtherTest": { "YetAnotherTest": 3, }, }, "fake_tests": { "Foo": { "Bar": 4, "Baz": 4, } } } def _process_options(self, options): parser = gtest_parallel.default_options_parser() options, binaries = parser.parse_args(options) self.assertEqual(len(binaries), 0) return options def _call_find_tests(self, test_data, options=None): subprocess_mock = SubprocessMock(test_data) options = self._process_options(options or []) with guard_patch_module('subprocess.check_output', subprocess_mock): tasks = gtest_parallel.find_tests(test_data.keys(), [], options, TestTimesMock(self, test_data)) # Clean transient tasks' log files created because # by default now output_dir is None. for task in tasks: if os.path.isfile(task.log_file): os.remove(task.log_file) return tasks, subprocess_mock def test_tasks_are_sorted(self): tasks, _ = self._call_find_tests( self.MULTIPLE_BINARIES_MULTIPLE_TESTS_ONE_FAILURE) self.assertEqual([task.last_execution_time for task in tasks], [None, 4, 4, 3, 2]) def test_does_not_run_disabled_tests_by_default(self): tasks, subprocess_mock = self._call_find_tests( self.ONE_DISABLED_ONE_ENABLED_TEST) self.assertEqual(len(tasks), 1) self.assertFalse("DISABLED_" in tasks[0].test_name) self.assertNotIn("--gtest_also_run_disabled_tests", subprocess_mock.last_invocation) def test_runs_disabled_tests_when_asked(self): tasks, subprocess_mock = self._call_find_tests( self.ONE_DISABLED_ONE_ENABLED_TEST, ['--gtest_also_run_disabled_tests']) self.assertEqual(len(tasks), 2) self.assertEqual(sorted([task.test_name for task in tasks]), ["FakeTest.DISABLED_Test2", "FakeTest.Test1"]) self.assertIn("--gtest_also_run_disabled_tests", subprocess_mock.last_invocation) def test_runs_failed_tests_by_default(self): tasks, _ = self._call_find_tests(self.ONE_FAILED_ONE_PASSED_TEST) self.assertEqual(len(tasks), 2) self.assertEqual(sorted([task.test_name for task in tasks]), ["FakeTest.FailedTest", "FakeTest.Test"]) self.assertEqual({task.last_execution_time for task in tasks}, {None, 1}) def test_runs_only_failed_tests_when_asked(self): tasks, _ = self._call_find_tests(self.ONE_FAILED_ONE_PASSED_TEST, ['--failed']) self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].test_binary, "fake_unittests") self.assertEqual(tasks[0].test_name, "FakeTest.FailedTest") self.assertIsNone(tasks[0].last_execution_time) def test_does_not_apply_gtest_filter_by_default(self): _, subprocess_mock = self._call_find_tests(self.ONE_TEST) self.assertFalse( any( arg.startswith('--gtest_filter=SomeFilter') for arg in subprocess_mock.last_invocation)) def test_applies_gtest_filter(self): _, subprocess_mock = self._call_find_tests(self.ONE_TEST, ['--gtest_filter=SomeFilter']) self.assertIn('--gtest_filter=SomeFilter', subprocess_mock.last_invocation) def test_applies_gtest_color_by_default(self): tasks, _ = self._call_find_tests(self.ONE_TEST) self.assertEqual(len(tasks), 1) self.assertIn('--gtest_color=yes', tasks[0].test_command) def test_applies_gtest_color(self): tasks, _ = self._call_find_tests(self.ONE_TEST, ['--gtest_color=Lemur']) self.assertEqual(len(tasks), 1) self.assertIn('--gtest_color=Lemur', tasks[0].test_command) def test_repeats_tasks_once_by_default(self): tasks, _ = self._call_find_tests(self.ONE_TEST) self.assertEqual(len(tasks), 1) def test_repeats_tasks_multiple_times(self): tasks, _ = self._call_find_tests(self.ONE_TEST, ['--repeat=3']) self.assertEqual(len(tasks), 3) # Test all tasks have the same test_name, test_binary and test_command all_tasks_set = set( (task.test_name, task.test_binary, tuple(task.test_command)) for task in tasks) self.assertEqual(len(all_tasks_set), 1) # Test tasks have consecutive execution_numbers starting from 1 self.assertEqual(sorted(task.execution_number for task in tasks), [1, 2, 3]) def test_gtest_list_tests_fails(self): def exit_mock(*args): raise AssertionError('Foo') options = self._process_options([]) with guard_patch_module('sys.exit', exit_mock): self.assertRaises(AssertionError, gtest_parallel.find_tests, [sys.executable], [], options, None) if __name__ == '__main__': unittest.main()