Tools/glib/api_test_runner.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (C) 2011, 2012, 2017 Igalia S.L.
   4 #
   5 # This library is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU Library General Public
   7 # License as published by the Free Software Foundation; either
   8 # version 2 of the License, or (at your option) any later version.
   9 #
  10 # This library is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 # Library General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU Library General Public License
  16 # along with this library; see the file COPYING.LIB.  If not, write to
  17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  18 # Boston, MA 02110-1301, USA.
  19
  20 import subprocess
  21 import os
  22 import errno
  23 import sys
  24 import re
  25 from signal import alarm, signal, SIGALRM, SIGKILL, SIGSEGV
  26
  27 top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
  28 sys.path.insert(0, os.path.join(top_level_directory, "Tools", "glib"))
  29 import common
  30 from webkitpy.common.host import Host
  31 from webkitpy.common.test_expectations import TestExpectations
  32
  33
  34 class TestTimeout(Exception):
  35     pass
  36
  37
  38 class TestRunner(object):
  39     TEST_DIRS = []
  40
  41     def __init__(self, port, options, tests=[]):
  42         self._options = options
  43
  44         self._build_type = "Debug" if self._options.debug else "Release"
  45         common.set_build_types((self._build_type,))
  46         self._port = Host().port_factory.get(port)
  47         self._driver = self._create_driver()
  48
  49         self._programs_path = common.binary_build_path()
  50         expectations_file = os.path.join(common.top_level_path(), "Tools", "TestWebKitAPI", "glib", "TestExpectations.json")
  51         self._expectations = TestExpectations(self._port.name(), expectations_file, self._build_type)
  52         self._tests = self._get_tests(tests)
  53         self._disabled_tests = []
  54
  55     def _test_programs_base_dir(self):
  56         return os.path.join(self._programs_path, "TestWebKitAPI")
  57
  58     def _get_tests_from_dir(self, test_dir):
  59         if not os.path.isdir(test_dir):
  60             return []
  61
  62         tests = []
  63         for test_file in os.listdir(test_dir):
  64             if not test_file.lower().startswith("test"):
  65                 continue
  66             test_path = os.path.join(test_dir, test_file)
  67             if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
  68                 tests.append(test_path)
  69         return tests
  70
  71     def _get_tests(self, initial_tests):
  72         tests = []
  73         for test in initial_tests:
  74             if os.path.isdir(test):
  75                 tests.extend(self._get_tests_from_dir(test))
  76             else:
  77                 tests.append(test)
  78         if tests:
  79             return tests
  80
  81         tests = []
  82         for test_dir in self.TEST_DIRS:
  83             absolute_test_dir = os.path.join(self._test_programs_base_dir(), test_dir)
  84             tests.extend(self._get_tests_from_dir(absolute_test_dir))
  85         return tests
  86
  87     def _create_driver(self, port_options=[]):
  88         self._port._display_server = self._options.display_server
  89         driver = self._port.create_driver(worker_number=0, no_timeout=True)._make_driver(pixel_tests=False)
  90         if not driver.check_driver(self._port):
  91             raise RuntimeError("Failed to check driver %s" % driver.__class__.__name__)
  92         return driver
  93
  94     def _setup_testing_environment(self):
  95         self._test_env = self._driver._setup_environ_for_test()
  96         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit")
  97         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.library_build_path()
  98         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
  99
 100         return True
 101
 102     def _tear_down_testing_environment(self):
 103         if self._driver:
 104             self._driver.stop()
 105
 106     def _test_cases_to_skip(self, test_program):
 107         if self._options.skipped_action != 'skip':
 108             return []
 109
 110         return self._expectations.skipped_subtests(os.path.basename(test_program))
 111
 112     def _should_run_test_program(self, test_program):
 113         for disabled_test in self._disabled_tests:
 114             if test_program.endswith(disabled_test):
 115                 return False
 116
 117         if self._options.skipped_action != 'skip':
 118             return True
 119
 120         return os.path.basename(test_program) not in self._expectations.skipped_tests()
 121
 122     def _kill_process(self, pid):
 123         try:
 124             os.kill(pid, SIGKILL)
 125         except OSError:
 126             # Process already died.
 127             pass
 128
 129     @staticmethod
 130     def _start_timeout(timeout):
 131         if timeout <= 0:
 132             return
 133
 134         def _alarm_handler(signum, frame):
 135             raise TestTimeout
 136
 137         signal(SIGALRM, _alarm_handler)
 138         alarm(timeout)
 139
 140     @staticmethod
 141     def _stop_timeout(timeout):
 142         if timeout <= 0:
 143             return
 144
 145         alarm(0)
 146
 147     def _waitpid(self, pid):
 148         while True:
 149             try:
 150                 dummy, status = os.waitpid(pid, 0)
 151                 if os.WIFSIGNALED(status):
 152                     return -os.WTERMSIG(status)
 153                 if os.WIFEXITED(status):
 154                     return os.WEXITSTATUS(status)
 155
 156                 # Should never happen
 157                 raise RuntimeError("Unknown child exit status!")
 158             except (OSError, IOError) as e:
 159                 if e.errno == errno.EINTR:
 160                     continue
 161                 if e.errno == errno.ECHILD:
 162                     # This happens if SIGCLD is set to be ignored or waiting
 163                     # for child processes has otherwise been disabled for our
 164                     # process.  This child is dead, we can't get the status.
 165                     return 0
 166                 raise
 167
 168     def _run_test_glib(self, test_program):
 169         command = ['gtester', '-k']
 170         if self._options.verbose:
 171             command.append('--verbose')
 172         for test_case in self._test_cases_to_skip(test_program):
 173             command.extend(['-s', test_case])
 174         command.append(test_program)
 175
 176         timeout = self._options.timeout
 177         test = os.path.join(os.path.basename(os.path.dirname(test_program)), os.path.basename(test_program))
 178         if self._expectations.is_slow(os.path.basename(test_program)):
 179             timeout *= 5
 180
 181         test_context = {"child-pid": -1, "did-timeout": False, "current_test": None}
 182
 183         def parse_line(line, test_context=test_context):
 184             if not line:
 185                 return
 186
 187             match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', line)
 188             if match:
 189                 test_context["child-pid"] = int(match.group('child_pid'))
 190                 sys.stdout.write(line)
 191                 return
 192
 193             def set_test_result(test, result):
 194                 if result == "FAIL":
 195                     if test_context["did-timeout"] and result == "FAIL":
 196                         test_context[test] = "TIMEOUT"
 197                     else:
 198                         test_context[test] = result
 199                 else:
 200                     test_context[test] = 'PASS'
 201                 test_context["did-timeout"] = False
 202                 test_context["current_test"] = None
 203                 self._stop_timeout(timeout)
 204                 self._start_timeout(timeout)
 205
 206             normalized_line = line.strip().replace(' ', '')
 207             if not normalized_line:
 208                 return
 209
 210             if normalized_line[0] == '/':
 211                 test, result = normalized_line.split(':', 1)
 212                 if result in ["OK", "FAIL"]:
 213                     set_test_result(test, result)
 214                 else:
 215                     test_context["current_test"] = test
 216             elif normalized_line in ["OK", "FAIL"]:
 217                 set_test_result(test_context["current_test"], normalized_line)
 218
 219             sys.stdout.write(line)
 220
 221         pid, fd = os.forkpty()
 222         if pid == 0:
 223             os.execvpe(command[0], command, self._test_env)
 224             sys.exit(0)
 225
 226         self._start_timeout(timeout)
 227
 228         while (True):
 229             try:
 230                 common.parse_output_lines(fd, parse_line)
 231                 break
 232             except TestTimeout:
 233                 assert test_context["child-pid"] > 0
 234                 self._kill_process(test_context["child-pid"])
 235                 test_context["child-pid"] = -1
 236                 test_context["did-timeout"] = True
 237
 238         self._stop_timeout(timeout)
 239         del test_context["child-pid"]
 240         del test_context["did-timeout"]
 241         del test_context["current_test"]
 242
 243         self._waitpid(pid)
 244         return test_context
 245
 246     def _get_tests_from_google_test_suite(self, test_program):
 247         try:
 248             output = subprocess.check_output([test_program, '--gtest_list_tests'], env=self._test_env)
 249         except subprocess.CalledProcessError:
 250             sys.stderr.write("ERROR: could not list available tests for binary %s.\n" % (test_program))
 251             sys.stderr.flush()
 252             return 1
 253
 254         skipped_test_cases = self._test_cases_to_skip(test_program)
 255
 256         tests = []
 257         prefix = None
 258         for line in output.split('\n'):
 259             if not line.startswith('  '):
 260                 prefix = line
 261                 continue
 262             else:
 263                 test_name = prefix + line.strip()
 264                 if not test_name in skipped_test_cases:
 265                     tests.append(test_name)
 266         return tests
 267
 268     def _run_google_test(self, test_program, subtest):
 269         command = [test_program, '--gtest_filter=%s' % (subtest)]
 270         timeout = self._options.timeout
 271         if self._expectations.is_slow(os.path.basename(test_program), subtest):
 272             timeout *= 5
 273
 274         pid, fd = os.forkpty()
 275         if pid == 0:
 276             os.execvpe(command[0], command, self._test_env)
 277             sys.exit(0)
 278
 279         self._start_timeout(timeout)
 280         try:
 281             common.parse_output_lines(fd, sys.stdout.write)
 282             status = self._waitpid(pid)
 283         except TestTimeout:
 284             self._kill_process(pid)
 285             return {subtest: "TIMEOUT"}
 286
 287         self._stop_timeout(timeout)
 288
 289         if status == -SIGSEGV:
 290             sys.stdout.write("**CRASH** %s\n" % subtest)
 291             sys.stdout.flush()
 292             return {subtest: "CRASH"}
 293
 294         if status != 0:
 295             return {subtest: "FAIL"}
 296
 297         return {subtest: "PASS"}
 298
 299     def _run_google_test_suite(self, test_program):
 300         result = {}
 301         for subtest in self._get_tests_from_google_test_suite(test_program):
 302             result.update(self._run_google_test(test_program, subtest))
 303         return result
 304
 305     def is_glib_test(self, test_program):
 306         raise NotImplementedError
 307
 308     def is_google_test(self, test_program):
 309         raise NotImplementedError
 310
 311     def _run_test(self, test_program):
 312         if self.is_glib_test(test_program):
 313             return self._run_test_glib(test_program)
 314
 315         if self.is_google_test(test_program):
 316             return self._run_google_test_suite(test_program)
 317
 318         return {}
 319
 320     def run_tests(self):
 321         if not self._tests:
 322             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._test_programs_base_dir()))
 323             sys.stderr.flush()
 324             return 1
 325
 326         if not self._setup_testing_environment():
 327             return 1
 328
 329         # Remove skipped tests now instead of when we find them, because
 330         # some tests might be skipped while setting up the test environment.
 331         self._tests = [test for test in self._tests if self._should_run_test_program(test)]
 332
 333         crashed_tests = {}
 334         failed_tests = {}
 335         timed_out_tests = {}
 336         passed_tests = {}
 337         try:
 338             for test in self._tests:
 339                 results = self._run_test(test)
 340                 for test_case, result in results.iteritems():
 341                     if result in self._expectations.get_expectation(os.path.basename(test), test_case):
 342                         continue
 343
 344                     if result == "FAIL":
 345                         failed_tests.setdefault(test, []).append(test_case)
 346                     elif result == "TIMEOUT":
 347                         timed_out_tests.setdefault(test, []).append(test_case)
 348                     elif result == "CRASH":
 349                         crashed_tests.setdefault(test, []).append(test_case)
 350                     elif result == "PASS":
 351                         passed_tests.setdefault(test, []).append(test_case)
 352         finally:
 353             self._tear_down_testing_environment()
 354
 355         def report(tests, title, base_dir):
 356             if not tests:
 357                 return
 358             sys.stdout.write("\nUnexpected %s (%d)\n" % (title, sum(len(value) for value in tests.itervalues())))
 359             for test in tests:
 360                 sys.stdout.write("    %s\n" % (test.replace(base_dir, '', 1)))
 361                 for test_case in tests[test]:
 362                     sys.stdout.write("        %s\n" % (test_case))
 363             sys.stdout.flush()
 364
 365         report(failed_tests, "failures", self._test_programs_base_dir())
 366         report(crashed_tests, "crashes", self._test_programs_base_dir())
 367         report(timed_out_tests, "timeouts", self._test_programs_base_dir())
 368         report(passed_tests, "passes", self._test_programs_base_dir())
 369
 370         return len(failed_tests) + len(timed_out_tests)
 371
 372
 373 def add_options(option_parser):
 374     option_parser.add_option('-r', '--release',
 375                              action='store_true', dest='release',
 376                              help='Run in Release')
 377     option_parser.add_option('-d', '--debug',
 378                              action='store_true', dest='debug',
 379                              help='Run in Debug')
 380     option_parser.add_option('-v', '--verbose',
 381                              action='store_true', dest='verbose',
 382                              help='Run gtester in verbose mode')
 383     option_parser.add_option('--skipped', action='store', dest='skipped_action',
 384                              choices=['skip', 'ignore', 'only'], default='skip',
 385                              metavar='skip|ignore|only',
 386                              help='Specifies how to treat the skipped tests')
 387     option_parser.add_option('-t', '--timeout',
 388                              action='store', type='int', dest='timeout', default=10,
 389                              help='Time in seconds until a test times out')