Tools/glib/api_test_runner.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (C) 2011, 2012, 2017 Igalia S.L.
   4 #
   5 # This library is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU Library General Public
   7 # License as published by the Free Software Foundation; either
   8 # version 2 of the License, or (at your option) any later version.
   9 #
  10 # This library is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 # Library General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU Library General Public License
  16 # along with this library; see the file COPYING.LIB.  If not, write to
  17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  18 # Boston, MA 02110-1301, USA.
  19
  20 import subprocess
  21 import os
  22 import errno
  23 import sys
  24 import re
  25 from signal import alarm, signal, SIGALRM, SIGKILL, SIGSEGV
  26
  27 top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
  28 sys.path.insert(0, os.path.join(top_level_directory, "Tools", "glib"))
  29 import common
  30 from webkitpy.common.host import Host
  31
  32
  33 class SkippedTest:
  34     ENTIRE_SUITE = None
  35
  36     def __init__(self, test, test_case, reason, bug, build_type=None):
  37         self.test = test
  38         self.test_case = test_case
  39         self.reason = reason
  40         self.bug = bug
  41         self.build_type = build_type
  42
  43     def __str__(self):
  44         skipped_test_str = "%s" % self.test
  45
  46         if not(self.skip_entire_suite()):
  47             skipped_test_str += " [%s]" % self.test_case
  48
  49         skipped_test_str += ": %s (https://bugs.webkit.org/show_bug.cgi?id=%d)" % (self.reason, self.bug)
  50         return skipped_test_str
  51
  52     def skip_entire_suite(self):
  53         return self.test_case == SkippedTest.ENTIRE_SUITE
  54
  55     def skip_for_build_type(self, build_type):
  56         if self.build_type is None:
  57             return True
  58
  59         return self.build_type == build_type
  60
  61
  62 class TestTimeout(Exception):
  63     pass
  64
  65
  66 class TestRunner(object):
  67     TEST_DIRS = []
  68     SKIPPED = []
  69     SLOW = []
  70
  71     def __init__(self, port, options, tests=[]):
  72         self._options = options
  73
  74         self._build_type = "Debug" if self._options.debug else "Release"
  75         common.set_build_types((self._build_type,))
  76         self._port = Host().port_factory.get(port)
  77         self._driver = self._create_driver()
  78
  79         self._programs_path = common.binary_build_path()
  80         self._tests = self._get_tests(tests)
  81         self._skipped_tests = [skipped for skipped in TestRunner.SKIPPED if skipped.skip_for_build_type(self._build_type)]
  82         self._disabled_tests = []
  83
  84     def _test_programs_base_dir(self):
  85         return os.path.join(self._programs_path, "TestWebKitAPI")
  86
  87     def _get_tests_from_dir(self, test_dir):
  88         if not os.path.isdir(test_dir):
  89             return []
  90
  91         tests = []
  92         for test_file in os.listdir(test_dir):
  93             if not test_file.lower().startswith("test"):
  94                 continue
  95             test_path = os.path.join(test_dir, test_file)
  96             if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
  97                 tests.append(test_path)
  98         return tests
  99
 100     def _get_tests(self, initial_tests):
 101         tests = []
 102         for test in initial_tests:
 103             if os.path.isdir(test):
 104                 tests.extend(self._get_tests_from_dir(test))
 105             else:
 106                 tests.append(test)
 107         if tests:
 108             return tests
 109
 110         tests = []
 111         for test_dir in self.TEST_DIRS:
 112             absolute_test_dir = os.path.join(self._test_programs_base_dir(), test_dir)
 113             tests.extend(self._get_tests_from_dir(absolute_test_dir))
 114         return tests
 115
 116     def _create_driver(self, port_options=[]):
 117         self._port._display_server = self._options.display_server
 118         driver = self._port.create_driver(worker_number=0, no_timeout=True)._make_driver(pixel_tests=False)
 119         if not driver.check_driver(self._port):
 120             raise RuntimeError("Failed to check driver %s" % driver.__class__.__name__)
 121         return driver
 122
 123     def _setup_testing_environment(self):
 124         self._test_env = self._driver._setup_environ_for_test()
 125         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit")
 126         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.library_build_path()
 127         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
 128
 129         return True
 130
 131     def _tear_down_testing_environment(self):
 132         if self._driver:
 133             self._driver.stop()
 134
 135     def _test_cases_to_skip(self, test_program):
 136         if self._options.skipped_action != 'skip':
 137             return []
 138
 139         test_cases = []
 140         for skipped in self._skipped_tests:
 141             if test_program.endswith(skipped.test) and not skipped.skip_entire_suite():
 142                 test_cases.append(skipped.test_case)
 143         return test_cases
 144
 145     def _should_run_test_program(self, test_program):
 146         for disabled_test in self._disabled_tests:
 147             if test_program.endswith(disabled_test):
 148                 return False
 149
 150         if self._options.skipped_action != 'skip':
 151             return True
 152
 153         for skipped in self._skipped_tests:
 154             if test_program.endswith(skipped.test) and skipped.skip_entire_suite():
 155                 return False
 156         return True
 157
 158     def _kill_process(self, pid):
 159         try:
 160             os.kill(pid, SIGKILL)
 161         except OSError:
 162             # Process already died.
 163             pass
 164
 165     @staticmethod
 166     def _start_timeout(timeout):
 167         if timeout <= 0:
 168             return
 169
 170         def _alarm_handler(signum, frame):
 171             raise TestTimeout
 172
 173         signal(SIGALRM, _alarm_handler)
 174         alarm(timeout)
 175
 176     @staticmethod
 177     def _stop_timeout(timeout):
 178         if timeout <= 0:
 179             return
 180
 181         alarm(0)
 182
 183     def _waitpid(self, pid):
 184         while True:
 185             try:
 186                 dummy, status = os.waitpid(pid, 0)
 187                 if os.WIFSIGNALED(status):
 188                     return -os.WTERMSIG(status)
 189                 if os.WIFEXITED(status):
 190                     return os.WEXITSTATUS(status)
 191
 192                 # Should never happen
 193                 raise RuntimeError("Unknown child exit status!")
 194             except (OSError, IOError) as e:
 195                 if e.errno == errno.EINTR:
 196                     continue
 197                 if e.errno == errno.ECHILD:
 198                     # This happens if SIGCLD is set to be ignored or waiting
 199                     # for child processes has otherwise been disabled for our
 200                     # process.  This child is dead, we can't get the status.
 201                     return 0
 202                 raise
 203
 204     def _run_test_glib(self, test_program):
 205         command = ['gtester', '-k']
 206         if self._options.verbose:
 207             command.append('--verbose')
 208         for test_case in self._test_cases_to_skip(test_program):
 209             command.extend(['-s', test_case])
 210         command.append(test_program)
 211
 212         timeout = self._options.timeout
 213         test = os.path.join(os.path.basename(os.path.dirname(test_program)), os.path.basename(test_program))
 214         if test in TestRunner.SLOW:
 215             timeout *= 5
 216
 217         test_context = {"child-pid": -1, "did-timeout": False, "current_test": None}
 218
 219         def parse_line(line, test_context=test_context):
 220             if not line:
 221                 return
 222
 223             match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', line)
 224             if match:
 225                 test_context["child-pid"] = int(match.group('child_pid'))
 226                 sys.stdout.write(line)
 227                 return
 228
 229             def set_test_result(test, result):
 230                 if result == "FAIL":
 231                     if test_context["did-timeout"] and result == "FAIL":
 232                         test_context[test] = "TIMEOUT"
 233                     else:
 234                         test_context[test] = result
 235                 test_context["did-timeout"] = False
 236                 test_context["current_test"] = None
 237                 self._stop_timeout(timeout)
 238                 self._start_timeout(timeout)
 239
 240             normalized_line = line.strip().replace(' ', '')
 241             if not normalized_line:
 242                 return
 243
 244             if normalized_line[0] == '/':
 245                 test, result = normalized_line.split(':', 1)
 246                 if result in ["OK", "FAIL"]:
 247                     set_test_result(test, result)
 248                 else:
 249                     test_context["current_test"] = test
 250             elif normalized_line in ["OK", "FAIL"]:
 251                 set_test_result(test_context["current_test"], normalized_line)
 252
 253             sys.stdout.write(line)
 254
 255         pid, fd = os.forkpty()
 256         if pid == 0:
 257             os.execvpe(command[0], command, self._test_env)
 258             sys.exit(0)
 259
 260         self._start_timeout(timeout)
 261
 262         while (True):
 263             try:
 264                 common.parse_output_lines(fd, parse_line)
 265                 break
 266             except TestTimeout:
 267                 assert test_context["child-pid"] > 0
 268                 self._kill_process(test_context["child-pid"])
 269                 test_context["child-pid"] = -1
 270                 test_context["did-timeout"] = True
 271
 272         self._stop_timeout(timeout)
 273         del test_context["child-pid"]
 274         del test_context["did-timeout"]
 275         del test_context["current_test"]
 276
 277         self._waitpid(pid)
 278         return test_context
 279
 280     def _get_tests_from_google_test_suite(self, test_program):
 281         try:
 282             output = subprocess.check_output([test_program, '--gtest_list_tests'], env=self._test_env)
 283         except subprocess.CalledProcessError:
 284             sys.stderr.write("ERROR: could not list available tests for binary %s.\n" % (test_program))
 285             sys.stderr.flush()
 286             return 1
 287
 288         skipped_test_cases = self._test_cases_to_skip(test_program)
 289
 290         tests = []
 291         prefix = None
 292         for line in output.split('\n'):
 293             if not line.startswith('  '):
 294                 prefix = line
 295                 continue
 296             else:
 297                 test_name = prefix + line.strip()
 298                 if not test_name in skipped_test_cases:
 299                     tests.append(test_name)
 300         return tests
 301
 302     def _run_google_test(self, test_program, subtest):
 303         command = [test_program, '--gtest_filter=%s' % (subtest)]
 304         timeout = self._options.timeout
 305         if subtest in TestRunner.SLOW:
 306             timeout *= 5
 307
 308         pid, fd = os.forkpty()
 309         if pid == 0:
 310             os.execvpe(command[0], command, self._test_env)
 311             sys.exit(0)
 312
 313         self._start_timeout(timeout)
 314         try:
 315             common.parse_output_lines(fd, sys.stdout.write)
 316             status = self._waitpid(pid)
 317         except TestTimeout:
 318             self._kill_process(pid)
 319             return {subtest: "TIMEOUT"}
 320
 321         self._stop_timeout(timeout)
 322
 323         if status == -SIGSEGV:
 324             sys.stdout.write("**CRASH** %s\n" % subtest)
 325             sys.stdout.flush()
 326             return {subtest: "CRASH"}
 327
 328         if status != 0:
 329             return {subtest: "FAIL"}
 330
 331         return {}
 332
 333     def _run_google_test_suite(self, test_program):
 334         result = {}
 335         for subtest in self._get_tests_from_google_test_suite(test_program):
 336             result.update(self._run_google_test(test_program, subtest))
 337         return result
 338
 339     def is_glib_test(self, test_program):
 340         raise NotImplementedError
 341
 342     def is_google_test(self, test_program):
 343         raise NotImplementedError
 344
 345     def _run_test(self, test_program):
 346         if self.is_glib_test(test_program):
 347             return self._run_test_glib(test_program)
 348
 349         if self.is_google_test(test_program):
 350             return self._run_google_test_suite(test_program)
 351
 352         return {}
 353
 354     def run_tests(self):
 355         if not self._tests:
 356             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._test_programs_base_dir()))
 357             sys.stderr.flush()
 358             return 1
 359
 360         if not self._setup_testing_environment():
 361             return 1
 362
 363         # Remove skipped tests now instead of when we find them, because
 364         # some tests might be skipped while setting up the test environment.
 365         self._tests = [test for test in self._tests if self._should_run_test_program(test)]
 366
 367         crashed_tests = {}
 368         failed_tests = {}
 369         timed_out_tests = {}
 370         try:
 371             for test in self._tests:
 372                 results = self._run_test(test)
 373                 for test_case, result in results.iteritems():
 374                     if result == "FAIL":
 375                         failed_tests.setdefault(test, []).append(test_case)
 376                     elif result == "TIMEOUT":
 377                         timed_out_tests.setdefault(test, []).append(test_case)
 378                     elif result == "CRASH":
 379                         crashed_tests.setdefault(test, []).append(test_case)
 380         finally:
 381             self._tear_down_testing_environment()
 382
 383         if failed_tests:
 384             sys.stdout.write("\nUnexpected failures (%d)\n" % (sum(len(value) for value in failed_tests.itervalues())))
 385             for test in failed_tests:
 386                 sys.stdout.write("    %s\n" % (test.replace(self._test_programs_base_dir(), '', 1)))
 387                 for test_case in failed_tests[test]:
 388                     sys.stdout.write("        %s\n" % (test_case))
 389             sys.stdout.flush()
 390
 391         if crashed_tests:
 392             sys.stdout.write("\nUnexpected crashes (%d)\n" % (sum(len(value) for value in crashed_tests.itervalues())))
 393             for test in crashed_tests:
 394                 sys.stdout.write("    %s\n" % (test.replace(self._test_programs_base_dir(), '', 1)))
 395                 for test_case in crashed_tests[test]:
 396                     sys.stdout.write("        %s\n" % (test_case))
 397             sys.stdout.flush()
 398
 399         if timed_out_tests:
 400             sys.stdout.write("\nUnexpected timeouts (%d)\n" % (sum(len(value) for value in timed_out_tests.itervalues())))
 401             for test in timed_out_tests:
 402                 sys.stdout.write("    %s\n" % (test.replace(self._test_programs_base_dir(), '', 1)))
 403                 for test_case in timed_out_tests[test]:
 404                     sys.stdout.write("        %s\n" % (test_case))
 405             sys.stdout.flush()
 406
 407         return len(failed_tests) + len(timed_out_tests)
 408
 409
 410 def add_options(option_parser):
 411     option_parser.add_option('-r', '--release',
 412                              action='store_true', dest='release',
 413                              help='Run in Release')
 414     option_parser.add_option('-d', '--debug',
 415                              action='store_true', dest='debug',
 416                              help='Run in Debug')
 417     option_parser.add_option('-v', '--verbose',
 418                              action='store_true', dest='verbose',
 419                              help='Run gtester in verbose mode')
 420     option_parser.add_option('--skipped', action='store', dest='skipped_action',
 421                              choices=['skip', 'ignore', 'only'], default='skip',
 422                              metavar='skip|ignore|only',
 423                              help='Specifies how to treat the skipped tests')
 424     option_parser.add_option('-t', '--timeout',
 425                              action='store', type='int', dest='timeout', default=10,
 426                              help='Time in seconds until a test times out')