Tools/Scripts/run-gtk-tests

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (C) 2011, 2012 Igalia S.L.
   4 #
   5 # This library is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU Library General Public
   7 # License as published by the Free Software Foundation; either
   8 # version 2 of the License, or (at your option) any later version.
   9 #
  10 # This library is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 # Library General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU Library General Public License
  16 # along with this library; see the file COPYING.LIB.  If not, write to
  17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  18 # Boston, MA 02110-1301, USA.
  19
  20 import subprocess
  21 import os
  22 import sys
  23 import optparse
  24 import re
  25 from signal import alarm, signal, SIGALRM, SIGKILL, SIGSEGV
  26 from gi.repository import Gio, GLib
  27
  28 top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
  29 sys.path.append(os.path.join(top_level_directory, "Tools", "jhbuild"))
  30 sys.path.append(os.path.join(top_level_directory, "Tools", "gtk"))
  31 import common
  32 import jhbuildutils
  33
  34 class SkippedTest:
  35     ENTIRE_SUITE = None
  36
  37     def __init__(self, test, test_case, reason, bug=None):
  38         self.test = test
  39         self.test_case = test_case
  40         self.reason = reason
  41         self.bug = bug
  42
  43     def __str__(self):
  44         skipped_test_str = "%s" % self.test
  45
  46         if not(self.skip_entire_suite()):
  47             skipped_test_str += " [%s]" % self.test_case
  48
  49         skipped_test_str += ": %s " % self.reason
  50         if self.bug is not None:
  51             skipped_test_str += "(https://bugs.webkit.org/show_bug.cgi?id=%d)" % self.bug
  52         return skipped_test_str
  53
  54     def skip_entire_suite(self):
  55         return self.test_case == SkippedTest.ENTIRE_SUITE
  56
  57 class TestTimeout(Exception):
  58     pass
  59
  60 class TestRunner:
  61     TEST_DIRS = [ "WebKit2Gtk", "WebKit2", "JavaScriptCore", "WTF", "WebCore" ]
  62
  63     SKIPPED = [
  64         SkippedTest("WebKit2Gtk/TestUIClient", "/webkit2/WebKitWebView/mouse-target", "Test times out after r150890", 117689),
  65         SkippedTest("WebKit2Gtk/TestContextMenu", SkippedTest.ENTIRE_SUITE, "Test times out after r150890", 117689),
  66         SkippedTest("WebKit2APITests/TestWebKitAccessibility", "/webkit2/WebKitAccessibility/atspi-basic-hierarchy", "Test is flaky", 132134),
  67         SkippedTest("WebKit2Gtk/TestWebKitWebView", "/webkit2/WebKitWebView/snapshot", "Test fails", 120404),
  68         SkippedTest("WebKit2Gtk/TestWebKitWebView", "/webkit2/WebKitWebView/page-visibility", "Test fails or times out", 131731),
  69         SkippedTest("WebKit2Gtk/TestCookieManager", "/webkit2/WebKitCookieManager/persistent-storage", "Test is flaky", 134580),
  70         SkippedTest("WebKit2/TestWebKit2", "WebKit2.MouseMoveAfterCrash", "Test is flaky", 85066),
  71         SkippedTest("WebKit2/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutForImages", "Test is flaky", 85066),
  72         SkippedTest("WebKit2/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutFrames", "Test fails", 85037),
  73         SkippedTest("WebKit2/TestWebKit2", "WebKit2.RestoreSessionStateContainingFormData", "Session State is not implemented in GTK+ port", 84960),
  74         SkippedTest("WebKit2/TestWebKit2", "WebKit2.SpacebarScrolling", "Test fails", 84961),
  75         SkippedTest("WebKit2/TestWebKit2", "WebKit2.WKConnection", "Tests fail and time out out", 84959),
  76         SkippedTest("WebKit2/TestWebKit2", "WebKit2.WKPageGetScaleFactorNotZero", "Test fails and times out", 88455),
  77         SkippedTest("WebKit2/TestWebKit2", "WebKit2.ForceRepaint", "Test times out", 105532),
  78         SkippedTest("WebKit2/TestWebKit2", "WebKit2.ReloadPageAfterCrash", "Test flakily times out", 110129),
  79         SkippedTest("WebKit2/TestWebKit2", "WebKit2.DidAssociateFormControls", "Test times out", 120302),
  80         SkippedTest("WebKit2/TestWebKit2", "WebKit2.InjectedBundleFrameHitTest", "Test times out", 120303),
  81         SkippedTest("WebKit2/TestWebKit2", "WebKit2.TerminateTwice", "Test causes crash on the next test", 121970),
  82         SkippedTest("WebKit2/TestWebKit2", "WebKit2.GeolocationTransitionToHighAccuracy", "Test causes crash on the next test", 125068),
  83         SkippedTest("WebKit2/TestWebKit2", "WebKit2.GeolocationTransitionToLowAccuracy", "Test causes crash on the next test", 125068),
  84     ]
  85
  86     def __init__(self, options, tests=[]):
  87         self._options = options
  88
  89         self._build_type = "Debug" if self._options.debug else "Release"
  90         common.set_build_types((self._build_type,))
  91
  92         self._programs_path = common.binary_build_path()
  93         self._tests = self._get_tests(tests)
  94         self._skipped_tests = TestRunner.SKIPPED
  95         self._disabled_tests = []
  96         if not sys.stdout.isatty():
  97             self._tty_colors_pattern = re.compile("\033\[[0-9;]*m")
  98
  99         # These SPI daemons need to be active for the accessibility tests to work.
 100         self._spi_registryd = None
 101         self._spi_bus_launcher = None
 102
 103     def _test_programs_base_dir(self):
 104         return os.path.join(self._programs_path, "TestWebKitAPI")
 105
 106     def _get_tests_from_dir(self, test_dir):
 107         if not os.path.isdir(test_dir):
 108             return []
 109
 110         tests = []
 111         for test_file in os.listdir(test_dir):
 112             if not test_file.lower().startswith("test"):
 113                 continue
 114             test_path = os.path.join(test_dir, test_file)
 115             if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
 116                 tests.append(test_path)
 117         return tests
 118
 119     def _get_tests(self, initial_tests):
 120         tests = []
 121         for test in initial_tests:
 122             if os.path.isdir(test):
 123                 tests.extend(self._get_tests_from_dir(test))
 124             else:
 125                 tests.append(test)
 126         if tests:
 127             return tests
 128
 129         tests = []
 130         for test_dir in self.TEST_DIRS:
 131             absolute_test_dir = os.path.join(self._test_programs_base_dir(), test_dir)
 132             tests.extend(self._get_tests_from_dir(absolute_test_dir))
 133         return tests
 134
 135     def _lookup_atspi2_binary(self, filename):
 136         exec_prefix = common.pkg_config_file_variable('atspi-2', 'exec_prefix')
 137         if not exec_prefix:
 138             return None
 139         for path in ['libexec', 'lib/at-spi2-core', 'lib32/at-spi2-core', 'lib64/at-spi2-core']:
 140             filepath = os.path.join(exec_prefix, path, filename)
 141             if os.path.isfile(filepath):
 142                 return filepath
 143
 144         return None
 145
 146     def _start_accessibility_daemons(self):
 147         spi_bus_launcher_path = self._lookup_atspi2_binary('at-spi-bus-launcher')
 148         spi_registryd_path = self._lookup_atspi2_binary('at-spi2-registryd')
 149         if not spi_bus_launcher_path or not spi_registryd_path:
 150             return False
 151
 152         try:
 153             self._spi_bus_launcher = subprocess.Popen([spi_bus_launcher_path], env=self._test_env)
 154         except:
 155             sys.stderr.write("Failed to launch the accessibility bus\n")
 156             sys.stderr.flush()
 157             return False
 158
 159         # We need to wait until the SPI bus is launched before trying to start the SPI
 160         # registry, so we spin a main loop until the bus name appears on DBus.
 161         loop = GLib.MainLoop()
 162         Gio.bus_watch_name(Gio.BusType.SESSION, 'org.a11y.Bus', Gio.BusNameWatcherFlags.NONE,
 163                            lambda *args: loop.quit(), None)
 164         loop.run()
 165
 166         try:
 167             self._spi_registryd = subprocess.Popen([spi_registryd_path], env=self._test_env)
 168         except:
 169             sys.stderr.write("Failed to launch the accessibility registry\n")
 170             sys.stderr.flush()
 171             return False
 172
 173         return True
 174
 175     def _run_xvfb(self):
 176         self._xvfb = None
 177         if not self._options.use_xvfb:
 178             return True
 179
 180         self._test_env["DISPLAY"] = self._options.display
 181
 182         try:
 183             self._xvfb = subprocess.Popen(["Xvfb", self._options.display, "-screen", "0", "800x600x24", "-nolisten", "tcp"],
 184                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 185         except Exception as e:
 186             sys.stderr.write("Failed to run Xvfb: %s\n" % e)
 187             sys.stderr.flush()
 188             return False
 189
 190         return True
 191
 192     def _setup_testing_environment(self):
 193         self._test_env = os.environ
 194         self._test_env['GSETTINGS_BACKEND'] = 'memory'
 195         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit2")
 196         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.library_build_path()
 197         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
 198
 199         if not self._run_xvfb():
 200             return False
 201
 202         # If we cannot start the accessibility daemons, we can just skip the accessibility tests.
 203         if not self._start_accessibility_daemons():
 204             print "Could not start accessibility bus, so disabling TestWebKitAccessibility"
 205             self._disabled_tests.append("WebKit2APITests/TestWebKitAccessibility")
 206         return True
 207
 208     def _tear_down_testing_environment(self):
 209         if self._spi_registryd:
 210             self._spi_registryd.terminate()
 211         if self._spi_bus_launcher:
 212             self._spi_bus_launcher.terminate()
 213         if self._xvfb:
 214             self._xvfb.terminate()
 215
 216     def _test_cases_to_skip(self, test_program):
 217         if self._options.skipped_action != 'skip':
 218             return []
 219
 220         test_cases = []
 221         for skipped in self._skipped_tests:
 222             if test_program.endswith(skipped.test) and not skipped.skip_entire_suite():
 223                 test_cases.append(skipped.test_case)
 224         return test_cases
 225
 226     def _should_run_test_program(self, test_program):
 227         for disabled_test in self._disabled_tests:
 228             if test_program.endswith(disabled_test):
 229                 return False
 230
 231         if self._options.skipped_action != 'skip':
 232             return True
 233
 234         for skipped in self._skipped_tests:
 235             if test_program.endswith(skipped.test) and skipped.skip_entire_suite():
 236                 return False
 237         return True
 238
 239     def _get_child_pid_from_test_output(self, output):
 240         if not output:
 241             return -1
 242         match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', output)
 243         if not match:
 244             return -1
 245         return int(match.group('child_pid'))
 246
 247     def _kill_process(self, pid):
 248         try:
 249             os.kill(pid, SIGKILL)
 250         except OSError:
 251             # Process already died.
 252             pass
 253
 254     def _run_test_command(self, command, timeout=-1):
 255         def alarm_handler(signum, frame):
 256             raise TestTimeout
 257
 258         child_pid = [-1]
 259         def parse_line(line, child_pid = child_pid):
 260             if child_pid[0] == -1:
 261                 child_pid[0] = self._get_child_pid_from_test_output(line)
 262
 263             if sys.stdout.isatty():
 264                 sys.stdout.write(line)
 265             else:
 266                 sys.stdout.write(self._tty_colors_pattern.sub('', line.replace('\r', '')))
 267
 268         def waitpid(pid):
 269             while True:
 270                 try:
 271                     return os.waitpid(pid, 0)
 272                 except (OSError, IOError) as e:
 273                     if e.errno == errno.EINTR:
 274                         continue
 275                     raise
 276
 277         def return_code_from_exit_status(status):
 278             if os.WIFSIGNALED(status):
 279                 return -os.WTERMSIG(status)
 280             elif os.WIFEXITED(status):
 281                 return os.WEXITSTATUS(status)
 282             else:
 283                 # Should never happen
 284                 raise RuntimeError("Unknown child exit status!")
 285
 286         pid, fd = os.forkpty()
 287         if pid == 0:
 288             os.execvpe(command[0], command, self._test_env)
 289             sys.exit(0)
 290
 291         if timeout > 0:
 292             signal(SIGALRM, alarm_handler)
 293             alarm(timeout)
 294
 295         try:
 296             common.parse_output_lines(fd, parse_line)
 297             if timeout > 0:
 298                 alarm(0)
 299         except TestTimeout:
 300             self._kill_process(pid)
 301             if child_pid[0] > 0:
 302                 self._kill_process(child_pid[0])
 303             raise
 304
 305         try:
 306             dummy, status = waitpid(pid)
 307         except OSError as e:
 308             if e.errno != errno.ECHILD:
 309                 raise
 310             # This happens if SIGCLD is set to be ignored or waiting
 311             # for child processes has otherwise been disabled for our
 312             # process.  This child is dead, we can't get the status.
 313             status = 0
 314
 315         return return_code_from_exit_status(status)
 316
 317     def _run_test_glib(self, test_program):
 318         tester_command = ['gtester', '-k']
 319         if self._options.verbose:
 320             tester_command.append('--verbose')
 321         for test_case in self._test_cases_to_skip(test_program):
 322             tester_command.extend(['-s', test_case])
 323         tester_command.append(test_program)
 324
 325         return self._run_test_command(tester_command, self._options.timeout)
 326
 327     def _get_tests_from_google_test_suite(self, test_program):
 328         try:
 329             output = subprocess.check_output([test_program, '--gtest_list_tests'])
 330         except subprocess.CalledProcessError:
 331             sys.stderr.write("ERROR: could not list available tests for binary %s.\n" % (test_program))
 332             sys.stderr.flush()
 333             return 1
 334
 335         skipped_test_cases = self._test_cases_to_skip(test_program)
 336
 337         tests = []
 338         prefix = None
 339         for line in output.split('\n'):
 340             if not line.startswith('  '):
 341                 prefix = line
 342                 continue
 343             else:
 344                 test_name = prefix + line.strip()
 345                 if not test_name in skipped_test_cases:
 346                     tests.append(test_name)
 347         return tests
 348
 349     def _run_google_test(self, test_program, subtest):
 350         test_command = [test_program, '--gtest_filter=%s' % (subtest)]
 351         return self._run_test_command(test_command, self._options.timeout)
 352
 353     def _run_google_test_suite(self, test_program):
 354         for subtest in self._get_tests_from_google_test_suite(test_program):
 355             retcode = self._run_google_test(test_program, subtest)
 356             if retcode:
 357                 return retcode
 358         return 0
 359
 360     def _run_test(self, test_program):
 361         basedir = os.path.basename(os.path.dirname(test_program))
 362         if basedir in ["WebKit2Gtk", "WebKitGtk"]:
 363             return self._run_test_glib(test_program)
 364
 365         if basedir in ["WebKit2", "JavaScriptCore", "WTF", "WebCore",  "WebCoreGtk"]:
 366             return self._run_google_test_suite(test_program)
 367
 368         return 1
 369
 370     def run_tests(self):
 371         if not self._tests:
 372             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._test_programs_base_dir()))
 373             sys.stderr.flush()
 374             return 1
 375
 376         if not self._setup_testing_environment():
 377             return 1
 378
 379         # Remove skipped tests now instead of when we find them, because
 380         # some tests might be skipped while setting up the test environment.
 381         self._tests = [test for test in self._tests if self._should_run_test_program(test)]
 382
 383         crashed_tests = []
 384         failed_tests = []
 385         timed_out_tests = []
 386         try:
 387             for test in self._tests:
 388                 exit_status_code = 0
 389                 try:
 390                     exit_status_code = self._run_test(test)
 391                 except TestTimeout:
 392                     sys.stdout.write("TEST: %s: TIMEOUT\n" % test)
 393                     sys.stdout.flush()
 394                     timed_out_tests.append(test)
 395
 396                 if exit_status_code == -SIGSEGV:
 397                     sys.stdout.write("TEST: %s: CRASHED\n" % test)
 398                     sys.stdout.flush()
 399                     crashed_tests.append(test)
 400                 elif exit_status_code != 0:
 401                     failed_tests.append(test)
 402         finally:
 403             self._tear_down_testing_environment()
 404
 405         if failed_tests:
 406             names = [test.replace(self._test_programs_base_dir(), '', 1) for test in failed_tests]
 407             sys.stdout.write("Tests failed (%d): %s\n" % (len(names), ", ".join(names)))
 408             sys.stdout.flush()
 409
 410         if crashed_tests:
 411             names = [test.replace(self._test_programs_base_dir(), '', 1) for test in crashed_tests]
 412             sys.stdout.write("Tests that crashed (%d): %s\n" % (len(names), ", ".join(names)))
 413             sys.stdout.flush()
 414
 415         if timed_out_tests:
 416             names = [test.replace(self._test_programs_base_dir(), '', 1) for test in timed_out_tests]
 417             sys.stdout.write("Tests that timed out (%d): %s\n" % (len(names), ", ".join(names)))
 418             sys.stdout.flush()
 419
 420         if self._skipped_tests and self._options.skipped_action == 'skip':
 421             sys.stdout.write("Tests skipped (%d):\n%s\n" %
 422                              (len(self._skipped_tests),
 423                               "\n".join([str(skipped) for skipped in self._skipped_tests])))
 424             sys.stdout.flush()
 425
 426         return len(failed_tests) + len(timed_out_tests)
 427
 428 if __name__ == "__main__":
 429     if not jhbuildutils.enter_jhbuild_environment_if_available("gtk"):
 430         print "***"
 431         print "*** Warning: jhbuild environment not present. Run update-webkitgtk-libs before build-webkit to ensure proper testing."
 432         print "***"
 433
 434     option_parser = optparse.OptionParser(usage='usage: %prog [options] [test...]')
 435     option_parser.add_option('-r', '--release',
 436                              action='store_true', dest='release',
 437                              help='Run in Release')
 438     option_parser.add_option('-d', '--debug',
 439                              action='store_true', dest='debug',
 440                              help='Run in Debug')
 441     option_parser.add_option('-v', '--verbose',
 442                              action='store_true', dest='verbose',
 443                              help='Run gtester in verbose mode')
 444     option_parser.add_option('--display', action='store', dest='display', default=':55',
 445                              help='Display to run Xvfb')
 446     option_parser.add_option('--skipped', action='store', dest='skipped_action',
 447                              choices=['skip', 'ignore', 'only'], default='skip',
 448                              metavar='skip|ignore|only',
 449                              help='Specifies how to treat the skipped tests')
 450     option_parser.add_option('-t', '--timeout',
 451                              action='store', type='int', dest='timeout', default=10,
 452                              help='Time in seconds until a test times out')
 453     option_parser.add_option('--no-xvfb', action='store_false', dest='use_xvfb', default=True,
 454                              help='Do not run tests under Xvfb')
 455     options, args = option_parser.parse_args()
 456
 457     sys.exit(TestRunner(options, args).run_tests())