Tools/Scripts/run-gtk-tests

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (C) 2011, 2012 Igalia S.L.
   4 #
   5 # This library is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU Library General Public
   7 # License as published by the Free Software Foundation; either
   8 # version 2 of the License, or (at your option) any later version.
   9 #
  10 # This library is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 # Library General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU Library General Public License
  16 # along with this library; see the file COPYING.LIB.  If not, write to
  17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  18 # Boston, MA 02110-1301, USA.
  19
  20 import subprocess
  21 import os
  22 import sys
  23 import optparse
  24 import re
  25 from signal import alarm, signal, SIGALRM, SIGKILL
  26 from gi.repository import Gio, GLib
  27
  28 top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
  29 sys.path.append(os.path.join(top_level_directory, "Tools", "jhbuild"))
  30 sys.path.append(os.path.join(top_level_directory, "Tools", "gtk"))
  31 import common
  32 import jhbuildutils
  33
  34 class SkippedTest:
  35     ENTIRE_SUITE = None
  36
  37     def __init__(self, test, test_case, reason, bug=None):
  38         self.test = test
  39         self.test_case = test_case
  40         self.reason = reason
  41         self.bug = bug
  42
  43     def __str__(self):
  44         skipped_test_str = "%s" % self.test
  45
  46         if not(self.skip_entire_suite()):
  47             skipped_test_str += " [%s]" % self.test_case
  48
  49         skipped_test_str += ": %s " % self.reason
  50         if self.bug is not None:
  51             skipped_test_str += "(https://bugs.webkit.org/show_bug.cgi?id=%d)" % self.bug
  52         return skipped_test_str
  53
  54     def skip_entire_suite(self):
  55         return self.test_case == SkippedTest.ENTIRE_SUITE
  56
  57 class TestTimeout(Exception):
  58     pass
  59
  60 class TestRunner:
  61     TEST_DIRS = [ "unittests", "WebKit2APITests", "TestWebKitAPI" ]
  62
  63     SKIPPED = [
  64         SkippedTest("unittests/testdownload", "/webkit/download/not-found", "Test fails in GTK Linux 64-bit Release bot", 82329),
  65         SkippedTest("unittests/testwebinspector", "/webkit/webinspector/close-and-inspect", "Test is flaky in GTK Linux 32-bit Release bot", 82869),
  66         SkippedTest("unittests/testwebresource", "/webkit/webresource/loading", "Test fails", 104689),
  67         SkippedTest("unittests/testwebresource", "/webkit/webresource/sub_resource_loading", "Test fails in GTK Linux 64-bit Release bot", 82330),
  68         SkippedTest("unittests/testwebview", "/webkit/webview/icon-uri", "Test times out in GTK Linux 64-bit Release bot", 82328),
  69         SkippedTest("WebKit2APITests/TestResources", "/webkit2/WebKitWebView/resources", "Test is flaky in GTK Linux 32-bit Release bot", 82868),
  70         SkippedTest("WebKit2APITests/TestWebKitWebView", SkippedTest.ENTIRE_SUITE, "Test times out after r150890", 117689),
  71         SkippedTest("WebKit2APITests/TestContextMenu", SkippedTest.ENTIRE_SUITE, "Test times out after r150890", 117689),
  72         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.CanHandleRequest", "Test fails", 88453),
  73         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.MouseMoveAfterCrash", "Test is flaky", 85066),
  74         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutForImages", "Test is flaky", 85066),
  75         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutFrames", "Test fails", 85037),
  76         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.RestoreSessionStateContainingFormData", "Session State is not implemented in GTK+ port", 84960),
  77         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.SpacebarScrolling", "Test fails", 84961),
  78         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.WKConnection", "Tests fail and time out out", 84959),
  79         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.WKPageGetScaleFactorNotZero", "Test fails and times out", 88455),
  80         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.ForceRepaint", "Test times out", 105532),
  81         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.ReloadPageAfterCrash", "Test flakily times out", 110129),
  82         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.DidAssociateFormControls", "Test times out", 120302),
  83         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.InjectedBundleFrameHitTest", "Test times out", 120303),
  84         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.ResizeReversePaginatedWebView", "Test fails", 120305),
  85         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.ScrollPinningBehaviors", "Test fails", 120306),
  86     ]
  87
  88     def __init__(self, options, tests=[]):
  89         self._options = options
  90         self._build_type = "Debug" if self._options.debug else "Release"
  91
  92         self._programs_path = common.build_path_for_build_types((self._build_type,), "Programs")
  93         self._tests = self._get_tests(tests)
  94         self._skipped_tests = TestRunner.SKIPPED
  95         if not sys.stdout.isatty():
  96             self._tty_colors_pattern = re.compile("\033\[[0-9;]*m")
  97
  98         # These SPI daemons need to be active for the accessibility tests to work.
  99         self._spi_registryd = None
 100         self._spi_bus_launcher = None
 101
 102     def _get_tests_from_dir(self, test_dir):
 103         if not os.path.isdir(test_dir):
 104             return []
 105
 106         tests = []
 107         for test_file in os.listdir(test_dir):
 108             if not test_file.lower().startswith("test"):
 109                 continue
 110             test_path = os.path.join(test_dir, test_file)
 111             if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
 112                 tests.append(test_path)
 113         return tests
 114
 115     def _get_tests(self, initial_tests):
 116         tests = []
 117         for test in initial_tests:
 118             if os.path.isdir(test):
 119                 tests.extend(self._get_tests_from_dir(test))
 120             else:
 121                 tests.append(test)
 122         if tests:
 123             return tests
 124
 125         tests = []
 126         for test_dir in self.TEST_DIRS:
 127             absolute_test_dir = os.path.join(self._programs_path, test_dir)
 128             tests.extend(self._get_tests_from_dir(absolute_test_dir))
 129         return tests
 130
 131     def _lookup_atspi2_binary(self, filename):
 132         exec_prefix = common.pkg_config_file_variable('atspi-2', 'exec_prefix')
 133         if not exec_prefix:
 134             return None
 135         for path in ['libexec', 'lib/at-spi2-core', 'lib32/at-spi2-core', 'lib64/at-spi2-core']:
 136             filepath = os.path.join(exec_prefix, path, filename)
 137             if os.path.isfile(filepath):
 138                 return filepath
 139
 140         return None
 141
 142     def _start_accessibility_daemons(self):
 143         spi_bus_launcher_path = self._lookup_atspi2_binary('at-spi-bus-launcher')
 144         spi_registryd_path = self._lookup_atspi2_binary('at-spi2-registryd')
 145         if not spi_bus_launcher_path or not spi_registryd_path:
 146             return False
 147
 148         try:
 149             self._ally_bus_launcher = subprocess.Popen([spi_bus_launcher_path], env=self._test_env)
 150         except:
 151             sys.stderr.write("Failed to launch the accessibility bus\n")
 152             sys.stderr.flush()
 153             return False
 154
 155         # We need to wait until the SPI bus is launched before trying to start the SPI
 156         # registry, so we spin a main loop until the bus name appears on DBus.
 157         loop = GLib.MainLoop()
 158         Gio.bus_watch_name(Gio.BusType.SESSION, 'org.a11y.Bus', Gio.BusNameWatcherFlags.NONE,
 159                            lambda *args: loop.quit(), None)
 160         loop.run()
 161
 162         try:
 163             self._spi_registryd = subprocess.Popen([spi_registryd_path], env=self._test_env)
 164         except:
 165             sys.stderr.write("Failed to launch the accessibility registry\n")
 166             sys.stderr.flush()
 167             return False
 168
 169         return True
 170
 171     def _run_xvfb(self):
 172         self._xvfb = None
 173         if not self._options.use_xvfb:
 174             return True
 175
 176         self._test_env["DISPLAY"] = self._options.display
 177
 178         try:
 179             self._xvfb = subprocess.Popen(["Xvfb", self._options.display, "-screen", "0", "800x600x24", "-nolisten", "tcp"],
 180                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 181         except Exception as e:
 182             sys.stderr.write("Failed to run Xvfb: %s\n" % e)
 183             sys.stderr.flush()
 184             return False
 185
 186         return True
 187
 188     def _setup_testing_environment(self):
 189         self._test_env = os.environ
 190         self._test_env["WEBKIT_INSPECTOR_PATH"] = os.path.abspath(os.path.join(self._programs_path, 'resources', 'inspector'))
 191         self._test_env['GSETTINGS_BACKEND'] = 'memory'
 192         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit2")
 193         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.build_path_for_build_types((self._build_type,), "Libraries")
 194         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
 195
 196         if not self._run_xvfb():
 197             return False
 198
 199         # If we cannot start the accessibility daemons, we can just skip the accessibility tests.
 200         if not self._start_accessibility_daemons():
 201             print "Could not start accessibility bus, so skipping TestWebKitAccessibility"
 202             self._skipped_tests.append(SkippedTest("WebKit2APITests/TestWebKitAccessibility", SkippedTest.ENTIRE_SUITE, "Could not start accessibility bus"))
 203         return True
 204
 205     def _tear_down_testing_environment(self):
 206         if self._spi_registryd:
 207             self._spi_registryd.terminate()
 208         if self._spi_bus_launcher:
 209             self._spi_bus_launcher.terminate()
 210         if self._xvfb:
 211             self._xvfb.terminate()
 212
 213     def _test_cases_to_skip(self, test_program):
 214         if self._options.skipped_action != 'skip':
 215             return []
 216
 217         test_cases = []
 218         for skipped in self._skipped_tests:
 219             if test_program.endswith(skipped.test) and not skipped.skip_entire_suite():
 220                 test_cases.append(skipped.test_case)
 221         return test_cases
 222
 223     def _should_run_test_program(self, test_program):
 224         # This is not affected by the command-line arguments, since programs are skipped for
 225         # problems in the harness, such as failing to start the accessibility bus.
 226         for skipped in self._skipped_tests:
 227             if test_program.endswith(skipped.test) and skipped.skip_entire_suite():
 228                 return False
 229         return True
 230
 231     def _get_child_pid_from_test_output(self, output):
 232         if not output:
 233             return -1
 234         match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', output)
 235         if not match:
 236             return -1
 237         return int(match.group('child_pid'))
 238
 239     def _kill_process(self, pid):
 240         try:
 241             os.kill(pid, SIGKILL)
 242         except OSError:
 243             # Process already died.
 244             pass
 245
 246     def _run_test_command(self, command, timeout=-1):
 247         def alarm_handler(signum, frame):
 248             raise TestTimeout
 249
 250         child_pid = [-1]
 251         def parse_line(line, child_pid = child_pid):
 252             if child_pid[0] == -1:
 253                 child_pid[0] = self._get_child_pid_from_test_output(line)
 254
 255             if sys.stdout.isatty():
 256                 sys.stdout.write(line)
 257             else:
 258                 sys.stdout.write(self._tty_colors_pattern.sub('', line.replace('\r', '')))
 259
 260         def waitpid(pid):
 261             while True:
 262                 try:
 263                     return os.waitpid(pid, 0)
 264                 except (OSError, IOError) as e:
 265                     if e.errno == errno.EINTR:
 266                         continue
 267                     raise
 268
 269         def return_code_from_exit_status(status):
 270             if os.WIFSIGNALED(status):
 271                 return -os.WTERMSIG(status)
 272             elif os.WIFEXITED(status):
 273                 return os.WEXITSTATUS(status)
 274             else:
 275                 # Should never happen
 276                 raise RuntimeError("Unknown child exit status!")
 277
 278         pid, fd = os.forkpty()
 279         if pid == 0:
 280             os.execvpe(command[0], command, self._test_env)
 281             sys.exit(0)
 282
 283         if timeout > 0:
 284             signal(SIGALRM, alarm_handler)
 285             alarm(timeout)
 286
 287         try:
 288             common.parse_output_lines(fd, parse_line)
 289             if timeout > 0:
 290                 alarm(0)
 291         except TestTimeout:
 292             self._kill_process(pid)
 293             if child_pid[0] > 0:
 294                 self._kill_process(child_pid[0])
 295             raise
 296
 297         try:
 298             dummy, status = waitpid(pid)
 299         except OSError as e:
 300             if e.errno != errno.ECHILD:
 301                 raise
 302             # This happens if SIGCLD is set to be ignored or waiting
 303             # for child processes has otherwise been disabled for our
 304             # process.  This child is dead, we can't get the status.
 305             status = 0
 306
 307         return not return_code_from_exit_status(status)
 308
 309     def _run_test_glib(self, test_program):
 310         tester_command = ['gtester']
 311         if self._options.verbose:
 312             tester_command.append('--verbose')
 313         for test_case in self._test_cases_to_skip(test_program):
 314             tester_command.extend(['-s', test_case])
 315         tester_command.append(test_program)
 316
 317         return self._run_test_command(tester_command, self._options.timeout)
 318
 319     def _run_test_google(self, test_program):
 320         tester_command = [test_program]
 321         skipped_tests_cases = self._test_cases_to_skip(test_program)
 322         if skipped_tests_cases:
 323             tester_command.append("--gtest_filter=-%s" % ":".join(skipped_tests_cases))
 324
 325         return self._run_test_command(tester_command, self._options.timeout)
 326
 327     def _run_test(self, test_program):
 328         if "unittests" in test_program or "WebKit2APITests" in test_program:
 329             return self._run_test_glib(test_program)
 330
 331         if "TestWebKitAPI" in test_program:
 332             return self._run_test_google(test_program)
 333
 334         return False
 335
 336     def run_tests(self):
 337         if not self._tests:
 338             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._programs_path))
 339             sys.stderr.flush()
 340             return 1
 341
 342         if not self._setup_testing_environment():
 343             return 1
 344
 345         # Remove skipped tests now instead of when we find them, because
 346         # some tests might be skipped while setting up the test environment.
 347         self._tests = [test for test in self._tests if self._should_run_test_program(test)]
 348
 349         failed_tests = []
 350         timed_out_tests = []
 351         try:
 352             for test in self._tests:
 353                 success = True
 354                 try:
 355                     success = self._run_test(test)
 356                 except TestTimeout:
 357                     sys.stdout.write("TEST: %s: TIMEOUT\n" % test)
 358                     sys.stdout.flush()
 359                     timed_out_tests.append(test)
 360
 361                 if not success:
 362                     failed_tests.append(test)
 363         finally:
 364             self._tear_down_testing_environment()
 365
 366         if failed_tests:
 367             names = [test.replace(self._programs_path, '', 1) for test in failed_tests]
 368             sys.stdout.write("Tests failed (%d): %s\n" % (len(names), ", ".join(names)))
 369             sys.stdout.flush()
 370
 371         if timed_out_tests:
 372             names = [test.replace(self._programs_path, '', 1) for test in timed_out_tests]
 373             sys.stdout.write("Tests that timed out (%d): %s\n" % (len(names), ", ".join(names)))
 374             sys.stdout.flush()
 375
 376         if self._skipped_tests and self._options.skipped_action == 'skip':
 377             sys.stdout.write("Tests skipped (%d):\n%s\n" %
 378                              (len(self._skipped_tests),
 379                               "\n".join([str(skipped) for skipped in self._skipped_tests])))
 380             sys.stdout.flush()
 381
 382         return len(failed_tests) + len(timed_out_tests)
 383
 384 if __name__ == "__main__":
 385     if not jhbuildutils.enter_jhbuild_environment_if_available("gtk"):
 386         print "***"
 387         print "*** Warning: jhbuild environment not present. Run update-webkitgtk-libs before build-webkit to ensure proper testing."
 388         print "***"
 389
 390     option_parser = optparse.OptionParser(usage='usage: %prog [options] [test...]')
 391     option_parser.add_option('-r', '--release',
 392                              action='store_true', dest='release',
 393                              help='Run in Release')
 394     option_parser.add_option('-d', '--debug',
 395                              action='store_true', dest='debug',
 396                              help='Run in Debug')
 397     option_parser.add_option('-v', '--verbose',
 398                              action='store_true', dest='verbose',
 399                              help='Run gtester in verbose mode')
 400     option_parser.add_option('--display', action='store', dest='display', default=':55',
 401                              help='Display to run Xvfb')
 402     option_parser.add_option('--skipped', action='store', dest='skipped_action',
 403                              choices=['skip', 'ignore', 'only'], default='skip',
 404                              metavar='skip|ignore|only',
 405                              help='Specifies how to treat the skipped tests')
 406     option_parser.add_option('-t', '--timeout',
 407                              action='store', type='int', dest='timeout', default=10,
 408                              help='Time in seconds until a test times out')
 409     option_parser.add_option('--no-xvfb', action='store_false', dest='use_xvfb', default=True,
 410                              help='Do not run tests under Xvfb')
 411     options, args = option_parser.parse_args()
 412
 413     sys.exit(TestRunner(options, args).run_tests())