[GTK] Include most of the WebKit2 unit tests into the build and testing
[WebKit-https.git] / Tools / Scripts / run-gtk-tests
1 #!/usr/bin/env python
2 #
3 # Copyright (C) 2011, 2012 Igalia S.L.
4 #
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Library General Public
7 # License as published by the Free Software Foundation; either
8 # version 2 of the License, or (at your option) any later version.
9 #
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 # Library General Public License for more details.
14 #
15 # You should have received a copy of the GNU Library General Public License
16 # along with this library; see the file COPYING.LIB.  If not, write to
17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 # Boston, MA 02110-1301, USA.
19
20 import subprocess
21 import os
22 import sys
23 import optparse
24 import re
25 from signal import alarm, signal, SIGALRM, SIGKILL
26 from gi.repository import Gio, GLib
27
28 top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
29 sys.path.append(os.path.join(top_level_directory, "Tools", "jhbuild"))
30 sys.path.append(os.path.join(top_level_directory, "Tools", "gtk"))
31 import common
32 import jhbuildutils
33
34 class SkippedTest:
35     ENTIRE_SUITE = None
36
37     def __init__(self, test, test_case, reason, bug=None):
38         self.test = test
39         self.test_case = test_case
40         self.reason = reason
41         self.bug = bug
42
43     def __str__(self):
44         skipped_test_str = "%s" % self.test
45
46         if not(self.skip_entire_suite()):
47             skipped_test_str += " [%s]" % self.test_case
48
49         skipped_test_str += ": %s " % self.reason
50         if self.bug is not None:
51             skipped_test_str += "(https://bugs.webkit.org/show_bug.cgi?id=%d)" % self.bug
52         return skipped_test_str
53
54     def skip_entire_suite(self):
55         return self.test_case == SkippedTest.ENTIRE_SUITE
56
57 class TestTimeout(Exception):
58     pass
59
60 class TestRunner:
61     TEST_DIRS = [ "unittests", "WebKit2APITests", "TestWebKitAPI" ]
62
63     SKIPPED = [
64         SkippedTest("unittests/testdownload", "/webkit/download/not-found", "Test fails in GTK Linux 64-bit Release bot", 82329),
65         SkippedTest("unittests/testwebinspector", "/webkit/webinspector/close-and-inspect", "Test is flaky in GTK Linux 32-bit Release bot", 82869),
66         SkippedTest("unittests/testwebresource", "/webkit/webresource/loading", "Test fails", 104689),
67         SkippedTest("unittests/testwebresource", "/webkit/webresource/sub_resource_loading", "Test fails in GTK Linux 64-bit Release bot", 82330),
68         SkippedTest("unittests/testwebview", "/webkit/webview/icon-uri", "Test times out in GTK Linux 64-bit Release bot", 82328),
69         SkippedTest("unittests/testatk", "/webkit/atk/getTextInParagraphAndBodyModerate", "Test fails", 105538),
70         SkippedTest("WebKit2APITests/TestResources", "/webkit2/WebKitWebView/resources", "Test is flaky in GTK Linux 32-bit Release bot", 82868),
71         SkippedTest("WebKit2APITests/TestWebKitAccessibility", "/webkit2/WebKitAccessibility/atspi-basic-hierarchy", "Test fails", 100408),
72         SkippedTest("WebKit2APITests/TestWebKitWebView", SkippedTest.ENTIRE_SUITE, "Test times out after r150890", 117689),
73         SkippedTest("WebKit2APITests/TestContextMenu", SkippedTest.ENTIRE_SUITE, "Test times out after r150890", 117689),
74         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.CanHandleRequest", "Test fails", 88453),
75         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.MouseMoveAfterCrash", "Test is flaky", 85066),
76         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutForImages", "Test is flaky", 85066),
77         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutFrames", "Test fails", 85037),
78         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.RestoreSessionStateContainingFormData", "Session State is not implemented in GTK+ port", 84960),
79         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.SpacebarScrolling", "Test fails", 84961),
80         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.WKConnection", "Tests fail and time out out", 84959),
81         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.WKPageGetScaleFactorNotZero", "Test fails and times out", 88455),
82         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.ForceRepaint", "Test times out", 105532),
83         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.ReloadPageAfterCrash", "Test flakily times out", 110129),
84         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.DidAssociateFormControls", "Test times out", 120302),
85         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.InjectedBundleFrameHitTest", "Test times out", 120303),
86         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.ResizeReversePaginatedWebView", "Test fails", 120305),
87         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.ScrollPinningBehaviors", "Test fails", 120306),
88     ]
89
90     def __init__(self, options, tests=[]):
91         self._options = options
92         self._build_type = "Debug" if self._options.debug else "Release"
93
94         self._programs_path = common.build_path_for_build_types((self._build_type,), "Programs")
95         self._tests = self._get_tests(tests)
96         self._skipped_tests = TestRunner.SKIPPED
97         if not sys.stdout.isatty():
98             self._tty_colors_pattern = re.compile("\033\[[0-9;]*m")
99
100         # These SPI daemons need to be active for the accessibility tests to work.
101         self._spi_registryd = None
102         self._spi_bus_launcher = None
103
104     def _get_tests_from_dir(self, test_dir):
105         if not os.path.isdir(test_dir):
106             return []
107
108         tests = []
109         for test_file in os.listdir(test_dir):
110             if not test_file.lower().startswith("test"):
111                 continue
112             test_path = os.path.join(test_dir, test_file)
113             if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
114                 tests.append(test_path)
115         return tests
116
117     def _get_tests(self, initial_tests):
118         tests = []
119         for test in initial_tests:
120             if os.path.isdir(test):
121                 tests.extend(self._get_tests_from_dir(test))
122             else:
123                 tests.append(test)
124         if tests:
125             return tests
126
127         tests = []
128         for test_dir in self.TEST_DIRS:
129             absolute_test_dir = os.path.join(self._programs_path, test_dir)
130             tests.extend(self._get_tests_from_dir(absolute_test_dir))
131         return tests
132
133     def _lookup_atspi2_binary(self, filename):
134         exec_prefix = common.pkg_config_file_variable('atspi-2', 'exec_prefix')
135         if not exec_prefix:
136             return None
137         for path in ['libexec', 'lib/at-spi2-core', 'lib32/at-spi2-core', 'lib64/at-spi2-core']:
138             filepath = os.path.join(exec_prefix, path, filename)
139             if os.path.isfile(filepath):
140                 return filepath
141
142         return None
143
144     def _start_accessibility_daemons(self):
145         spi_bus_launcher_path = self._lookup_atspi2_binary('at-spi-bus-launcher')
146         spi_registryd_path = self._lookup_atspi2_binary('at-spi2-registryd')
147         if not spi_bus_launcher_path or not spi_registryd_path:
148             return False
149
150         try:
151             self._ally_bus_launcher = subprocess.Popen([spi_bus_launcher_path], env=self._test_env)
152         except:
153             sys.stderr.write("Failed to launch the accessibility bus\n")
154             sys.stderr.flush()
155             return False
156
157         # We need to wait until the SPI bus is launched before trying to start the SPI
158         # registry, so we spin a main loop until the bus name appears on DBus.
159         loop = GLib.MainLoop()
160         Gio.bus_watch_name(Gio.BusType.SESSION, 'org.a11y.Bus', Gio.BusNameWatcherFlags.NONE,
161                            lambda *args: loop.quit(), None)
162         loop.run()
163
164         try:
165             self._spi_registryd = subprocess.Popen([spi_registryd_path], env=self._test_env)
166         except:
167             sys.stderr.write("Failed to launch the accessibility registry\n")
168             sys.stderr.flush()
169             return False
170
171         return True
172
173     def _run_xvfb(self):
174         self._xvfb = None
175         if not self._options.use_xvfb:
176             return True
177
178         self._test_env["DISPLAY"] = self._options.display
179
180         try:
181             self._xvfb = subprocess.Popen(["Xvfb", self._options.display, "-screen", "0", "800x600x24", "-nolisten", "tcp"],
182                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
183         except Exception as e:
184             sys.stderr.write("Failed to run Xvfb: %s\n" % e)
185             sys.stderr.flush()
186             return False
187
188         return True
189
190     def _setup_testing_environment(self):
191         self._test_env = os.environ
192         self._test_env["WEBKIT_INSPECTOR_PATH"] = os.path.abspath(os.path.join(self._programs_path, 'resources', 'inspector'))
193         self._test_env['GSETTINGS_BACKEND'] = 'memory'
194         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit2")
195         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.build_path_for_build_types((self._build_type,), "Libraries")
196         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
197
198         if not self._run_xvfb():
199             return False
200
201         # If we cannot start the accessibility daemons, we can just skip the accessibility tests.
202         if not self._start_accessibility_daemons():
203             print "Could not start accessibility bus, so skipping TestWebKitAccessibility"
204             self._skipped_tests.append(SkippedTest("WebKit2APITests/TestWebKitAccessibility", SkippedTest.ENTIRE_SUITE, "Could not start accessibility bus"))
205         return True
206
207     def _tear_down_testing_environment(self):
208         if self._spi_registryd:
209             self._spi_registryd.terminate()
210         if self._spi_bus_launcher:
211             self._spi_bus_launcher.terminate()
212         if self._xvfb:
213             self._xvfb.terminate()
214
215     def _test_cases_to_skip(self, test_program):
216         if self._options.skipped_action != 'skip':
217             return []
218
219         test_cases = []
220         for skipped in self._skipped_tests:
221             if test_program.endswith(skipped.test) and not skipped.skip_entire_suite():
222                 test_cases.append(skipped.test_case)
223         return test_cases
224
225     def _should_run_test_program(self, test_program):
226         # This is not affected by the command-line arguments, since programs are skipped for
227         # problems in the harness, such as failing to start the accessibility bus.
228         for skipped in self._skipped_tests:
229             if test_program.endswith(skipped.test) and skipped.skip_entire_suite():
230                 return False
231         return True
232
233     def _get_child_pid_from_test_output(self, output):
234         if not output:
235             return -1
236         match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', output)
237         if not match:
238             return -1
239         return int(match.group('child_pid'))
240
241     def _kill_process(self, pid):
242         try:
243             os.kill(pid, SIGKILL)
244         except OSError:
245             # Process already died.
246             pass
247
248     def _run_test_command(self, command, timeout=-1):
249         def alarm_handler(signum, frame):
250             raise TestTimeout
251
252         child_pid = [-1]
253         def parse_line(line, child_pid = child_pid):
254             if child_pid[0] == -1:
255                 child_pid[0] = self._get_child_pid_from_test_output(line)
256
257             if sys.stdout.isatty():
258                 sys.stdout.write(line)
259             else:
260                 sys.stdout.write(self._tty_colors_pattern.sub('', line.replace('\r', '')))
261
262         def waitpid(pid):
263             while True:
264                 try:
265                     return os.waitpid(pid, 0)
266                 except (OSError, IOError) as e:
267                     if e.errno == errno.EINTR:
268                         continue
269                     raise
270
271         def return_code_from_exit_status(status):
272             if os.WIFSIGNALED(status):
273                 return -os.WTERMSIG(status)
274             elif os.WIFEXITED(status):
275                 return os.WEXITSTATUS(status)
276             else:
277                 # Should never happen
278                 raise RuntimeError("Unknown child exit status!")
279
280         pid, fd = os.forkpty()
281         if pid == 0:
282             os.execvpe(command[0], command, self._test_env)
283             sys.exit(0)
284
285         if timeout > 0:
286             signal(SIGALRM, alarm_handler)
287             alarm(timeout)
288
289         try:
290             common.parse_output_lines(fd, parse_line)
291             if timeout > 0:
292                 alarm(0)
293         except TestTimeout:
294             self._kill_process(pid)
295             if child_pid[0] > 0:
296                 self._kill_process(child_pid[0])
297             raise
298
299         try:
300             dummy, status = waitpid(pid)
301         except OSError as e:
302             if e.errno != errno.ECHILD:
303                 raise
304             # This happens if SIGCLD is set to be ignored or waiting
305             # for child processes has otherwise been disabled for our
306             # process.  This child is dead, we can't get the status.
307             status = 0
308
309         return not return_code_from_exit_status(status)
310
311     def _run_test_glib(self, test_program):
312         tester_command = ['gtester']
313         if self._options.verbose:
314             tester_command.append('--verbose')
315         for test_case in self._test_cases_to_skip(test_program):
316             tester_command.extend(['-s', test_case])
317         tester_command.append(test_program)
318
319         return self._run_test_command(tester_command, self._options.timeout)
320
321     def _run_test_google(self, test_program):
322         tester_command = [test_program]
323         skipped_tests_cases = self._test_cases_to_skip(test_program)
324         if skipped_tests_cases:
325             tester_command.append("--gtest_filter=-%s" % ":".join(skipped_tests_cases))
326
327         return self._run_test_command(tester_command, self._options.timeout)
328
329     def _run_test(self, test_program):
330         if "unittests" in test_program or "WebKit2APITests" in test_program:
331             return self._run_test_glib(test_program)
332
333         if "TestWebKitAPI" in test_program:
334             return self._run_test_google(test_program)
335
336         return False
337
338     def run_tests(self):
339         if not self._tests:
340             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._programs_path))
341             sys.stderr.flush()
342             return 1
343
344         if not self._setup_testing_environment():
345             return 1
346
347         # Remove skipped tests now instead of when we find them, because
348         # some tests might be skipped while setting up the test environment.
349         self._tests = [test for test in self._tests if self._should_run_test_program(test)]
350
351         failed_tests = []
352         timed_out_tests = []
353         try:
354             for test in self._tests:
355                 success = True
356                 try:
357                     success = self._run_test(test)
358                 except TestTimeout:
359                     sys.stdout.write("TEST: %s: TIMEOUT\n" % test)
360                     sys.stdout.flush()
361                     timed_out_tests.append(test)
362
363                 if not success:
364                     failed_tests.append(test)
365         finally:
366             self._tear_down_testing_environment()
367
368         if failed_tests:
369             names = [test.replace(self._programs_path, '', 1) for test in failed_tests]
370             sys.stdout.write("Tests failed (%d): %s\n" % (len(names), ", ".join(names)))
371             sys.stdout.flush()
372
373         if timed_out_tests:
374             names = [test.replace(self._programs_path, '', 1) for test in timed_out_tests]
375             sys.stdout.write("Tests that timed out (%d): %s\n" % (len(names), ", ".join(names)))
376             sys.stdout.flush()
377
378         if self._skipped_tests and self._options.skipped_action == 'skip':
379             sys.stdout.write("Tests skipped (%d):\n%s\n" %
380                              (len(self._skipped_tests),
381                               "\n".join([str(skipped) for skipped in self._skipped_tests])))
382             sys.stdout.flush()
383
384         return len(failed_tests) + len(timed_out_tests)
385
386 if __name__ == "__main__":
387     if not jhbuildutils.enter_jhbuild_environment_if_available("gtk"):
388         print "***"
389         print "*** Warning: jhbuild environment not present. Run update-webkitgtk-libs before build-webkit to ensure proper testing."
390         print "***"
391
392     option_parser = optparse.OptionParser(usage='usage: %prog [options] [test...]')
393     option_parser.add_option('-r', '--release',
394                              action='store_true', dest='release',
395                              help='Run in Release')
396     option_parser.add_option('-d', '--debug',
397                              action='store_true', dest='debug',
398                              help='Run in Debug')
399     option_parser.add_option('-v', '--verbose',
400                              action='store_true', dest='verbose',
401                              help='Run gtester in verbose mode')
402     option_parser.add_option('--display', action='store', dest='display', default=':55',
403                              help='Display to run Xvfb')
404     option_parser.add_option('--skipped', action='store', dest='skipped_action',
405                              choices=['skip', 'ignore', 'only'], default='skip',
406                              metavar='skip|ignore|only',
407                              help='Specifies how to treat the skipped tests')
408     option_parser.add_option('-t', '--timeout',
409                              action='store', type='int', dest='timeout', default=10,
410                              help='Time in seconds until a test times out')
411     option_parser.add_option('--no-xvfb', action='store_false', dest='use_xvfb', default=True,
412                              help='Do not run tests under Xvfb')
413     options, args = option_parser.parse_args()
414
415     sys.exit(TestRunner(options, args).run_tests())