Unreviewed. Unskip inspector server unit tests.
[WebKit-https.git] / Tools / Scripts / run-gtk-tests
1 #!/usr/bin/env python
2 #
3 # Copyright (C) 2011, 2012 Igalia S.L.
4 #
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Library General Public
7 # License as published by the Free Software Foundation; either
8 # version 2 of the License, or (at your option) any later version.
9 #
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 # Library General Public License for more details.
14 #
15 # You should have received a copy of the GNU Library General Public License
16 # along with this library; see the file COPYING.LIB.  If not, write to
17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 # Boston, MA 02110-1301, USA.
19
20 import subprocess
21 import os
22 import sys
23 import optparse
24 import re
25 from signal import alarm, signal, SIGALRM, SIGKILL, SIGSEGV
26 from gi.repository import Gio, GLib
27
28 top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
29 sys.path.append(os.path.join(top_level_directory, "Tools", "jhbuild"))
30 sys.path.append(os.path.join(top_level_directory, "Tools", "gtk"))
31 import common
32 import jhbuildutils
33
34 class SkippedTest:
35     ENTIRE_SUITE = None
36
37     def __init__(self, test, test_case, reason, bug=None):
38         self.test = test
39         self.test_case = test_case
40         self.reason = reason
41         self.bug = bug
42
43     def __str__(self):
44         skipped_test_str = "%s" % self.test
45
46         if not(self.skip_entire_suite()):
47             skipped_test_str += " [%s]" % self.test_case
48
49         skipped_test_str += ": %s " % self.reason
50         if self.bug is not None:
51             skipped_test_str += "(https://bugs.webkit.org/show_bug.cgi?id=%d)" % self.bug
52         return skipped_test_str
53
54     def skip_entire_suite(self):
55         return self.test_case == SkippedTest.ENTIRE_SUITE
56
57 class TestTimeout(Exception):
58     pass
59
60 class TestRunner:
61     TEST_DIRS = [ "WebKit2Gtk", "WebKit2", "JavaScriptCore", "WTF", "WebCore" ]
62
63     SKIPPED = [
64         SkippedTest("WebKit2Gtk/TestUIClient", "/webkit2/WebKitWebView/mouse-target", "Test times out after r150890", 117689),
65         SkippedTest("WebKit2Gtk/TestContextMenu", SkippedTest.ENTIRE_SUITE, "Test times out after r150890", 117689),
66         SkippedTest("WebKit2APITests/TestWebKitAccessibility", "/webkit2/WebKitAccessibility/atspi-basic-hierarchy", "Test is flaky", 132134),
67         SkippedTest("WebKit2Gtk/TestWebKitWebView", "/webkit2/WebKitWebView/snapshot", "Test fails", 120404),
68         SkippedTest("WebKit2Gtk/TestWebKitWebView", "/webkit2/WebKitWebView/page-visibility", "Test fails or times out", 131731),
69         SkippedTest("WebKit2Gtk/TestCookieManager", "/webkit2/WebKitCookieManager/persistent-storage", "Test is flaky", 134580),
70         SkippedTest("WebKit2/TestWebKit2", "WebKit2.MouseMoveAfterCrash", "Test is flaky", 85066),
71         SkippedTest("WebKit2/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutForImages", "Test is flaky", 85066),
72         SkippedTest("WebKit2/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutFrames", "Test fails", 85037),
73         SkippedTest("WebKit2/TestWebKit2", "WebKit2.RestoreSessionStateContainingFormData", "Session State is not implemented in GTK+ port", 84960),
74         SkippedTest("WebKit2/TestWebKit2", "WebKit2.SpacebarScrolling", "Test fails", 84961),
75         SkippedTest("WebKit2/TestWebKit2", "WebKit2.WKConnection", "Tests fail and time out out", 84959),
76         SkippedTest("WebKit2/TestWebKit2", "WebKit2.WKPageGetScaleFactorNotZero", "Test fails and times out", 88455),
77         SkippedTest("WebKit2/TestWebKit2", "WebKit2.ForceRepaint", "Test times out", 105532),
78         SkippedTest("WebKit2/TestWebKit2", "WebKit2.ReloadPageAfterCrash", "Test flakily times out", 110129),
79         SkippedTest("WebKit2/TestWebKit2", "WebKit2.DidAssociateFormControls", "Test times out", 120302),
80         SkippedTest("WebKit2/TestWebKit2", "WebKit2.InjectedBundleFrameHitTest", "Test times out", 120303),
81         SkippedTest("WebKit2/TestWebKit2", "WebKit2.TerminateTwice", "Test causes crash on the next test", 121970),
82         SkippedTest("WebKit2/TestWebKit2", "WebKit2.GeolocationTransitionToHighAccuracy", "Test causes crash on the next test", 125068),
83         SkippedTest("WebKit2/TestWebKit2", "WebKit2.GeolocationTransitionToLowAccuracy", "Test causes crash on the next test", 125068),
84     ]
85
86     def __init__(self, options, tests=[]):
87         self._options = options
88
89         self._build_type = "Debug" if self._options.debug else "Release"
90         common.set_build_types((self._build_type,))
91
92         self._programs_path = common.binary_build_path()
93         self._tests = self._get_tests(tests)
94         self._skipped_tests = TestRunner.SKIPPED
95         self._disabled_tests = []
96         if not sys.stdout.isatty():
97             self._tty_colors_pattern = re.compile("\033\[[0-9;]*m")
98
99         # These SPI daemons need to be active for the accessibility tests to work.
100         self._spi_registryd = None
101         self._spi_bus_launcher = None
102
103     def _test_programs_base_dir(self):
104         return os.path.join(self._programs_path, "TestWebKitAPI")
105
106     def _get_tests_from_dir(self, test_dir):
107         if not os.path.isdir(test_dir):
108             return []
109
110         tests = []
111         for test_file in os.listdir(test_dir):
112             if not test_file.lower().startswith("test"):
113                 continue
114             test_path = os.path.join(test_dir, test_file)
115             if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
116                 tests.append(test_path)
117         return tests
118
119     def _get_tests(self, initial_tests):
120         tests = []
121         for test in initial_tests:
122             if os.path.isdir(test):
123                 tests.extend(self._get_tests_from_dir(test))
124             else:
125                 tests.append(test)
126         if tests:
127             return tests
128
129         tests = []
130         for test_dir in self.TEST_DIRS:
131             absolute_test_dir = os.path.join(self._test_programs_base_dir(), test_dir)
132             tests.extend(self._get_tests_from_dir(absolute_test_dir))
133         return tests
134
135     def _lookup_atspi2_binary(self, filename):
136         exec_prefix = common.pkg_config_file_variable('atspi-2', 'exec_prefix')
137         if not exec_prefix:
138             return None
139         for path in ['libexec', 'lib/at-spi2-core', 'lib32/at-spi2-core', 'lib64/at-spi2-core']:
140             filepath = os.path.join(exec_prefix, path, filename)
141             if os.path.isfile(filepath):
142                 return filepath
143
144         return None
145
146     def _start_accessibility_daemons(self):
147         spi_bus_launcher_path = self._lookup_atspi2_binary('at-spi-bus-launcher')
148         spi_registryd_path = self._lookup_atspi2_binary('at-spi2-registryd')
149         if not spi_bus_launcher_path or not spi_registryd_path:
150             return False
151
152         try:
153             self._spi_bus_launcher = subprocess.Popen([spi_bus_launcher_path], env=self._test_env)
154         except:
155             sys.stderr.write("Failed to launch the accessibility bus\n")
156             sys.stderr.flush()
157             return False
158
159         # We need to wait until the SPI bus is launched before trying to start the SPI
160         # registry, so we spin a main loop until the bus name appears on DBus.
161         loop = GLib.MainLoop()
162         Gio.bus_watch_name(Gio.BusType.SESSION, 'org.a11y.Bus', Gio.BusNameWatcherFlags.NONE,
163                            lambda *args: loop.quit(), None)
164         loop.run()
165
166         try:
167             self._spi_registryd = subprocess.Popen([spi_registryd_path], env=self._test_env)
168         except:
169             sys.stderr.write("Failed to launch the accessibility registry\n")
170             sys.stderr.flush()
171             return False
172
173         return True
174
175     def _run_xvfb(self):
176         self._xvfb = None
177         if not self._options.use_xvfb:
178             return True
179
180         self._test_env["DISPLAY"] = self._options.display
181
182         try:
183             self._xvfb = subprocess.Popen(["Xvfb", self._options.display, "-screen", "0", "800x600x24", "-nolisten", "tcp"],
184                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
185         except Exception as e:
186             sys.stderr.write("Failed to run Xvfb: %s\n" % e)
187             sys.stderr.flush()
188             return False
189
190         return True
191
192     def _setup_testing_environment(self):
193         self._test_env = os.environ
194         self._test_env['GSETTINGS_BACKEND'] = 'memory'
195         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit2")
196         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.library_build_path()
197         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
198
199         if not self._run_xvfb():
200             return False
201
202         # If we cannot start the accessibility daemons, we can just skip the accessibility tests.
203         if not self._start_accessibility_daemons():
204             print "Could not start accessibility bus, so disabling TestWebKitAccessibility"
205             self._disabled_tests.append("WebKit2APITests/TestWebKitAccessibility")
206         return True
207
208     def _tear_down_testing_environment(self):
209         if self._spi_registryd:
210             self._spi_registryd.terminate()
211         if self._spi_bus_launcher:
212             self._spi_bus_launcher.terminate()
213         if self._xvfb:
214             self._xvfb.terminate()
215
216     def _test_cases_to_skip(self, test_program):
217         if self._options.skipped_action != 'skip':
218             return []
219
220         test_cases = []
221         for skipped in self._skipped_tests:
222             if test_program.endswith(skipped.test) and not skipped.skip_entire_suite():
223                 test_cases.append(skipped.test_case)
224         return test_cases
225
226     def _should_run_test_program(self, test_program):
227         for disabled_test in self._disabled_tests:
228             if test_program.endswith(disabled_test):
229                 return False
230
231         if self._options.skipped_action != 'skip':
232             return True
233
234         for skipped in self._skipped_tests:
235             if test_program.endswith(skipped.test) and skipped.skip_entire_suite():
236                 return False
237         return True
238
239     def _get_child_pid_from_test_output(self, output):
240         if not output:
241             return -1
242         match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', output)
243         if not match:
244             return -1
245         return int(match.group('child_pid'))
246
247     def _kill_process(self, pid):
248         try:
249             os.kill(pid, SIGKILL)
250         except OSError:
251             # Process already died.
252             pass
253
254     def _run_test_command(self, command, timeout=-1):
255         def alarm_handler(signum, frame):
256             raise TestTimeout
257
258         child_pid = [-1]
259         def parse_line(line, child_pid = child_pid):
260             if child_pid[0] == -1:
261                 child_pid[0] = self._get_child_pid_from_test_output(line)
262
263             if sys.stdout.isatty():
264                 sys.stdout.write(line)
265             else:
266                 sys.stdout.write(self._tty_colors_pattern.sub('', line.replace('\r', '')))
267
268         def waitpid(pid):
269             while True:
270                 try:
271                     return os.waitpid(pid, 0)
272                 except (OSError, IOError) as e:
273                     if e.errno == errno.EINTR:
274                         continue
275                     raise
276
277         def return_code_from_exit_status(status):
278             if os.WIFSIGNALED(status):
279                 return -os.WTERMSIG(status)
280             elif os.WIFEXITED(status):
281                 return os.WEXITSTATUS(status)
282             else:
283                 # Should never happen
284                 raise RuntimeError("Unknown child exit status!")
285
286         pid, fd = os.forkpty()
287         if pid == 0:
288             os.execvpe(command[0], command, self._test_env)
289             sys.exit(0)
290
291         if timeout > 0:
292             signal(SIGALRM, alarm_handler)
293             alarm(timeout)
294
295         try:
296             common.parse_output_lines(fd, parse_line)
297             if timeout > 0:
298                 alarm(0)
299         except TestTimeout:
300             self._kill_process(pid)
301             if child_pid[0] > 0:
302                 self._kill_process(child_pid[0])
303             raise
304
305         try:
306             dummy, status = waitpid(pid)
307         except OSError as e:
308             if e.errno != errno.ECHILD:
309                 raise
310             # This happens if SIGCLD is set to be ignored or waiting
311             # for child processes has otherwise been disabled for our
312             # process.  This child is dead, we can't get the status.
313             status = 0
314
315         return return_code_from_exit_status(status)
316
317     def _run_test_glib(self, test_program):
318         tester_command = ['gtester', '-k']
319         if self._options.verbose:
320             tester_command.append('--verbose')
321         for test_case in self._test_cases_to_skip(test_program):
322             tester_command.extend(['-s', test_case])
323         tester_command.append(test_program)
324
325         return self._run_test_command(tester_command, self._options.timeout)
326
327     def _get_tests_from_google_test_suite(self, test_program):
328         try:
329             output = subprocess.check_output([test_program, '--gtest_list_tests'])
330         except subprocess.CalledProcessError:
331             sys.stderr.write("ERROR: could not list available tests for binary %s.\n" % (test_program))
332             sys.stderr.flush()
333             return 1
334
335         skipped_test_cases = self._test_cases_to_skip(test_program)
336
337         tests = []
338         prefix = None
339         for line in output.split('\n'):
340             if not line.startswith('  '):
341                 prefix = line
342                 continue
343             else:
344                 test_name = prefix + line.strip()
345                 if not test_name in skipped_test_cases:
346                     tests.append(test_name)
347         return tests
348
349     def _run_google_test(self, test_program, subtest):
350         test_command = [test_program, '--gtest_filter=%s' % (subtest)]
351         return self._run_test_command(test_command, self._options.timeout)
352
353     def _run_google_test_suite(self, test_program):
354         for subtest in self._get_tests_from_google_test_suite(test_program):
355             retcode = self._run_google_test(test_program, subtest)
356             if retcode:
357                 return retcode
358         return 0
359
360     def _run_test(self, test_program):
361         basedir = os.path.basename(os.path.dirname(test_program))
362         if basedir in ["WebKit2Gtk", "WebKitGtk"]:
363             return self._run_test_glib(test_program)
364
365         if basedir in ["WebKit2", "JavaScriptCore", "WTF", "WebCore",  "WebCoreGtk"]:
366             return self._run_google_test_suite(test_program)
367
368         return 1
369
370     def run_tests(self):
371         if not self._tests:
372             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._test_programs_base_dir()))
373             sys.stderr.flush()
374             return 1
375
376         if not self._setup_testing_environment():
377             return 1
378
379         # Remove skipped tests now instead of when we find them, because
380         # some tests might be skipped while setting up the test environment.
381         self._tests = [test for test in self._tests if self._should_run_test_program(test)]
382
383         crashed_tests = []
384         failed_tests = []
385         timed_out_tests = []
386         try:
387             for test in self._tests:
388                 exit_status_code = 0
389                 try:
390                     exit_status_code = self._run_test(test)
391                 except TestTimeout:
392                     sys.stdout.write("TEST: %s: TIMEOUT\n" % test)
393                     sys.stdout.flush()
394                     timed_out_tests.append(test)
395
396                 if exit_status_code == -SIGSEGV:
397                     sys.stdout.write("TEST: %s: CRASHED\n" % test)
398                     sys.stdout.flush()
399                     crashed_tests.append(test)
400                 elif exit_status_code != 0:
401                     failed_tests.append(test)
402         finally:
403             self._tear_down_testing_environment()
404
405         if failed_tests:
406             names = [test.replace(self._test_programs_base_dir(), '', 1) for test in failed_tests]
407             sys.stdout.write("Tests failed (%d): %s\n" % (len(names), ", ".join(names)))
408             sys.stdout.flush()
409
410         if crashed_tests:
411             names = [test.replace(self._test_programs_base_dir(), '', 1) for test in crashed_tests]
412             sys.stdout.write("Tests that crashed (%d): %s\n" % (len(names), ", ".join(names)))
413             sys.stdout.flush()
414
415         if timed_out_tests:
416             names = [test.replace(self._test_programs_base_dir(), '', 1) for test in timed_out_tests]
417             sys.stdout.write("Tests that timed out (%d): %s\n" % (len(names), ", ".join(names)))
418             sys.stdout.flush()
419
420         if self._skipped_tests and self._options.skipped_action == 'skip':
421             sys.stdout.write("Tests skipped (%d):\n%s\n" %
422                              (len(self._skipped_tests),
423                               "\n".join([str(skipped) for skipped in self._skipped_tests])))
424             sys.stdout.flush()
425
426         return len(failed_tests) + len(timed_out_tests)
427
428 if __name__ == "__main__":
429     if not jhbuildutils.enter_jhbuild_environment_if_available("gtk"):
430         print "***"
431         print "*** Warning: jhbuild environment not present. Run update-webkitgtk-libs before build-webkit to ensure proper testing."
432         print "***"
433
434     option_parser = optparse.OptionParser(usage='usage: %prog [options] [test...]')
435     option_parser.add_option('-r', '--release',
436                              action='store_true', dest='release',
437                              help='Run in Release')
438     option_parser.add_option('-d', '--debug',
439                              action='store_true', dest='debug',
440                              help='Run in Debug')
441     option_parser.add_option('-v', '--verbose',
442                              action='store_true', dest='verbose',
443                              help='Run gtester in verbose mode')
444     option_parser.add_option('--display', action='store', dest='display', default=':55',
445                              help='Display to run Xvfb')
446     option_parser.add_option('--skipped', action='store', dest='skipped_action',
447                              choices=['skip', 'ignore', 'only'], default='skip',
448                              metavar='skip|ignore|only',
449                              help='Specifies how to treat the skipped tests')
450     option_parser.add_option('-t', '--timeout',
451                              action='store', type='int', dest='timeout', default=10,
452                              help='Time in seconds until a test times out')
453     option_parser.add_option('--no-xvfb', action='store_false', dest='use_xvfb', default=True,
454                              help='Do not run tests under Xvfb')
455     options, args = option_parser.parse_args()
456
457     sys.exit(TestRunner(options, args).run_tests())