fb8233bfa129bf9d3ecc71c7aaf30055b21e71e0
[WebKit-https.git] / Tools / Scripts / run-gtk-tests
1 #!/usr/bin/env python
2 #
3 # Copyright (C) 2011, 2012 Igalia S.L.
4 #
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Library General Public
7 # License as published by the Free Software Foundation; either
8 # version 2 of the License, or (at your option) any later version.
9 #
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 # Library General Public License for more details.
14 #
15 # You should have received a copy of the GNU Library General Public License
16 # along with this library; see the file COPYING.LIB.  If not, write to
17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 # Boston, MA 02110-1301, USA.
19
20 import subprocess
21 import os
22 import sys
23 import optparse
24 import re
25 from signal import alarm, signal, SIGALRM, SIGKILL, SIGSEGV
26 from gi.repository import Gio, GLib
27
28 top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
29 sys.path.append(os.path.join(top_level_directory, "Tools", "jhbuild"))
30 sys.path.append(os.path.join(top_level_directory, "Tools", "gtk"))
31 import common
32 import jhbuildutils
33
34 class SkippedTest:
35     ENTIRE_SUITE = None
36
37     def __init__(self, test, test_case, reason, bug, build_type=None):
38         self.test = test
39         self.test_case = test_case
40         self.reason = reason
41         self.bug = bug
42         self.build_type = build_type
43
44     def __str__(self):
45         skipped_test_str = "%s" % self.test
46
47         if not(self.skip_entire_suite()):
48             skipped_test_str += " [%s]" % self.test_case
49
50         skipped_test_str += ": %s (https://bugs.webkit.org/show_bug.cgi?id=%d)" % (self.reason, self.bug)
51         return skipped_test_str
52
53     def skip_entire_suite(self):
54         return self.test_case == SkippedTest.ENTIRE_SUITE
55
56     def skip_for_build_type(self, build_type):
57         if self.build_type is None:
58             return True;
59
60         return self.build_type == build_type
61
62 class TestTimeout(Exception):
63     pass
64
65 class TestRunner:
66     TEST_DIRS = [ "WebKit2Gtk", "WebKit2", "JavaScriptCore", "WTF", "WebCore" ]
67
68     SKIPPED = [
69         SkippedTest("WebKit2Gtk/TestUIClient", "/webkit2/WebKitWebView/mouse-target", "Test times out after r150890", 117689),
70         SkippedTest("WebKit2Gtk/TestCookieManager", "/webkit2/WebKitCookieManager/persistent-storage", "Test is flaky", 134580),
71         SkippedTest("WebKit2Gtk/TestWebViewEditor", "/webkit2/WebKitWebView/editable/editable", "Test hits an assertion in Debug builds", 151654, "Debug"),
72         SkippedTest("WebKit2Gtk/TestWebExtensions", "/webkit2/WebKitWebView/install-missing-plugins-permission-request", "Test times out", 147822),
73         SkippedTest("WebKit2/TestWebKit2", "WebKit2.MouseMoveAfterCrash", "Test is flaky", 85066),
74         SkippedTest("WebKit2/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutForImages", "Test is flaky", 85066),
75         SkippedTest("WebKit2/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutFrames", "Test fails", 85037),
76         SkippedTest("WebKit2/TestWebKit2", "WebKit2.SpacebarScrolling", "Test fails", 84961),
77         SkippedTest("WebKit2/TestWebKit2", "WebKit2.WKConnection", "Tests fail and time out out", 84959),
78         SkippedTest("WebKit2/TestWebKit2", "WebKit2.ForceRepaint", "Test times out", 105532),
79         SkippedTest("WebKit2/TestWebKit2", "WebKit2.ReloadPageAfterCrash", "Test flakily times out", 110129),
80         SkippedTest("WebKit2/TestWebKit2", "WebKit2.DidAssociateFormControls", "Test times out", 120302),
81         SkippedTest("WebKit2/TestWebKit2", "WebKit2.InjectedBundleFrameHitTest", "Test times out", 120303),
82         SkippedTest("WebKit2/TestWebKit2", "WebKit2.TerminateTwice", "Test causes crash on the next test", 121970),
83         SkippedTest("WebKit2/TestWebKit2", "WebKit2.GeolocationTransitionToHighAccuracy", "Test causes crash on the next test", 125068),
84         SkippedTest("WebKit2/TestWebKit2", "WebKit2.GeolocationTransitionToLowAccuracy", "Test causes crash on the next test", 125068),
85     ]
86
87     SLOW = [
88         "WTF_Lock.ContendedShortSection",
89         "WTF_WordLock.ContendedShortSection"
90     ]
91
92     def __init__(self, options, tests=[]):
93         self._options = options
94
95         self._build_type = "Debug" if self._options.debug else "Release"
96         common.set_build_types((self._build_type,))
97
98         self._programs_path = common.binary_build_path()
99         self._tests = self._get_tests(tests)
100         self._skipped_tests = [skipped for skipped in TestRunner.SKIPPED if skipped.skip_for_build_type(self._build_type)]
101         self._disabled_tests = []
102
103         # These SPI daemons need to be active for the accessibility tests to work.
104         self._spi_registryd = None
105         self._spi_bus_launcher = None
106
107     def _test_programs_base_dir(self):
108         return os.path.join(self._programs_path, "TestWebKitAPI")
109
110     def _get_tests_from_dir(self, test_dir):
111         if not os.path.isdir(test_dir):
112             return []
113
114         tests = []
115         for test_file in os.listdir(test_dir):
116             if not test_file.lower().startswith("test"):
117                 continue
118             test_path = os.path.join(test_dir, test_file)
119             if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
120                 tests.append(test_path)
121         return tests
122
123     def _get_tests(self, initial_tests):
124         tests = []
125         for test in initial_tests:
126             if os.path.isdir(test):
127                 tests.extend(self._get_tests_from_dir(test))
128             else:
129                 tests.append(test)
130         if tests:
131             return tests
132
133         tests = []
134         for test_dir in self.TEST_DIRS:
135             absolute_test_dir = os.path.join(self._test_programs_base_dir(), test_dir)
136             tests.extend(self._get_tests_from_dir(absolute_test_dir))
137         return tests
138
139     def _lookup_atspi2_binary(self, filename):
140         exec_prefix = common.pkg_config_file_variable('atspi-2', 'exec_prefix')
141         if not exec_prefix:
142             return None
143         for path in ['libexec', 'lib/at-spi2-core', 'lib32/at-spi2-core', 'lib64/at-spi2-core']:
144             filepath = os.path.join(exec_prefix, path, filename)
145             if os.path.isfile(filepath):
146                 return filepath
147
148         return None
149
150     def _start_accessibility_daemons(self):
151         spi_bus_launcher_path = self._lookup_atspi2_binary('at-spi-bus-launcher')
152         spi_registryd_path = self._lookup_atspi2_binary('at-spi2-registryd')
153         if not spi_bus_launcher_path or not spi_registryd_path:
154             return False
155
156         try:
157             self._spi_bus_launcher = subprocess.Popen([spi_bus_launcher_path], env=self._test_env)
158         except:
159             sys.stderr.write("Failed to launch the accessibility bus\n")
160             sys.stderr.flush()
161             return False
162
163         # We need to wait until the SPI bus is launched before trying to start the SPI
164         # registry, so we spin a main loop until the bus name appears on DBus.
165         loop = GLib.MainLoop()
166         Gio.bus_watch_name(Gio.BusType.SESSION, 'org.a11y.Bus', Gio.BusNameWatcherFlags.NONE,
167                            lambda *args: loop.quit(), None)
168         loop.run()
169
170         try:
171             self._spi_registryd = subprocess.Popen([spi_registryd_path], env=self._test_env)
172         except:
173             sys.stderr.write("Failed to launch the accessibility registry\n")
174             sys.stderr.flush()
175             return False
176
177         return True
178
179     def _run_xvfb(self):
180         self._xvfb = None
181         if not self._options.use_xvfb:
182             return True
183
184         self._test_env["DISPLAY"] = self._options.display
185
186         try:
187             self._xvfb = subprocess.Popen(["Xvfb", self._options.display, "-screen", "0", "800x600x24", "-nolisten", "tcp"],
188                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
189         except Exception as e:
190             sys.stderr.write("Failed to run Xvfb: %s\n" % e)
191             sys.stderr.flush()
192             return False
193
194         return True
195
196     def _setup_testing_environment(self):
197         self._test_env = os.environ
198         self._test_env['GSETTINGS_BACKEND'] = 'memory'
199         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit2")
200         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.library_build_path()
201         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
202         self._test_env["OWR_USE_TEST_SOURCES"] = '1'
203
204         if not self._run_xvfb():
205             return False
206
207         # If we cannot start the accessibility daemons, we can just skip the accessibility tests.
208         if not self._start_accessibility_daemons():
209             print "Could not start accessibility bus, so disabling TestWebKitAccessibility"
210             self._disabled_tests.append("WebKit2APITests/TestWebKitAccessibility")
211         return True
212
213     def _tear_down_testing_environment(self):
214         if self._spi_registryd:
215             self._spi_registryd.terminate()
216         if self._spi_bus_launcher:
217             self._spi_bus_launcher.terminate()
218         if self._xvfb:
219             self._xvfb.terminate()
220
221     def _test_cases_to_skip(self, test_program):
222         if self._options.skipped_action != 'skip':
223             return []
224
225         test_cases = []
226         for skipped in self._skipped_tests:
227             if test_program.endswith(skipped.test) and not skipped.skip_entire_suite():
228                 test_cases.append(skipped.test_case)
229         return test_cases
230
231     def _should_run_test_program(self, test_program):
232         for disabled_test in self._disabled_tests:
233             if test_program.endswith(disabled_test):
234                 return False
235
236         if self._options.skipped_action != 'skip':
237             return True
238
239         for skipped in self._skipped_tests:
240             if test_program.endswith(skipped.test) and skipped.skip_entire_suite():
241                 return False
242         return True
243
244     def _get_child_pid_from_test_output(self, output):
245         if not output:
246             return -1
247         match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', output)
248         if not match:
249             return -1
250         return int(match.group('child_pid'))
251
252     def _kill_process(self, pid):
253         try:
254             os.kill(pid, SIGKILL)
255         except OSError:
256             # Process already died.
257             pass
258
259     def _run_test_command(self, command, timeout=-1):
260         def alarm_handler(signum, frame):
261             raise TestTimeout
262
263         child_pid = [-1]
264         def parse_line(line, child_pid = child_pid):
265             if child_pid[0] == -1:
266                 child_pid[0] = self._get_child_pid_from_test_output(line)
267
268             sys.stdout.write(line)
269
270         def waitpid(pid):
271             while True:
272                 try:
273                     return os.waitpid(pid, 0)
274                 except (OSError, IOError) as e:
275                     if e.errno == errno.EINTR:
276                         continue
277                     raise
278
279         def return_code_from_exit_status(status):
280             if os.WIFSIGNALED(status):
281                 return -os.WTERMSIG(status)
282             elif os.WIFEXITED(status):
283                 return os.WEXITSTATUS(status)
284             else:
285                 # Should never happen
286                 raise RuntimeError("Unknown child exit status!")
287
288         pid, fd = os.forkpty()
289         if pid == 0:
290             os.execvpe(command[0], command, self._test_env)
291             sys.exit(0)
292
293         if timeout > 0:
294             signal(SIGALRM, alarm_handler)
295             alarm(timeout)
296
297         try:
298             common.parse_output_lines(fd, parse_line)
299             if timeout > 0:
300                 alarm(0)
301         except TestTimeout:
302             self._kill_process(pid)
303             if child_pid[0] > 0:
304                 self._kill_process(child_pid[0])
305             raise
306
307         try:
308             dummy, status = waitpid(pid)
309         except OSError as e:
310             if e.errno != errno.ECHILD:
311                 raise
312             # This happens if SIGCLD is set to be ignored or waiting
313             # for child processes has otherwise been disabled for our
314             # process.  This child is dead, we can't get the status.
315             status = 0
316
317         return return_code_from_exit_status(status)
318
319     def _run_test_glib(self, test_program):
320         tester_command = ['gtester', '-k']
321         if self._options.verbose:
322             tester_command.append('--verbose')
323         for test_case in self._test_cases_to_skip(test_program):
324             tester_command.extend(['-s', test_case])
325         tester_command.append(test_program)
326
327         return self._run_test_command(tester_command, self._options.timeout)
328
329     def _get_tests_from_google_test_suite(self, test_program):
330         try:
331             output = subprocess.check_output([test_program, '--gtest_list_tests'])
332         except subprocess.CalledProcessError:
333             sys.stderr.write("ERROR: could not list available tests for binary %s.\n" % (test_program))
334             sys.stderr.flush()
335             return 1
336
337         skipped_test_cases = self._test_cases_to_skip(test_program)
338
339         tests = []
340         prefix = None
341         for line in output.split('\n'):
342             if not line.startswith('  '):
343                 prefix = line
344                 continue
345             else:
346                 test_name = prefix + line.strip()
347                 if not test_name in skipped_test_cases:
348                     tests.append(test_name)
349         return tests
350
351     def _run_google_test(self, test_program, subtest):
352         test_command = [test_program, '--gtest_filter=%s' % (subtest)]
353         timeout = self._options.timeout
354         if subtest in TestRunner.SLOW:
355             timeout *= 5
356
357         status = self._run_test_command(test_command, timeout)
358         if status == -SIGSEGV:
359             sys.stdout.write("**CRASH** %s\n" % subtest)
360             sys.stdout.flush()
361         return status
362
363     def _run_google_test_suite(self, test_program):
364         retcode = 0
365         for subtest in self._get_tests_from_google_test_suite(test_program):
366             if self._run_google_test(test_program, subtest):
367                 retcode = 1
368         return retcode
369
370     def _run_test(self, test_program):
371         basedir = os.path.basename(os.path.dirname(test_program))
372         if basedir in ["WebKit2Gtk", "WebKitGtk"]:
373             return self._run_test_glib(test_program)
374
375         if basedir in ["WebKit2", "JavaScriptCore", "WTF", "WebCore",  "WebCoreGtk"]:
376             return self._run_google_test_suite(test_program)
377
378         return 1
379
380     def run_tests(self):
381         if not self._tests:
382             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._test_programs_base_dir()))
383             sys.stderr.flush()
384             return 1
385
386         if not self._setup_testing_environment():
387             return 1
388
389         # Remove skipped tests now instead of when we find them, because
390         # some tests might be skipped while setting up the test environment.
391         self._tests = [test for test in self._tests if self._should_run_test_program(test)]
392
393         crashed_tests = []
394         failed_tests = []
395         timed_out_tests = []
396         try:
397             for test in self._tests:
398                 exit_status_code = 0
399                 try:
400                     exit_status_code = self._run_test(test)
401                 except TestTimeout:
402                     sys.stdout.write("TEST: %s: TIMEOUT\n" % test)
403                     sys.stdout.flush()
404                     timed_out_tests.append(test)
405
406                 if exit_status_code == -SIGSEGV:
407                     sys.stdout.write("TEST: %s: CRASHED\n" % test)
408                     sys.stdout.flush()
409                     crashed_tests.append(test)
410                 elif exit_status_code != 0:
411                     failed_tests.append(test)
412         finally:
413             self._tear_down_testing_environment()
414
415         if failed_tests:
416             names = [test.replace(self._test_programs_base_dir(), '', 1) for test in failed_tests]
417             sys.stdout.write("Tests failed (%d): %s\n" % (len(names), ", ".join(names)))
418             sys.stdout.flush()
419
420         if crashed_tests:
421             names = [test.replace(self._test_programs_base_dir(), '', 1) for test in crashed_tests]
422             sys.stdout.write("Tests that crashed (%d): %s\n" % (len(names), ", ".join(names)))
423             sys.stdout.flush()
424
425         if timed_out_tests:
426             names = [test.replace(self._test_programs_base_dir(), '', 1) for test in timed_out_tests]
427             sys.stdout.write("Tests that timed out (%d): %s\n" % (len(names), ", ".join(names)))
428             sys.stdout.flush()
429
430         if self._skipped_tests and self._options.skipped_action == 'skip':
431             sys.stdout.write("Tests skipped (%d):\n%s\n" %
432                              (len(self._skipped_tests),
433                               "\n".join([str(skipped) for skipped in self._skipped_tests])))
434             sys.stdout.flush()
435
436         return len(failed_tests) + len(timed_out_tests)
437
438 if __name__ == "__main__":
439     if not jhbuildutils.enter_jhbuild_environment_if_available("gtk"):
440         print "***"
441         print "*** Warning: jhbuild environment not present. Run update-webkitgtk-libs before build-webkit to ensure proper testing."
442         print "***"
443
444     option_parser = optparse.OptionParser(usage='usage: %prog [options] [test...]')
445     option_parser.add_option('-r', '--release',
446                              action='store_true', dest='release',
447                              help='Run in Release')
448     option_parser.add_option('-d', '--debug',
449                              action='store_true', dest='debug',
450                              help='Run in Debug')
451     option_parser.add_option('-v', '--verbose',
452                              action='store_true', dest='verbose',
453                              help='Run gtester in verbose mode')
454     option_parser.add_option('--display', action='store', dest='display', default=':55',
455                              help='Display to run Xvfb')
456     option_parser.add_option('--skipped', action='store', dest='skipped_action',
457                              choices=['skip', 'ignore', 'only'], default='skip',
458                              metavar='skip|ignore|only',
459                              help='Specifies how to treat the skipped tests')
460     option_parser.add_option('-t', '--timeout',
461                              action='store', type='int', dest='timeout', default=10,
462                              help='Time in seconds until a test times out')
463     option_parser.add_option('--no-xvfb', action='store_false', dest='use_xvfb', default=True,
464                              help='Do not run tests under Xvfb')
465     options, args = option_parser.parse_args()
466
467     sys.exit(TestRunner(options, args).run_tests())