[GTK][EFL] Upgrade OpenWebRTC dependency
[WebKit-https.git] / Tools / Scripts / run-gtk-tests
1 #!/usr/bin/env python
2 #
3 # Copyright (C) 2011, 2012 Igalia S.L.
4 #
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Library General Public
7 # License as published by the Free Software Foundation; either
8 # version 2 of the License, or (at your option) any later version.
9 #
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 # Library General Public License for more details.
14 #
15 # You should have received a copy of the GNU Library General Public License
16 # along with this library; see the file COPYING.LIB.  If not, write to
17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 # Boston, MA 02110-1301, USA.
19
20 import subprocess
21 import os
22 import sys
23 import optparse
24 import re
25 from signal import alarm, signal, SIGALRM, SIGKILL, SIGSEGV
26 from gi.repository import Gio, GLib
27
28 top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
29 sys.path.append(os.path.join(top_level_directory, "Tools", "jhbuild"))
30 sys.path.append(os.path.join(top_level_directory, "Tools", "gtk"))
31 import common
32 import jhbuildutils
33
34 class SkippedTest:
35     ENTIRE_SUITE = None
36
37     def __init__(self, test, test_case, reason, bug, build_type=None):
38         self.test = test
39         self.test_case = test_case
40         self.reason = reason
41         self.bug = bug
42         self.build_type = build_type
43
44     def __str__(self):
45         skipped_test_str = "%s" % self.test
46
47         if not(self.skip_entire_suite()):
48             skipped_test_str += " [%s]" % self.test_case
49
50         skipped_test_str += ": %s (https://bugs.webkit.org/show_bug.cgi?id=%d)" % (self.reason, self.bug)
51         return skipped_test_str
52
53     def skip_entire_suite(self):
54         return self.test_case == SkippedTest.ENTIRE_SUITE
55
56     def skip_for_build_type(self, build_type):
57         if self.build_type is None:
58             return True;
59
60         return self.build_type == build_type
61
62 class TestTimeout(Exception):
63     pass
64
65 class TestRunner:
66     TEST_DIRS = [ "WebKit2Gtk", "WebKit2", "JavaScriptCore", "WTF", "WebCore" ]
67
68     SKIPPED = [
69         SkippedTest("WebKit2Gtk/TestUIClient", "/webkit2/WebKitWebView/mouse-target", "Test times out after r150890", 117689),
70         SkippedTest("WebKit2Gtk/TestCookieManager", "/webkit2/WebKitCookieManager/persistent-storage", "Test is flaky", 134580),
71         SkippedTest("WebKit2Gtk/TestWebViewEditor", "/webkit2/WebKitWebView/editable/editable", "Test hits an assertion in Debug builds", 151654, "Debug"),
72         SkippedTest("WebKit2Gtk/TestWebExtensions", "/webkit2/WebKitWebView/install-missing-plugins-permission-request", "Test times out", 147822),
73         SkippedTest("WebKit2/TestWebKit2", "WebKit2.MouseMoveAfterCrash", "Test is flaky", 85066),
74         SkippedTest("WebKit2/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutForImages", "Test is flaky", 85066),
75         SkippedTest("WebKit2/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutFrames", "Test fails", 85037),
76         SkippedTest("WebKit2/TestWebKit2", "WebKit2.SpacebarScrolling", "Test fails", 84961),
77         SkippedTest("WebKit2/TestWebKit2", "WebKit2.WKConnection", "Tests fail and time out out", 84959),
78         SkippedTest("WebKit2/TestWebKit2", "WebKit2.ForceRepaint", "Test times out", 105532),
79         SkippedTest("WebKit2/TestWebKit2", "WebKit2.ReloadPageAfterCrash", "Test flakily times out", 110129),
80         SkippedTest("WebKit2/TestWebKit2", "WebKit2.DidAssociateFormControls", "Test times out", 120302),
81         SkippedTest("WebKit2/TestWebKit2", "WebKit2.InjectedBundleFrameHitTest", "Test times out", 120303),
82         SkippedTest("WebKit2/TestWebKit2", "WebKit2.TerminateTwice", "Test causes crash on the next test", 121970),
83         SkippedTest("WebKit2/TestWebKit2", "WebKit2.GeolocationTransitionToHighAccuracy", "Test causes crash on the next test", 125068),
84         SkippedTest("WebKit2/TestWebKit2", "WebKit2.GeolocationTransitionToLowAccuracy", "Test causes crash on the next test", 125068),
85         SkippedTest("WebKit2/UserMedia", "WebKit2.UserMediaBasic", "We will not test by default until the upgrade gstreamer to 1.9", 153540),
86     ]
87
88     SLOW = [
89         "WTF_Lock.ContendedShortSection",
90         "WTF_WordLock.ContendedShortSection"
91     ]
92
93     def __init__(self, options, tests=[]):
94         self._options = options
95
96         self._build_type = "Debug" if self._options.debug else "Release"
97         common.set_build_types((self._build_type,))
98
99         self._programs_path = common.binary_build_path()
100         self._tests = self._get_tests(tests)
101         self._skipped_tests = [skipped for skipped in TestRunner.SKIPPED if skipped.skip_for_build_type(self._build_type)]
102         self._disabled_tests = []
103
104         # These SPI daemons need to be active for the accessibility tests to work.
105         self._spi_registryd = None
106         self._spi_bus_launcher = None
107
108     def _test_programs_base_dir(self):
109         return os.path.join(self._programs_path, "TestWebKitAPI")
110
111     def _get_tests_from_dir(self, test_dir):
112         if not os.path.isdir(test_dir):
113             return []
114
115         tests = []
116         for test_file in os.listdir(test_dir):
117             if not test_file.lower().startswith("test"):
118                 continue
119             test_path = os.path.join(test_dir, test_file)
120             if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
121                 tests.append(test_path)
122         return tests
123
124     def _get_tests(self, initial_tests):
125         tests = []
126         for test in initial_tests:
127             if os.path.isdir(test):
128                 tests.extend(self._get_tests_from_dir(test))
129             else:
130                 tests.append(test)
131         if tests:
132             return tests
133
134         tests = []
135         for test_dir in self.TEST_DIRS:
136             absolute_test_dir = os.path.join(self._test_programs_base_dir(), test_dir)
137             tests.extend(self._get_tests_from_dir(absolute_test_dir))
138         return tests
139
140     def _lookup_atspi2_binary(self, filename):
141         exec_prefix = common.pkg_config_file_variable('atspi-2', 'exec_prefix')
142         if not exec_prefix:
143             return None
144         for path in ['libexec', 'lib/at-spi2-core', 'lib32/at-spi2-core', 'lib64/at-spi2-core']:
145             filepath = os.path.join(exec_prefix, path, filename)
146             if os.path.isfile(filepath):
147                 return filepath
148
149         return None
150
151     def _start_accessibility_daemons(self):
152         spi_bus_launcher_path = self._lookup_atspi2_binary('at-spi-bus-launcher')
153         spi_registryd_path = self._lookup_atspi2_binary('at-spi2-registryd')
154         if not spi_bus_launcher_path or not spi_registryd_path:
155             return False
156
157         try:
158             self._spi_bus_launcher = subprocess.Popen([spi_bus_launcher_path], env=self._test_env)
159         except:
160             sys.stderr.write("Failed to launch the accessibility bus\n")
161             sys.stderr.flush()
162             return False
163
164         # We need to wait until the SPI bus is launched before trying to start the SPI
165         # registry, so we spin a main loop until the bus name appears on DBus.
166         loop = GLib.MainLoop()
167         Gio.bus_watch_name(Gio.BusType.SESSION, 'org.a11y.Bus', Gio.BusNameWatcherFlags.NONE,
168                            lambda *args: loop.quit(), None)
169         loop.run()
170
171         try:
172             self._spi_registryd = subprocess.Popen([spi_registryd_path], env=self._test_env)
173         except:
174             sys.stderr.write("Failed to launch the accessibility registry\n")
175             sys.stderr.flush()
176             return False
177
178         return True
179
180     def _run_xvfb(self):
181         self._xvfb = None
182         if not self._options.use_xvfb:
183             return True
184
185         self._test_env["DISPLAY"] = self._options.display
186
187         try:
188             self._xvfb = subprocess.Popen(["Xvfb", self._options.display, "-screen", "0", "800x600x24", "-nolisten", "tcp"],
189                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
190         except Exception as e:
191             sys.stderr.write("Failed to run Xvfb: %s\n" % e)
192             sys.stderr.flush()
193             return False
194
195         return True
196
197     def _setup_testing_environment(self):
198         self._test_env = os.environ
199         self._test_env['GSETTINGS_BACKEND'] = 'memory'
200         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit2")
201         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.library_build_path()
202         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
203         self._test_env["OWR_USE_TEST_SOURCES"] = '1'
204
205         if not self._run_xvfb():
206             return False
207
208         # If we cannot start the accessibility daemons, we can just skip the accessibility tests.
209         if not self._start_accessibility_daemons():
210             print "Could not start accessibility bus, so disabling TestWebKitAccessibility"
211             self._disabled_tests.append("WebKit2APITests/TestWebKitAccessibility")
212         return True
213
214     def _tear_down_testing_environment(self):
215         if self._spi_registryd:
216             self._spi_registryd.terminate()
217         if self._spi_bus_launcher:
218             self._spi_bus_launcher.terminate()
219         if self._xvfb:
220             self._xvfb.terminate()
221
222     def _test_cases_to_skip(self, test_program):
223         if self._options.skipped_action != 'skip':
224             return []
225
226         test_cases = []
227         for skipped in self._skipped_tests:
228             if test_program.endswith(skipped.test) and not skipped.skip_entire_suite():
229                 test_cases.append(skipped.test_case)
230         return test_cases
231
232     def _should_run_test_program(self, test_program):
233         for disabled_test in self._disabled_tests:
234             if test_program.endswith(disabled_test):
235                 return False
236
237         if self._options.skipped_action != 'skip':
238             return True
239
240         for skipped in self._skipped_tests:
241             if test_program.endswith(skipped.test) and skipped.skip_entire_suite():
242                 return False
243         return True
244
245     def _get_child_pid_from_test_output(self, output):
246         if not output:
247             return -1
248         match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', output)
249         if not match:
250             return -1
251         return int(match.group('child_pid'))
252
253     def _kill_process(self, pid):
254         try:
255             os.kill(pid, SIGKILL)
256         except OSError:
257             # Process already died.
258             pass
259
260     def _run_test_command(self, command, timeout=-1):
261         def alarm_handler(signum, frame):
262             raise TestTimeout
263
264         child_pid = [-1]
265         def parse_line(line, child_pid = child_pid):
266             if child_pid[0] == -1:
267                 child_pid[0] = self._get_child_pid_from_test_output(line)
268
269             sys.stdout.write(line)
270
271         def waitpid(pid):
272             while True:
273                 try:
274                     return os.waitpid(pid, 0)
275                 except (OSError, IOError) as e:
276                     if e.errno == errno.EINTR:
277                         continue
278                     raise
279
280         def return_code_from_exit_status(status):
281             if os.WIFSIGNALED(status):
282                 return -os.WTERMSIG(status)
283             elif os.WIFEXITED(status):
284                 return os.WEXITSTATUS(status)
285             else:
286                 # Should never happen
287                 raise RuntimeError("Unknown child exit status!")
288
289         pid, fd = os.forkpty()
290         if pid == 0:
291             os.execvpe(command[0], command, self._test_env)
292             sys.exit(0)
293
294         if timeout > 0:
295             signal(SIGALRM, alarm_handler)
296             alarm(timeout)
297
298         try:
299             common.parse_output_lines(fd, parse_line)
300             if timeout > 0:
301                 alarm(0)
302         except TestTimeout:
303             self._kill_process(pid)
304             if child_pid[0] > 0:
305                 self._kill_process(child_pid[0])
306             raise
307
308         try:
309             dummy, status = waitpid(pid)
310         except OSError as e:
311             if e.errno != errno.ECHILD:
312                 raise
313             # This happens if SIGCLD is set to be ignored or waiting
314             # for child processes has otherwise been disabled for our
315             # process.  This child is dead, we can't get the status.
316             status = 0
317
318         return return_code_from_exit_status(status)
319
320     def _run_test_glib(self, test_program):
321         tester_command = ['gtester', '-k']
322         if self._options.verbose:
323             tester_command.append('--verbose')
324         for test_case in self._test_cases_to_skip(test_program):
325             tester_command.extend(['-s', test_case])
326         tester_command.append(test_program)
327
328         return self._run_test_command(tester_command, self._options.timeout)
329
330     def _get_tests_from_google_test_suite(self, test_program):
331         try:
332             output = subprocess.check_output([test_program, '--gtest_list_tests'])
333         except subprocess.CalledProcessError:
334             sys.stderr.write("ERROR: could not list available tests for binary %s.\n" % (test_program))
335             sys.stderr.flush()
336             return 1
337
338         skipped_test_cases = self._test_cases_to_skip(test_program)
339
340         tests = []
341         prefix = None
342         for line in output.split('\n'):
343             if not line.startswith('  '):
344                 prefix = line
345                 continue
346             else:
347                 test_name = prefix + line.strip()
348                 if not test_name in skipped_test_cases:
349                     tests.append(test_name)
350         return tests
351
352     def _run_google_test(self, test_program, subtest):
353         test_command = [test_program, '--gtest_filter=%s' % (subtest)]
354         timeout = self._options.timeout
355         if subtest in TestRunner.SLOW:
356             timeout *= 5
357
358         status = self._run_test_command(test_command, timeout)
359         if status == -SIGSEGV:
360             sys.stdout.write("**CRASH** %s\n" % subtest)
361             sys.stdout.flush()
362         return status
363
364     def _run_google_test_suite(self, test_program):
365         retcode = 0
366         for subtest in self._get_tests_from_google_test_suite(test_program):
367             if self._run_google_test(test_program, subtest):
368                 retcode = 1
369         return retcode
370
371     def _run_test(self, test_program):
372         basedir = os.path.basename(os.path.dirname(test_program))
373         if basedir in ["WebKit2Gtk", "WebKitGtk"]:
374             return self._run_test_glib(test_program)
375
376         if basedir in ["WebKit2", "JavaScriptCore", "WTF", "WebCore",  "WebCoreGtk"]:
377             return self._run_google_test_suite(test_program)
378
379         return 1
380
381     def run_tests(self):
382         if not self._tests:
383             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._test_programs_base_dir()))
384             sys.stderr.flush()
385             return 1
386
387         if not self._setup_testing_environment():
388             return 1
389
390         # Remove skipped tests now instead of when we find them, because
391         # some tests might be skipped while setting up the test environment.
392         self._tests = [test for test in self._tests if self._should_run_test_program(test)]
393
394         crashed_tests = []
395         failed_tests = []
396         timed_out_tests = []
397         try:
398             for test in self._tests:
399                 exit_status_code = 0
400                 try:
401                     exit_status_code = self._run_test(test)
402                 except TestTimeout:
403                     sys.stdout.write("TEST: %s: TIMEOUT\n" % test)
404                     sys.stdout.flush()
405                     timed_out_tests.append(test)
406
407                 if exit_status_code == -SIGSEGV:
408                     sys.stdout.write("TEST: %s: CRASHED\n" % test)
409                     sys.stdout.flush()
410                     crashed_tests.append(test)
411                 elif exit_status_code != 0:
412                     failed_tests.append(test)
413         finally:
414             self._tear_down_testing_environment()
415
416         if failed_tests:
417             names = [test.replace(self._test_programs_base_dir(), '', 1) for test in failed_tests]
418             sys.stdout.write("Tests failed (%d): %s\n" % (len(names), ", ".join(names)))
419             sys.stdout.flush()
420
421         if crashed_tests:
422             names = [test.replace(self._test_programs_base_dir(), '', 1) for test in crashed_tests]
423             sys.stdout.write("Tests that crashed (%d): %s\n" % (len(names), ", ".join(names)))
424             sys.stdout.flush()
425
426         if timed_out_tests:
427             names = [test.replace(self._test_programs_base_dir(), '', 1) for test in timed_out_tests]
428             sys.stdout.write("Tests that timed out (%d): %s\n" % (len(names), ", ".join(names)))
429             sys.stdout.flush()
430
431         if self._skipped_tests and self._options.skipped_action == 'skip':
432             sys.stdout.write("Tests skipped (%d):\n%s\n" %
433                              (len(self._skipped_tests),
434                               "\n".join([str(skipped) for skipped in self._skipped_tests])))
435             sys.stdout.flush()
436
437         return len(failed_tests) + len(timed_out_tests)
438
439 if __name__ == "__main__":
440     if not jhbuildutils.enter_jhbuild_environment_if_available("gtk"):
441         print "***"
442         print "*** Warning: jhbuild environment not present. Run update-webkitgtk-libs before build-webkit to ensure proper testing."
443         print "***"
444
445     option_parser = optparse.OptionParser(usage='usage: %prog [options] [test...]')
446     option_parser.add_option('-r', '--release',
447                              action='store_true', dest='release',
448                              help='Run in Release')
449     option_parser.add_option('-d', '--debug',
450                              action='store_true', dest='debug',
451                              help='Run in Debug')
452     option_parser.add_option('-v', '--verbose',
453                              action='store_true', dest='verbose',
454                              help='Run gtester in verbose mode')
455     option_parser.add_option('--display', action='store', dest='display', default=':55',
456                              help='Display to run Xvfb')
457     option_parser.add_option('--skipped', action='store', dest='skipped_action',
458                              choices=['skip', 'ignore', 'only'], default='skip',
459                              metavar='skip|ignore|only',
460                              help='Specifies how to treat the skipped tests')
461     option_parser.add_option('-t', '--timeout',
462                              action='store', type='int', dest='timeout', default=10,
463                              help='Time in seconds until a test times out')
464     option_parser.add_option('--no-xvfb', action='store_false', dest='use_xvfb', default=True,
465                              help='Do not run tests under Xvfb')
466     options, args = option_parser.parse_args()
467
468     sys.exit(TestRunner(options, args).run_tests())