[GTK] Test /webkit/atk/getTextInParagraphAndBodyModerate fails
[WebKit-https.git] / Tools / Scripts / run-gtk-tests
1 #!/usr/bin/env python
2 #
3 # Copyright (C) 2011, 2012 Igalia S.L.
4 #
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Library General Public
7 # License as published by the Free Software Foundation; either
8 # version 2 of the License, or (at your option) any later version.
9 #
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 # Library General Public License for more details.
14 #
15 # You should have received a copy of the GNU Library General Public License
16 # along with this library; see the file COPYING.LIB.  If not, write to
17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 # Boston, MA 02110-1301, USA.
19
20 import subprocess
21 import os
22 import sys
23 import optparse
24 import re
25 from signal import alarm, signal, SIGALRM, SIGKILL
26 from gi.repository import Gio, GLib
27
28 top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
29 sys.path.append(os.path.join(top_level_directory, "Tools", "jhbuild"))
30 sys.path.append(os.path.join(top_level_directory, "Tools", "gtk"))
31 import common
32 import jhbuildutils
33
34 class SkippedTest:
35     ENTIRE_SUITE = None
36
37     def __init__(self, test, test_case, reason, bug=None):
38         self.test = test
39         self.test_case = test_case
40         self.reason = reason
41         self.bug = bug
42
43     def __str__(self):
44         skipped_test_str = "%s" % self.test
45
46         if not(self.skip_entire_suite()):
47             skipped_test_str += " [%s]" % self.test_case
48
49         skipped_test_str += ": %s " % self.reason
50         if self.bug is not None:
51             skipped_test_str += "(https://bugs.webkit.org/show_bug.cgi?id=%d)" % self.bug
52         return skipped_test_str
53
54     def skip_entire_suite(self):
55         return self.test_case == SkippedTest.ENTIRE_SUITE
56
57 class TestTimeout(Exception):
58     pass
59
60 class TestRunner:
61     TEST_DIRS = [ "unittests", "WebKit2APITests", "TestWebKitAPI" ]
62
63     SKIPPED = [
64         SkippedTest("unittests/testdownload", "/webkit/download/not-found", "Test fails in GTK Linux 64-bit Release bot", 82329),
65         SkippedTest("unittests/testwebinspector", "/webkit/webinspector/close-and-inspect", "Test is flaky in GTK Linux 32-bit Release bot", 82869),
66         SkippedTest("unittests/testwebresource", "/webkit/webresource/loading", "Test fails", 104689),
67         SkippedTest("unittests/testwebresource", "/webkit/webresource/sub_resource_loading", "Test fails in GTK Linux 64-bit Release bot", 82330),
68         SkippedTest("unittests/testwebview", "/webkit/webview/icon-uri", "Test times out in GTK Linux 64-bit Release bot", 82328),
69         SkippedTest("WebKit2APITests/TestResources", "/webkit2/WebKitWebView/resources", "Test is flaky in GTK Linux 32-bit Release bot", 82868),
70         SkippedTest("WebKit2APITests/TestWebKitWebView", SkippedTest.ENTIRE_SUITE, "Test times out after r150890", 117689),
71         SkippedTest("WebKit2APITests/TestContextMenu", SkippedTest.ENTIRE_SUITE, "Test times out after r150890", 117689),
72         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.CanHandleRequest", "Test fails", 88453),
73         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.MouseMoveAfterCrash", "Test is flaky", 85066),
74         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutForImages", "Test is flaky", 85066),
75         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutFrames", "Test fails", 85037),
76         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.RestoreSessionStateContainingFormData", "Session State is not implemented in GTK+ port", 84960),
77         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.SpacebarScrolling", "Test fails", 84961),
78         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.WKConnection", "Tests fail and time out out", 84959),
79         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.WKPageGetScaleFactorNotZero", "Test fails and times out", 88455),
80         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.ForceRepaint", "Test times out", 105532),
81         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.ReloadPageAfterCrash", "Test flakily times out", 110129),
82         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.DidAssociateFormControls", "Test times out", 120302),
83         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.InjectedBundleFrameHitTest", "Test times out", 120303),
84         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.ResizeReversePaginatedWebView", "Test fails", 120305),
85         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.ScrollPinningBehaviors", "Test fails", 120306),
86     ]
87
88     def __init__(self, options, tests=[]):
89         self._options = options
90         self._build_type = "Debug" if self._options.debug else "Release"
91
92         self._programs_path = common.build_path_for_build_types((self._build_type,), "Programs")
93         self._tests = self._get_tests(tests)
94         self._skipped_tests = TestRunner.SKIPPED
95         if not sys.stdout.isatty():
96             self._tty_colors_pattern = re.compile("\033\[[0-9;]*m")
97
98         # These SPI daemons need to be active for the accessibility tests to work.
99         self._spi_registryd = None
100         self._spi_bus_launcher = None
101
102     def _get_tests_from_dir(self, test_dir):
103         if not os.path.isdir(test_dir):
104             return []
105
106         tests = []
107         for test_file in os.listdir(test_dir):
108             if not test_file.lower().startswith("test"):
109                 continue
110             test_path = os.path.join(test_dir, test_file)
111             if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
112                 tests.append(test_path)
113         return tests
114
115     def _get_tests(self, initial_tests):
116         tests = []
117         for test in initial_tests:
118             if os.path.isdir(test):
119                 tests.extend(self._get_tests_from_dir(test))
120             else:
121                 tests.append(test)
122         if tests:
123             return tests
124
125         tests = []
126         for test_dir in self.TEST_DIRS:
127             absolute_test_dir = os.path.join(self._programs_path, test_dir)
128             tests.extend(self._get_tests_from_dir(absolute_test_dir))
129         return tests
130
131     def _lookup_atspi2_binary(self, filename):
132         exec_prefix = common.pkg_config_file_variable('atspi-2', 'exec_prefix')
133         if not exec_prefix:
134             return None
135         for path in ['libexec', 'lib/at-spi2-core', 'lib32/at-spi2-core', 'lib64/at-spi2-core']:
136             filepath = os.path.join(exec_prefix, path, filename)
137             if os.path.isfile(filepath):
138                 return filepath
139
140         return None
141
142     def _start_accessibility_daemons(self):
143         spi_bus_launcher_path = self._lookup_atspi2_binary('at-spi-bus-launcher')
144         spi_registryd_path = self._lookup_atspi2_binary('at-spi2-registryd')
145         if not spi_bus_launcher_path or not spi_registryd_path:
146             return False
147
148         try:
149             self._ally_bus_launcher = subprocess.Popen([spi_bus_launcher_path], env=self._test_env)
150         except:
151             sys.stderr.write("Failed to launch the accessibility bus\n")
152             sys.stderr.flush()
153             return False
154
155         # We need to wait until the SPI bus is launched before trying to start the SPI
156         # registry, so we spin a main loop until the bus name appears on DBus.
157         loop = GLib.MainLoop()
158         Gio.bus_watch_name(Gio.BusType.SESSION, 'org.a11y.Bus', Gio.BusNameWatcherFlags.NONE,
159                            lambda *args: loop.quit(), None)
160         loop.run()
161
162         try:
163             self._spi_registryd = subprocess.Popen([spi_registryd_path], env=self._test_env)
164         except:
165             sys.stderr.write("Failed to launch the accessibility registry\n")
166             sys.stderr.flush()
167             return False
168
169         return True
170
171     def _run_xvfb(self):
172         self._xvfb = None
173         if not self._options.use_xvfb:
174             return True
175
176         self._test_env["DISPLAY"] = self._options.display
177
178         try:
179             self._xvfb = subprocess.Popen(["Xvfb", self._options.display, "-screen", "0", "800x600x24", "-nolisten", "tcp"],
180                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
181         except Exception as e:
182             sys.stderr.write("Failed to run Xvfb: %s\n" % e)
183             sys.stderr.flush()
184             return False
185
186         return True
187
188     def _setup_testing_environment(self):
189         self._test_env = os.environ
190         self._test_env["WEBKIT_INSPECTOR_PATH"] = os.path.abspath(os.path.join(self._programs_path, 'resources', 'inspector'))
191         self._test_env['GSETTINGS_BACKEND'] = 'memory'
192         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit2")
193         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.build_path_for_build_types((self._build_type,), "Libraries")
194         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
195
196         if not self._run_xvfb():
197             return False
198
199         # If we cannot start the accessibility daemons, we can just skip the accessibility tests.
200         if not self._start_accessibility_daemons():
201             print "Could not start accessibility bus, so skipping TestWebKitAccessibility"
202             self._skipped_tests.append(SkippedTest("WebKit2APITests/TestWebKitAccessibility", SkippedTest.ENTIRE_SUITE, "Could not start accessibility bus"))
203         return True
204
205     def _tear_down_testing_environment(self):
206         if self._spi_registryd:
207             self._spi_registryd.terminate()
208         if self._spi_bus_launcher:
209             self._spi_bus_launcher.terminate()
210         if self._xvfb:
211             self._xvfb.terminate()
212
213     def _test_cases_to_skip(self, test_program):
214         if self._options.skipped_action != 'skip':
215             return []
216
217         test_cases = []
218         for skipped in self._skipped_tests:
219             if test_program.endswith(skipped.test) and not skipped.skip_entire_suite():
220                 test_cases.append(skipped.test_case)
221         return test_cases
222
223     def _should_run_test_program(self, test_program):
224         # This is not affected by the command-line arguments, since programs are skipped for
225         # problems in the harness, such as failing to start the accessibility bus.
226         for skipped in self._skipped_tests:
227             if test_program.endswith(skipped.test) and skipped.skip_entire_suite():
228                 return False
229         return True
230
231     def _get_child_pid_from_test_output(self, output):
232         if not output:
233             return -1
234         match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', output)
235         if not match:
236             return -1
237         return int(match.group('child_pid'))
238
239     def _kill_process(self, pid):
240         try:
241             os.kill(pid, SIGKILL)
242         except OSError:
243             # Process already died.
244             pass
245
246     def _run_test_command(self, command, timeout=-1):
247         def alarm_handler(signum, frame):
248             raise TestTimeout
249
250         child_pid = [-1]
251         def parse_line(line, child_pid = child_pid):
252             if child_pid[0] == -1:
253                 child_pid[0] = self._get_child_pid_from_test_output(line)
254
255             if sys.stdout.isatty():
256                 sys.stdout.write(line)
257             else:
258                 sys.stdout.write(self._tty_colors_pattern.sub('', line.replace('\r', '')))
259
260         def waitpid(pid):
261             while True:
262                 try:
263                     return os.waitpid(pid, 0)
264                 except (OSError, IOError) as e:
265                     if e.errno == errno.EINTR:
266                         continue
267                     raise
268
269         def return_code_from_exit_status(status):
270             if os.WIFSIGNALED(status):
271                 return -os.WTERMSIG(status)
272             elif os.WIFEXITED(status):
273                 return os.WEXITSTATUS(status)
274             else:
275                 # Should never happen
276                 raise RuntimeError("Unknown child exit status!")
277
278         pid, fd = os.forkpty()
279         if pid == 0:
280             os.execvpe(command[0], command, self._test_env)
281             sys.exit(0)
282
283         if timeout > 0:
284             signal(SIGALRM, alarm_handler)
285             alarm(timeout)
286
287         try:
288             common.parse_output_lines(fd, parse_line)
289             if timeout > 0:
290                 alarm(0)
291         except TestTimeout:
292             self._kill_process(pid)
293             if child_pid[0] > 0:
294                 self._kill_process(child_pid[0])
295             raise
296
297         try:
298             dummy, status = waitpid(pid)
299         except OSError as e:
300             if e.errno != errno.ECHILD:
301                 raise
302             # This happens if SIGCLD is set to be ignored or waiting
303             # for child processes has otherwise been disabled for our
304             # process.  This child is dead, we can't get the status.
305             status = 0
306
307         return not return_code_from_exit_status(status)
308
309     def _run_test_glib(self, test_program):
310         tester_command = ['gtester']
311         if self._options.verbose:
312             tester_command.append('--verbose')
313         for test_case in self._test_cases_to_skip(test_program):
314             tester_command.extend(['-s', test_case])
315         tester_command.append(test_program)
316
317         return self._run_test_command(tester_command, self._options.timeout)
318
319     def _run_test_google(self, test_program):
320         tester_command = [test_program]
321         skipped_tests_cases = self._test_cases_to_skip(test_program)
322         if skipped_tests_cases:
323             tester_command.append("--gtest_filter=-%s" % ":".join(skipped_tests_cases))
324
325         return self._run_test_command(tester_command, self._options.timeout)
326
327     def _run_test(self, test_program):
328         if "unittests" in test_program or "WebKit2APITests" in test_program:
329             return self._run_test_glib(test_program)
330
331         if "TestWebKitAPI" in test_program:
332             return self._run_test_google(test_program)
333
334         return False
335
336     def run_tests(self):
337         if not self._tests:
338             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._programs_path))
339             sys.stderr.flush()
340             return 1
341
342         if not self._setup_testing_environment():
343             return 1
344
345         # Remove skipped tests now instead of when we find them, because
346         # some tests might be skipped while setting up the test environment.
347         self._tests = [test for test in self._tests if self._should_run_test_program(test)]
348
349         failed_tests = []
350         timed_out_tests = []
351         try:
352             for test in self._tests:
353                 success = True
354                 try:
355                     success = self._run_test(test)
356                 except TestTimeout:
357                     sys.stdout.write("TEST: %s: TIMEOUT\n" % test)
358                     sys.stdout.flush()
359                     timed_out_tests.append(test)
360
361                 if not success:
362                     failed_tests.append(test)
363         finally:
364             self._tear_down_testing_environment()
365
366         if failed_tests:
367             names = [test.replace(self._programs_path, '', 1) for test in failed_tests]
368             sys.stdout.write("Tests failed (%d): %s\n" % (len(names), ", ".join(names)))
369             sys.stdout.flush()
370
371         if timed_out_tests:
372             names = [test.replace(self._programs_path, '', 1) for test in timed_out_tests]
373             sys.stdout.write("Tests that timed out (%d): %s\n" % (len(names), ", ".join(names)))
374             sys.stdout.flush()
375
376         if self._skipped_tests and self._options.skipped_action == 'skip':
377             sys.stdout.write("Tests skipped (%d):\n%s\n" %
378                              (len(self._skipped_tests),
379                               "\n".join([str(skipped) for skipped in self._skipped_tests])))
380             sys.stdout.flush()
381
382         return len(failed_tests) + len(timed_out_tests)
383
384 if __name__ == "__main__":
385     if not jhbuildutils.enter_jhbuild_environment_if_available("gtk"):
386         print "***"
387         print "*** Warning: jhbuild environment not present. Run update-webkitgtk-libs before build-webkit to ensure proper testing."
388         print "***"
389
390     option_parser = optparse.OptionParser(usage='usage: %prog [options] [test...]')
391     option_parser.add_option('-r', '--release',
392                              action='store_true', dest='release',
393                              help='Run in Release')
394     option_parser.add_option('-d', '--debug',
395                              action='store_true', dest='debug',
396                              help='Run in Debug')
397     option_parser.add_option('-v', '--verbose',
398                              action='store_true', dest='verbose',
399                              help='Run gtester in verbose mode')
400     option_parser.add_option('--display', action='store', dest='display', default=':55',
401                              help='Display to run Xvfb')
402     option_parser.add_option('--skipped', action='store', dest='skipped_action',
403                              choices=['skip', 'ignore', 'only'], default='skip',
404                              metavar='skip|ignore|only',
405                              help='Specifies how to treat the skipped tests')
406     option_parser.add_option('-t', '--timeout',
407                              action='store', type='int', dest='timeout', default=10,
408                              help='Time in seconds until a test times out')
409     option_parser.add_option('--no-xvfb', action='store_false', dest='use_xvfb', default=True,
410                              help='Do not run tests under Xvfb')
411     options, args = option_parser.parse_args()
412
413     sys.exit(TestRunner(options, args).run_tests())