[WPE] Add run-wpe-tests script to run WPE glib API tests
[WebKit-https.git] / Tools / glib / api_test_runner.py
1 #!/usr/bin/env python
2 #
3 # Copyright (C) 2011, 2012, 2017 Igalia S.L.
4 #
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Library General Public
7 # License as published by the Free Software Foundation; either
8 # version 2 of the License, or (at your option) any later version.
9 #
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 # Library General Public License for more details.
14 #
15 # You should have received a copy of the GNU Library General Public License
16 # along with this library; see the file COPYING.LIB.  If not, write to
17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 # Boston, MA 02110-1301, USA.
19
20 import subprocess
21 import os
22 import errno
23 import sys
24 import re
25 from signal import alarm, signal, SIGALRM, SIGKILL, SIGSEGV
26
27 top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
28 sys.path.insert(0, os.path.join(top_level_directory, "Tools", "glib"))
29 import common
30 from webkitpy.common.host import Host
31
32
33 class SkippedTest:
34     ENTIRE_SUITE = None
35
36     def __init__(self, test, test_case, reason, bug, build_type=None):
37         self.test = test
38         self.test_case = test_case
39         self.reason = reason
40         self.bug = bug
41         self.build_type = build_type
42
43     def __str__(self):
44         skipped_test_str = "%s" % self.test
45
46         if not(self.skip_entire_suite()):
47             skipped_test_str += " [%s]" % self.test_case
48
49         skipped_test_str += ": %s (https://bugs.webkit.org/show_bug.cgi?id=%d)" % (self.reason, self.bug)
50         return skipped_test_str
51
52     def skip_entire_suite(self):
53         return self.test_case == SkippedTest.ENTIRE_SUITE
54
55     def skip_for_build_type(self, build_type):
56         if self.build_type is None:
57             return True
58
59         return self.build_type == build_type
60
61
62 class TestTimeout(Exception):
63     pass
64
65
66 class TestRunner(object):
67     TEST_DIRS = []
68     SKIPPED = []
69     SLOW = []
70
71     def __init__(self, port, options, tests=[]):
72         self._options = options
73
74         self._build_type = "Debug" if self._options.debug else "Release"
75         common.set_build_types((self._build_type,))
76         self._port = Host().port_factory.get(port)
77         self._driver = self._create_driver()
78
79         self._programs_path = common.binary_build_path()
80         self._tests = self._get_tests(tests)
81         self._skipped_tests = [skipped for skipped in TestRunner.SKIPPED if skipped.skip_for_build_type(self._build_type)]
82         self._disabled_tests = []
83
84     def _test_programs_base_dir(self):
85         return os.path.join(self._programs_path, "TestWebKitAPI")
86
87     def _get_tests_from_dir(self, test_dir):
88         if not os.path.isdir(test_dir):
89             return []
90
91         tests = []
92         for test_file in os.listdir(test_dir):
93             if not test_file.lower().startswith("test"):
94                 continue
95             test_path = os.path.join(test_dir, test_file)
96             if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
97                 tests.append(test_path)
98         return tests
99
100     def _get_tests(self, initial_tests):
101         tests = []
102         for test in initial_tests:
103             if os.path.isdir(test):
104                 tests.extend(self._get_tests_from_dir(test))
105             else:
106                 tests.append(test)
107         if tests:
108             return tests
109
110         tests = []
111         for test_dir in self.TEST_DIRS:
112             absolute_test_dir = os.path.join(self._test_programs_base_dir(), test_dir)
113             tests.extend(self._get_tests_from_dir(absolute_test_dir))
114         return tests
115
116     def _create_driver(self, port_options=[]):
117         self._port._display_server = self._options.display_server
118         driver = self._port.create_driver(worker_number=0, no_timeout=True)._make_driver(pixel_tests=False)
119         if not driver.check_driver(self._port):
120             raise RuntimeError("Failed to check driver %s" % driver.__class__.__name__)
121         return driver
122
123     def _setup_testing_environment(self):
124         self._test_env = self._driver._setup_environ_for_test()
125         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit")
126         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.library_build_path()
127         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
128
129         return True
130
131     def _tear_down_testing_environment(self):
132         if self._driver:
133             self._driver.stop()
134
135     def _test_cases_to_skip(self, test_program):
136         if self._options.skipped_action != 'skip':
137             return []
138
139         test_cases = []
140         for skipped in self._skipped_tests:
141             if test_program.endswith(skipped.test) and not skipped.skip_entire_suite():
142                 test_cases.append(skipped.test_case)
143         return test_cases
144
145     def _should_run_test_program(self, test_program):
146         for disabled_test in self._disabled_tests:
147             if test_program.endswith(disabled_test):
148                 return False
149
150         if self._options.skipped_action != 'skip':
151             return True
152
153         for skipped in self._skipped_tests:
154             if test_program.endswith(skipped.test) and skipped.skip_entire_suite():
155                 return False
156         return True
157
158     def _kill_process(self, pid):
159         try:
160             os.kill(pid, SIGKILL)
161         except OSError:
162             # Process already died.
163             pass
164
165     @staticmethod
166     def _start_timeout(timeout):
167         if timeout <= 0:
168             return
169
170         def _alarm_handler(signum, frame):
171             raise TestTimeout
172
173         signal(SIGALRM, _alarm_handler)
174         alarm(timeout)
175
176     @staticmethod
177     def _stop_timeout(timeout):
178         if timeout <= 0:
179             return
180
181         alarm(0)
182
183     def _waitpid(self, pid):
184         while True:
185             try:
186                 dummy, status = os.waitpid(pid, 0)
187                 if os.WIFSIGNALED(status):
188                     return -os.WTERMSIG(status)
189                 if os.WIFEXITED(status):
190                     return os.WEXITSTATUS(status)
191
192                 # Should never happen
193                 raise RuntimeError("Unknown child exit status!")
194             except (OSError, IOError) as e:
195                 if e.errno == errno.EINTR:
196                     continue
197                 if e.errno == errno.ECHILD:
198                     # This happens if SIGCLD is set to be ignored or waiting
199                     # for child processes has otherwise been disabled for our
200                     # process.  This child is dead, we can't get the status.
201                     return 0
202                 raise
203
204     def _run_test_glib(self, test_program):
205         command = ['gtester', '-k']
206         if self._options.verbose:
207             command.append('--verbose')
208         for test_case in self._test_cases_to_skip(test_program):
209             command.extend(['-s', test_case])
210         command.append(test_program)
211
212         timeout = self._options.timeout
213         test = os.path.join(os.path.basename(os.path.dirname(test_program)), os.path.basename(test_program))
214         if test in TestRunner.SLOW:
215             timeout *= 5
216
217         test_context = {"child-pid": -1, "did-timeout": False, "current_test": None}
218
219         def parse_line(line, test_context=test_context):
220             if not line:
221                 return
222
223             match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', line)
224             if match:
225                 test_context["child-pid"] = int(match.group('child_pid'))
226                 sys.stdout.write(line)
227                 return
228
229             def set_test_result(test, result):
230                 if result == "FAIL":
231                     if test_context["did-timeout"] and result == "FAIL":
232                         test_context[test] = "TIMEOUT"
233                     else:
234                         test_context[test] = result
235                 test_context["did-timeout"] = False
236                 test_context["current_test"] = None
237                 self._stop_timeout(timeout)
238                 self._start_timeout(timeout)
239
240             normalized_line = line.strip().replace(' ', '')
241             if not normalized_line:
242                 return
243
244             if normalized_line[0] == '/':
245                 test, result = normalized_line.split(':', 1)
246                 if result in ["OK", "FAIL"]:
247                     set_test_result(test, result)
248                 else:
249                     test_context["current_test"] = test
250             elif normalized_line in ["OK", "FAIL"]:
251                 set_test_result(test_context["current_test"], normalized_line)
252
253             sys.stdout.write(line)
254
255         pid, fd = os.forkpty()
256         if pid == 0:
257             os.execvpe(command[0], command, self._test_env)
258             sys.exit(0)
259
260         self._start_timeout(timeout)
261
262         while (True):
263             try:
264                 common.parse_output_lines(fd, parse_line)
265                 break
266             except TestTimeout:
267                 assert test_context["child-pid"] > 0
268                 self._kill_process(test_context["child-pid"])
269                 test_context["child-pid"] = -1
270                 test_context["did-timeout"] = True
271
272         self._stop_timeout(timeout)
273         del test_context["child-pid"]
274         del test_context["did-timeout"]
275         del test_context["current_test"]
276
277         self._waitpid(pid)
278         return test_context
279
280     def _get_tests_from_google_test_suite(self, test_program):
281         try:
282             output = subprocess.check_output([test_program, '--gtest_list_tests'], env=self._test_env)
283         except subprocess.CalledProcessError:
284             sys.stderr.write("ERROR: could not list available tests for binary %s.\n" % (test_program))
285             sys.stderr.flush()
286             return 1
287
288         skipped_test_cases = self._test_cases_to_skip(test_program)
289
290         tests = []
291         prefix = None
292         for line in output.split('\n'):
293             if not line.startswith('  '):
294                 prefix = line
295                 continue
296             else:
297                 test_name = prefix + line.strip()
298                 if not test_name in skipped_test_cases:
299                     tests.append(test_name)
300         return tests
301
302     def _run_google_test(self, test_program, subtest):
303         command = [test_program, '--gtest_filter=%s' % (subtest)]
304         timeout = self._options.timeout
305         if subtest in TestRunner.SLOW:
306             timeout *= 5
307
308         pid, fd = os.forkpty()
309         if pid == 0:
310             os.execvpe(command[0], command, self._test_env)
311             sys.exit(0)
312
313         self._start_timeout(timeout)
314         try:
315             common.parse_output_lines(fd, sys.stdout.write)
316             status = self._waitpid(pid)
317         except TestTimeout:
318             self._kill_process(pid)
319             return {subtest: "TIMEOUT"}
320
321         self._stop_timeout(timeout)
322
323         if status == -SIGSEGV:
324             sys.stdout.write("**CRASH** %s\n" % subtest)
325             sys.stdout.flush()
326             return {subtest: "CRASH"}
327
328         if status != 0:
329             return {subtest: "FAIL"}
330
331         return {}
332
333     def _run_google_test_suite(self, test_program):
334         result = {}
335         for subtest in self._get_tests_from_google_test_suite(test_program):
336             result.update(self._run_google_test(test_program, subtest))
337         return result
338
339     def is_glib_test(self, test_program):
340         raise NotImplementedError
341
342     def is_google_test(self, test_program):
343         raise NotImplementedError
344
345     def _run_test(self, test_program):
346         if self.is_glib_test(test_program):
347             return self._run_test_glib(test_program)
348
349         if self.is_google_test(test_program):
350             return self._run_google_test_suite(test_program)
351
352         return {}
353
354     def run_tests(self):
355         if not self._tests:
356             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._test_programs_base_dir()))
357             sys.stderr.flush()
358             return 1
359
360         if not self._setup_testing_environment():
361             return 1
362
363         # Remove skipped tests now instead of when we find them, because
364         # some tests might be skipped while setting up the test environment.
365         self._tests = [test for test in self._tests if self._should_run_test_program(test)]
366
367         crashed_tests = {}
368         failed_tests = {}
369         timed_out_tests = {}
370         try:
371             for test in self._tests:
372                 results = self._run_test(test)
373                 for test_case, result in results.iteritems():
374                     if result == "FAIL":
375                         failed_tests.setdefault(test, []).append(test_case)
376                     elif result == "TIMEOUT":
377                         timed_out_tests.setdefault(test, []).append(test_case)
378                     elif result == "CRASH":
379                         crashed_tests.setdefault(test, []).append(test_case)
380         finally:
381             self._tear_down_testing_environment()
382
383         if failed_tests:
384             sys.stdout.write("\nUnexpected failures (%d)\n" % (sum(len(value) for value in failed_tests.itervalues())))
385             for test in failed_tests:
386                 sys.stdout.write("    %s\n" % (test.replace(self._test_programs_base_dir(), '', 1)))
387                 for test_case in failed_tests[test]:
388                     sys.stdout.write("        %s\n" % (test_case))
389             sys.stdout.flush()
390
391         if crashed_tests:
392             sys.stdout.write("\nUnexpected crashes (%d)\n" % (sum(len(value) for value in crashed_tests.itervalues())))
393             for test in crashed_tests:
394                 sys.stdout.write("    %s\n" % (test.replace(self._test_programs_base_dir(), '', 1)))
395                 for test_case in crashed_tests[test]:
396                     sys.stdout.write("        %s\n" % (test_case))
397             sys.stdout.flush()
398
399         if timed_out_tests:
400             sys.stdout.write("\nUnexpected timeouts (%d)\n" % (sum(len(value) for value in timed_out_tests.itervalues())))
401             for test in timed_out_tests:
402                 sys.stdout.write("    %s\n" % (test.replace(self._test_programs_base_dir(), '', 1)))
403                 for test_case in timed_out_tests[test]:
404                     sys.stdout.write("        %s\n" % (test_case))
405             sys.stdout.flush()
406
407         return len(failed_tests) + len(timed_out_tests)
408
409
410 def add_options(option_parser):
411     option_parser.add_option('-r', '--release',
412                              action='store_true', dest='release',
413                              help='Run in Release')
414     option_parser.add_option('-d', '--debug',
415                              action='store_true', dest='debug',
416                              help='Run in Debug')
417     option_parser.add_option('-v', '--verbose',
418                              action='store_true', dest='verbose',
419                              help='Run gtester in verbose mode')
420     option_parser.add_option('--skipped', action='store', dest='skipped_action',
421                              choices=['skip', 'ignore', 'only'], default='skip',
422                              metavar='skip|ignore|only',
423                              help='Specifies how to treat the skipped tests')
424     option_parser.add_option('-t', '--timeout',
425                              action='store', type='int', dest='timeout', default=10,
426                              help='Time in seconds until a test times out')