[GTK][WPE] Add support for unit test expectations
[WebKit-https.git] / Tools / glib / api_test_runner.py
1 #!/usr/bin/env python
2 #
3 # Copyright (C) 2011, 2012, 2017 Igalia S.L.
4 #
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Library General Public
7 # License as published by the Free Software Foundation; either
8 # version 2 of the License, or (at your option) any later version.
9 #
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 # Library General Public License for more details.
14 #
15 # You should have received a copy of the GNU Library General Public License
16 # along with this library; see the file COPYING.LIB.  If not, write to
17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 # Boston, MA 02110-1301, USA.
19
20 import subprocess
21 import os
22 import errno
23 import sys
24 import re
25 from signal import alarm, signal, SIGALRM, SIGKILL, SIGSEGV
26
27 top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
28 sys.path.insert(0, os.path.join(top_level_directory, "Tools", "glib"))
29 import common
30 from webkitpy.common.host import Host
31 from webkitpy.common.test_expectations import TestExpectations
32
33
34 class TestTimeout(Exception):
35     pass
36
37
38 class TestRunner(object):
39     TEST_DIRS = []
40
41     def __init__(self, port, options, tests=[]):
42         self._options = options
43
44         self._build_type = "Debug" if self._options.debug else "Release"
45         common.set_build_types((self._build_type,))
46         self._port = Host().port_factory.get(port)
47         self._driver = self._create_driver()
48
49         self._programs_path = common.binary_build_path()
50         expectations_file = os.path.join(common.top_level_path(), "Tools", "TestWebKitAPI", "glib", "TestExpectations.json")
51         self._expectations = TestExpectations(self._port.name(), expectations_file, self._build_type)
52         self._tests = self._get_tests(tests)
53         self._disabled_tests = []
54
55     def _test_programs_base_dir(self):
56         return os.path.join(self._programs_path, "TestWebKitAPI")
57
58     def _get_tests_from_dir(self, test_dir):
59         if not os.path.isdir(test_dir):
60             return []
61
62         tests = []
63         for test_file in os.listdir(test_dir):
64             if not test_file.lower().startswith("test"):
65                 continue
66             test_path = os.path.join(test_dir, test_file)
67             if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
68                 tests.append(test_path)
69         return tests
70
71     def _get_tests(self, initial_tests):
72         tests = []
73         for test in initial_tests:
74             if os.path.isdir(test):
75                 tests.extend(self._get_tests_from_dir(test))
76             else:
77                 tests.append(test)
78         if tests:
79             return tests
80
81         tests = []
82         for test_dir in self.TEST_DIRS:
83             absolute_test_dir = os.path.join(self._test_programs_base_dir(), test_dir)
84             tests.extend(self._get_tests_from_dir(absolute_test_dir))
85         return tests
86
87     def _create_driver(self, port_options=[]):
88         self._port._display_server = self._options.display_server
89         driver = self._port.create_driver(worker_number=0, no_timeout=True)._make_driver(pixel_tests=False)
90         if not driver.check_driver(self._port):
91             raise RuntimeError("Failed to check driver %s" % driver.__class__.__name__)
92         return driver
93
94     def _setup_testing_environment(self):
95         self._test_env = self._driver._setup_environ_for_test()
96         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit")
97         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.library_build_path()
98         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
99
100         return True
101
102     def _tear_down_testing_environment(self):
103         if self._driver:
104             self._driver.stop()
105
106     def _test_cases_to_skip(self, test_program):
107         if self._options.skipped_action != 'skip':
108             return []
109
110         return self._expectations.skipped_subtests(os.path.basename(test_program))
111
112     def _should_run_test_program(self, test_program):
113         for disabled_test in self._disabled_tests:
114             if test_program.endswith(disabled_test):
115                 return False
116
117         if self._options.skipped_action != 'skip':
118             return True
119
120         return os.path.basename(test_program) not in self._expectations.skipped_tests()
121
122     def _kill_process(self, pid):
123         try:
124             os.kill(pid, SIGKILL)
125         except OSError:
126             # Process already died.
127             pass
128
129     @staticmethod
130     def _start_timeout(timeout):
131         if timeout <= 0:
132             return
133
134         def _alarm_handler(signum, frame):
135             raise TestTimeout
136
137         signal(SIGALRM, _alarm_handler)
138         alarm(timeout)
139
140     @staticmethod
141     def _stop_timeout(timeout):
142         if timeout <= 0:
143             return
144
145         alarm(0)
146
147     def _waitpid(self, pid):
148         while True:
149             try:
150                 dummy, status = os.waitpid(pid, 0)
151                 if os.WIFSIGNALED(status):
152                     return -os.WTERMSIG(status)
153                 if os.WIFEXITED(status):
154                     return os.WEXITSTATUS(status)
155
156                 # Should never happen
157                 raise RuntimeError("Unknown child exit status!")
158             except (OSError, IOError) as e:
159                 if e.errno == errno.EINTR:
160                     continue
161                 if e.errno == errno.ECHILD:
162                     # This happens if SIGCLD is set to be ignored or waiting
163                     # for child processes has otherwise been disabled for our
164                     # process.  This child is dead, we can't get the status.
165                     return 0
166                 raise
167
168     def _run_test_glib(self, test_program):
169         command = ['gtester', '-k']
170         if self._options.verbose:
171             command.append('--verbose')
172         for test_case in self._test_cases_to_skip(test_program):
173             command.extend(['-s', test_case])
174         command.append(test_program)
175
176         timeout = self._options.timeout
177         test = os.path.join(os.path.basename(os.path.dirname(test_program)), os.path.basename(test_program))
178         if self._expectations.is_slow(os.path.basename(test_program)):
179             timeout *= 5
180
181         test_context = {"child-pid": -1, "did-timeout": False, "current_test": None}
182
183         def parse_line(line, test_context=test_context):
184             if not line:
185                 return
186
187             match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', line)
188             if match:
189                 test_context["child-pid"] = int(match.group('child_pid'))
190                 sys.stdout.write(line)
191                 return
192
193             def set_test_result(test, result):
194                 if result == "FAIL":
195                     if test_context["did-timeout"] and result == "FAIL":
196                         test_context[test] = "TIMEOUT"
197                     else:
198                         test_context[test] = result
199                 else:
200                     test_context[test] = 'PASS'
201                 test_context["did-timeout"] = False
202                 test_context["current_test"] = None
203                 self._stop_timeout(timeout)
204                 self._start_timeout(timeout)
205
206             normalized_line = line.strip().replace(' ', '')
207             if not normalized_line:
208                 return
209
210             if normalized_line[0] == '/':
211                 test, result = normalized_line.split(':', 1)
212                 if result in ["OK", "FAIL"]:
213                     set_test_result(test, result)
214                 else:
215                     test_context["current_test"] = test
216             elif normalized_line in ["OK", "FAIL"]:
217                 set_test_result(test_context["current_test"], normalized_line)
218
219             sys.stdout.write(line)
220
221         pid, fd = os.forkpty()
222         if pid == 0:
223             os.execvpe(command[0], command, self._test_env)
224             sys.exit(0)
225
226         self._start_timeout(timeout)
227
228         while (True):
229             try:
230                 common.parse_output_lines(fd, parse_line)
231                 break
232             except TestTimeout:
233                 assert test_context["child-pid"] > 0
234                 self._kill_process(test_context["child-pid"])
235                 test_context["child-pid"] = -1
236                 test_context["did-timeout"] = True
237
238         self._stop_timeout(timeout)
239         del test_context["child-pid"]
240         del test_context["did-timeout"]
241         del test_context["current_test"]
242
243         self._waitpid(pid)
244         return test_context
245
246     def _get_tests_from_google_test_suite(self, test_program):
247         try:
248             output = subprocess.check_output([test_program, '--gtest_list_tests'], env=self._test_env)
249         except subprocess.CalledProcessError:
250             sys.stderr.write("ERROR: could not list available tests for binary %s.\n" % (test_program))
251             sys.stderr.flush()
252             return 1
253
254         skipped_test_cases = self._test_cases_to_skip(test_program)
255
256         tests = []
257         prefix = None
258         for line in output.split('\n'):
259             if not line.startswith('  '):
260                 prefix = line
261                 continue
262             else:
263                 test_name = prefix + line.strip()
264                 if not test_name in skipped_test_cases:
265                     tests.append(test_name)
266         return tests
267
268     def _run_google_test(self, test_program, subtest):
269         command = [test_program, '--gtest_filter=%s' % (subtest)]
270         timeout = self._options.timeout
271         if self._expectations.is_slow(os.path.basename(test_program), subtest):
272             timeout *= 5
273
274         pid, fd = os.forkpty()
275         if pid == 0:
276             os.execvpe(command[0], command, self._test_env)
277             sys.exit(0)
278
279         self._start_timeout(timeout)
280         try:
281             common.parse_output_lines(fd, sys.stdout.write)
282             status = self._waitpid(pid)
283         except TestTimeout:
284             self._kill_process(pid)
285             return {subtest: "TIMEOUT"}
286
287         self._stop_timeout(timeout)
288
289         if status == -SIGSEGV:
290             sys.stdout.write("**CRASH** %s\n" % subtest)
291             sys.stdout.flush()
292             return {subtest: "CRASH"}
293
294         if status != 0:
295             return {subtest: "FAIL"}
296
297         return {subtest: "PASS"}
298
299     def _run_google_test_suite(self, test_program):
300         result = {}
301         for subtest in self._get_tests_from_google_test_suite(test_program):
302             result.update(self._run_google_test(test_program, subtest))
303         return result
304
305     def is_glib_test(self, test_program):
306         raise NotImplementedError
307
308     def is_google_test(self, test_program):
309         raise NotImplementedError
310
311     def _run_test(self, test_program):
312         if self.is_glib_test(test_program):
313             return self._run_test_glib(test_program)
314
315         if self.is_google_test(test_program):
316             return self._run_google_test_suite(test_program)
317
318         return {}
319
320     def run_tests(self):
321         if not self._tests:
322             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._test_programs_base_dir()))
323             sys.stderr.flush()
324             return 1
325
326         if not self._setup_testing_environment():
327             return 1
328
329         # Remove skipped tests now instead of when we find them, because
330         # some tests might be skipped while setting up the test environment.
331         self._tests = [test for test in self._tests if self._should_run_test_program(test)]
332
333         crashed_tests = {}
334         failed_tests = {}
335         timed_out_tests = {}
336         passed_tests = {}
337         try:
338             for test in self._tests:
339                 results = self._run_test(test)
340                 for test_case, result in results.iteritems():
341                     if result in self._expectations.get_expectation(os.path.basename(test), test_case):
342                         continue
343
344                     if result == "FAIL":
345                         failed_tests.setdefault(test, []).append(test_case)
346                     elif result == "TIMEOUT":
347                         timed_out_tests.setdefault(test, []).append(test_case)
348                     elif result == "CRASH":
349                         crashed_tests.setdefault(test, []).append(test_case)
350                     elif result == "PASS":
351                         passed_tests.setdefault(test, []).append(test_case)
352         finally:
353             self._tear_down_testing_environment()
354
355         def report(tests, title, base_dir):
356             if not tests:
357                 return
358             sys.stdout.write("\nUnexpected %s (%d)\n" % (title, sum(len(value) for value in tests.itervalues())))
359             for test in tests:
360                 sys.stdout.write("    %s\n" % (test.replace(base_dir, '', 1)))
361                 for test_case in tests[test]:
362                     sys.stdout.write("        %s\n" % (test_case))
363             sys.stdout.flush()
364
365         report(failed_tests, "failures", self._test_programs_base_dir())
366         report(crashed_tests, "crashes", self._test_programs_base_dir())
367         report(timed_out_tests, "timeouts", self._test_programs_base_dir())
368         report(passed_tests, "passes", self._test_programs_base_dir())
369
370         return len(failed_tests) + len(timed_out_tests)
371
372
373 def add_options(option_parser):
374     option_parser.add_option('-r', '--release',
375                              action='store_true', dest='release',
376                              help='Run in Release')
377     option_parser.add_option('-d', '--debug',
378                              action='store_true', dest='debug',
379                              help='Run in Debug')
380     option_parser.add_option('-v', '--verbose',
381                              action='store_true', dest='verbose',
382                              help='Run gtester in verbose mode')
383     option_parser.add_option('--skipped', action='store', dest='skipped_action',
384                              choices=['skip', 'ignore', 'only'], default='skip',
385                              metavar='skip|ignore|only',
386                              help='Specifies how to treat the skipped tests')
387     option_parser.add_option('-t', '--timeout',
388                              action='store', type='int', dest='timeout', default=10,
389                              help='Time in seconds until a test times out')