WebDriver: add support for test expectations
[WebKit-https.git] / Tools / Scripts / webkitpy / webdriver_tests / pytest_runner.py
1 # Copyright (C) 2017 Igalia S.L.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
13 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
16 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
17 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
18 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
19 # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
20 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
21 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import errno
24 import json
25 import os
26 import shutil
27 import sys
28 import tempfile
29
30 from webkitpy.common.system.filesystem import FileSystem
31 from webkitpy.common.webkit_finder import WebKitFinder
32 import webkitpy.thirdparty.autoinstalled.pytest
33 import webkitpy.thirdparty.autoinstalled.pytest_timeout
34 import pytest
35 from _pytest.main import EXIT_INTERNALERROR
36
37
38 class TemporaryDirectory(object):
39
40     def __enter__(self):
41         self.path = tempfile.mkdtemp(prefix="pytest-")
42         return self.path
43
44     def __exit__(self, *args):
45         try:
46             shutil.rmtree(self.path)
47         except OSError as e:
48             # no such file or directory
49             if e.errno != errno.ENOENT:
50                 raise
51
52
53 class CollectRecorder(object):
54
55     def __init__(self):
56         self.tests = []
57
58     def pytest_collectreport(self, report):
59         if report.nodeid:
60             self.tests.append(report.nodeid)
61
62
63 class HarnessResultRecorder(object):
64
65     def __init__(self):
66         self.outcome = ('OK', None)
67
68     def pytest_collectreport(self, report):
69         if report.outcome == 'failed':
70             self.outcome = ('ERROR', None)
71         elif report.outcome == 'skipped':
72             self.outcome = ('SKIP', None)
73
74
75 class SubtestResultRecorder(object):
76
77     def __init__(self):
78         self.results = []
79
80     def pytest_runtest_logreport(self, report):
81         if report.passed and report.when == 'call':
82             self.record_pass(report)
83         elif report.failed:
84             if report.when != 'call':
85                 self.record_error(report)
86             else:
87                 self.record_fail(report)
88         elif report.skipped:
89             self.record_skip(report)
90
91     def _was_timeout(self, report):
92         return hasattr(report.longrepr, 'reprcrash') and report.longrepr.reprcrash.message.startswith('Failed: Timeout >')
93
94     def record_pass(self, report):
95         if hasattr(report, 'wasxfail'):
96             if report.wasxfail == 'Timeout':
97                 self.record(report.nodeid, 'XPASS_TIMEOUT')
98             else:
99                 self.record(report.nodeid, 'XPASS')
100         else:
101             self.record(report.nodeid, 'PASS')
102
103     def record_fail(self, report):
104         if self._was_timeout(report):
105             self.record(report.nodeid, 'TIMEOUT', stack=report.longrepr)
106         else:
107             self.record(report.nodeid, 'FAIL', stack=report.longrepr)
108
109     def record_error(self, report):
110         # error in setup/teardown
111         if report.when != 'call':
112             message = '%s error' % report.when
113         self.record(report.nodeid, 'ERROR', message, report.longrepr)
114
115     def record_skip(self, report):
116         if hasattr(report, 'wasxfail'):
117             if self._was_timeout(report) and report.wasxfail != 'Timeout':
118                 self.record(report.nodeid, 'TIMEOUT', stack=report.longrepr)
119             else:
120                 self.record(report.nodeid, 'XFAIL')
121         else:
122             self.record(report.nodeid, 'SKIP')
123
124     def record(self, test, status, message=None, stack=None):
125         if stack is not None:
126             stack = str(stack)
127         new_result = (test, status, message, stack)
128         self.results.append(new_result)
129
130
131 class TestExpectationsMarker(object):
132
133     def __init__(self, expectations):
134         self._expectations = expectations
135         self._base_dir = WebKitFinder(FileSystem()).path_from_webkit_base('WebDriverTests')
136
137     def pytest_collection_modifyitems(self, session, config, items):
138         for item in items:
139             test = os.path.relpath(str(item.fspath), self._base_dir)
140             expected = self._expectations.get_expectation(test, item.name)[0]
141             if expected == 'FAIL':
142                 item.add_marker(pytest.mark.xfail)
143             elif expected == 'TIMEOUT':
144                 item.add_marker(pytest.mark.xfail(reason="Timeout"))
145             elif expected == 'SKIP':
146                 item.add_marker(pytest.mark.skip)
147
148
149 def collect(directory, args):
150     collect_recorder = CollectRecorder()
151     stdout = sys.stdout
152     with open(os.devnull, 'wb') as devnull:
153         sys.stdout = devnull
154         with TemporaryDirectory() as cache_directory:
155             cmd = ['--collect-only',
156                    '--basetemp=%s' % cache_directory]
157             cmd.extend(args)
158             cmd.append(directory)
159             pytest.main(cmd, plugins=[collect_recorder])
160     sys.stdout = stdout
161     return collect_recorder.tests
162
163
164 def run(path, args, timeout, env, expectations):
165     harness_recorder = HarnessResultRecorder()
166     subtests_recorder = SubtestResultRecorder()
167     expectations_marker = TestExpectationsMarker(expectations)
168     _environ = dict(os.environ)
169     os.environ.clear()
170     os.environ.update(env)
171
172     with TemporaryDirectory() as cache_directory:
173         try:
174             cmd = ['--verbose',
175                    '--capture=no',
176                    '--basetemp=%s' % cache_directory,
177                    '--showlocals',
178                    '--timeout=%s' % timeout,
179                    '-p', 'no:cacheprovider',
180                    '-p', 'pytest_timeout']
181             cmd.extend(args)
182             cmd.append(path)
183             result = pytest.main(cmd, plugins=[harness_recorder, subtests_recorder, expectations_marker])
184
185             if result == EXIT_INTERNALERROR:
186                 harness_recorder.outcome = ('ERROR', None)
187         except Exception as e:
188             harness_recorder.outcome = ('ERROR', str(e))
189
190     os.environ.clear()
191     os.environ.update(_environ)
192
193     return harness_recorder.outcome, subtests_recorder.results