WebDriver: ignore the driver in selenium test names when getting expectations
[WebKit-https.git] / Tools / Scripts / webkitpy / webdriver_tests / pytest_runner.py
1 # Copyright (C) 2017 Igalia S.L.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
13 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
16 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
17 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
18 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
19 # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
20 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
21 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import errno
24 import json
25 import os
26 import shutil
27 import sys
28 import tempfile
29
30 from webkitpy.common.system.filesystem import FileSystem
31 from webkitpy.common.webkit_finder import WebKitFinder
32 import webkitpy.thirdparty.autoinstalled.pytest
33 import webkitpy.thirdparty.autoinstalled.pytest_timeout
34 import pytest
35 from _pytest.main import EXIT_INTERNALERROR
36
37
38 class TemporaryDirectory(object):
39
40     def __enter__(self):
41         self.path = tempfile.mkdtemp(prefix="pytest-")
42         return self.path
43
44     def __exit__(self, *args):
45         try:
46             shutil.rmtree(self.path)
47         except OSError as e:
48             # no such file or directory
49             if e.errno != errno.ENOENT:
50                 raise
51
52
53 class CollectRecorder(object):
54
55     def __init__(self):
56         self.tests = []
57
58     def pytest_collectreport(self, report):
59         if report.nodeid:
60             self.tests.append(report.nodeid)
61
62
63 class HarnessResultRecorder(object):
64
65     def __init__(self):
66         self.outcome = ('OK', None)
67
68     def pytest_collectreport(self, report):
69         if report.outcome == 'failed':
70             self.outcome = ('ERROR', None)
71         elif report.outcome == 'skipped':
72             self.outcome = ('SKIP', None)
73
74
75 class SubtestResultRecorder(object):
76
77     def __init__(self):
78         self.results = []
79
80     def pytest_runtest_logreport(self, report):
81         if report.passed and report.when == 'call':
82             self.record_pass(report)
83         elif report.failed:
84             if report.when != 'call':
85                 self.record_error(report)
86             else:
87                 self.record_fail(report)
88         elif report.skipped:
89             self.record_skip(report)
90
91     def _was_timeout(self, report):
92         return hasattr(report.longrepr, 'reprcrash') and report.longrepr.reprcrash.message.startswith('Failed: Timeout >')
93
94     def record_pass(self, report):
95         if hasattr(report, 'wasxfail'):
96             if report.wasxfail == 'Timeout':
97                 self.record(report.nodeid, 'XPASS_TIMEOUT')
98             else:
99                 self.record(report.nodeid, 'XPASS')
100         else:
101             self.record(report.nodeid, 'PASS')
102
103     def record_fail(self, report):
104         if self._was_timeout(report):
105             self.record(report.nodeid, 'TIMEOUT', stack=report.longrepr)
106         else:
107             self.record(report.nodeid, 'FAIL', stack=report.longrepr)
108
109     def record_error(self, report):
110         # error in setup/teardown
111         if report.when != 'call':
112             message = '%s error' % report.when
113         self.record(report.nodeid, 'ERROR', message, report.longrepr)
114
115     def record_skip(self, report):
116         if hasattr(report, 'wasxfail'):
117             if self._was_timeout(report) and report.wasxfail != 'Timeout':
118                 self.record(report.nodeid, 'TIMEOUT', stack=report.longrepr)
119             else:
120                 self.record(report.nodeid, 'XFAIL')
121         else:
122             self.record(report.nodeid, 'SKIP')
123
124     def record(self, test, status, message=None, stack=None):
125         if stack is not None:
126             stack = str(stack)
127         new_result = (test, status, message, stack)
128         self.results.append(new_result)
129
130
131 class TestExpectationsMarker(object):
132
133     def __init__(self, expectations, ignore_param):
134         self._expectations = expectations
135         self._ignore_param = ignore_param
136         self._base_dir = WebKitFinder(FileSystem()).path_from_webkit_base('WebDriverTests')
137
138     def _item_name(self, item):
139         if self._ignore_param is None:
140             return item.name
141
142         single_param = '[%s]' % self._ignore_param
143         if item.name.endswith(single_param):
144             return item.name[:-len(single_param)]
145
146         param = '[%s-' % self._ignore_param
147         if param in item.name:
148             return item.name.replace('%s-' % self._ignore_param, '')
149
150         return item.name
151
152     def pytest_collection_modifyitems(self, session, config, items):
153         for item in items:
154             test = os.path.relpath(str(item.fspath), self._base_dir)
155             expected = self._expectations.get_expectation(test, self._item_name(item))[0]
156             if expected == 'FAIL':
157                 item.add_marker(pytest.mark.xfail)
158             elif expected == 'TIMEOUT':
159                 item.add_marker(pytest.mark.xfail(reason="Timeout"))
160             elif expected == 'SKIP':
161                 item.add_marker(pytest.mark.skip)
162
163
164 def collect(directory, args):
165     collect_recorder = CollectRecorder()
166     stdout = sys.stdout
167     with open(os.devnull, 'wb') as devnull:
168         sys.stdout = devnull
169         with TemporaryDirectory() as cache_directory:
170             cmd = ['--collect-only',
171                    '--basetemp=%s' % cache_directory]
172             cmd.extend(args)
173             cmd.append(directory)
174             pytest.main(cmd, plugins=[collect_recorder])
175     sys.stdout = stdout
176     return collect_recorder.tests
177
178
179 def run(path, args, timeout, env, expectations, ignore_param=None):
180     harness_recorder = HarnessResultRecorder()
181     subtests_recorder = SubtestResultRecorder()
182     expectations_marker = TestExpectationsMarker(expectations, ignore_param)
183     _environ = dict(os.environ)
184     os.environ.clear()
185     os.environ.update(env)
186
187     with TemporaryDirectory() as cache_directory:
188         try:
189             cmd = ['--verbose',
190                    '--capture=no',
191                    '--basetemp=%s' % cache_directory,
192                    '--showlocals',
193                    '--timeout=%s' % timeout,
194                    '-p', 'no:cacheprovider',
195                    '-p', 'pytest_timeout']
196             cmd.extend(args)
197             cmd.append(path)
198             result = pytest.main(cmd, plugins=[harness_recorder, subtests_recorder, expectations_marker])
199
200             if result == EXIT_INTERNALERROR:
201                 harness_recorder.outcome = ('ERROR', None)
202         except Exception as e:
203             harness_recorder.outcome = ('ERROR', str(e))
204
205     os.environ.clear()
206     os.environ.update(_environ)
207
208     return harness_recorder.outcome, subtests_recorder.results