webkitpy: Implement device type specific expected results (Part 2)
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / controllers / layout_test_finder.py
1 # Copyright (C) 2012 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import errno
30 import logging
31 import re
32
33 from webkitpy.layout_tests.models import test_expectations
34
35
36 _log = logging.getLogger(__name__)
37
38
39 class LayoutTestFinder(object):
40     def __init__(self, port, options):
41         self._port = port
42         self._options = options
43         self._filesystem = self._port.host.filesystem
44         self.LAYOUT_TESTS_DIRECTORY = 'LayoutTests'
45
46     def find_tests(self, options, args, device_type=None):
47         paths = self._strip_test_dir_prefixes(args)
48         if options and options.test_list:
49             paths += self._strip_test_dir_prefixes(self._read_test_names_from_file(options.test_list, self._port.TEST_PATH_SEPARATOR))
50         test_files = self._port.tests(paths, device_type=device_type)
51         return (paths, test_files)
52
53     def find_touched_tests(self, new_or_modified_paths, apply_skip_expectations=True):
54         potential_test_paths = []
55         for test_file in new_or_modified_paths:
56             if not test_file.startswith(self.LAYOUT_TESTS_DIRECTORY):
57                 continue
58
59             test_file = self._strip_test_dir_prefix(test_file)
60             test_paths = self._port.potential_test_names_from_expected_file(test_file)
61             if test_paths:
62                 potential_test_paths.extend(test_paths)
63             else:
64                 potential_test_paths.append(test_file)
65
66         if not potential_test_paths:
67             return None
68
69         tests = self._port.tests(list(set(potential_test_paths)))
70         if not apply_skip_expectations:
71             return tests
72
73         expectations = test_expectations.TestExpectations(self._port, tests, force_expectations_pass=False)
74         expectations.parse_all_expectations()
75         tests_to_skip = self.skip_tests(potential_test_paths, tests, expectations, None)
76         return [test for test in tests if test not in tests_to_skip]
77
78     def _strip_test_dir_prefixes(self, paths):
79         return [self._strip_test_dir_prefix(path) for path in paths if path]
80
81     def _strip_test_dir_prefix(self, path):
82         # Handle both "LayoutTests/foo/bar.html" and "LayoutTests\foo\bar.html" if
83         # the filesystem uses '\\' as a directory separator.
84         if path.startswith(self.LAYOUT_TESTS_DIRECTORY + self._port.TEST_PATH_SEPARATOR):
85             return path[len(self.LAYOUT_TESTS_DIRECTORY + self._port.TEST_PATH_SEPARATOR):]
86         if path.startswith(self.LAYOUT_TESTS_DIRECTORY + self._filesystem.sep):
87             return path[len(self.LAYOUT_TESTS_DIRECTORY + self._filesystem.sep):]
88         return path
89
90     def _read_test_names_from_file(self, filenames, test_path_separator):
91         fs = self._filesystem
92         tests = []
93         for filename in filenames:
94             try:
95                 if test_path_separator != fs.sep:
96                     filename = filename.replace(test_path_separator, fs.sep)
97                 file_contents = fs.read_text_file(filename).split('\n')
98                 for line in file_contents:
99                     line = self._strip_comments(line)
100                     if line:
101                         tests.append(line)
102             except IOError as e:
103                 if e.errno == errno.ENOENT:
104                     _log.critical('')
105                     _log.critical('--test-list file "%s" not found' % file)
106                 raise
107         return tests
108
109     @staticmethod
110     def _strip_comments(line):
111         commentIndex = line.find('//')
112         if commentIndex is -1:
113             commentIndex = len(line)
114
115         line = re.sub(r'\s+', ' ', line[:commentIndex].strip())
116         if line == '':
117             return None
118         else:
119             return line
120
121     def skip_tests(self, paths, all_tests_list, expectations, http_tests):
122         all_tests = set(all_tests_list)
123
124         tests_to_skip = expectations.model().get_tests_with_result_type(test_expectations.SKIP)
125         if self._options.skip_failing_tests:
126             tests_to_skip.update(expectations.model().get_tests_with_result_type(test_expectations.FAIL))
127             tests_to_skip.update(expectations.model().get_tests_with_result_type(test_expectations.FLAKY))
128
129         if self._options.skipped == 'only':
130             tests_to_skip = all_tests - tests_to_skip
131         elif self._options.skipped == 'ignore':
132             tests_to_skip = set()
133         elif self._options.skipped != 'always':
134             # make sure we're explicitly running any tests passed on the command line; equivalent to 'default'.
135             tests_to_skip -= set(paths)
136
137         # unless of course we don't want to run the HTTP tests :)
138         if not self._options.http:
139             tests_to_skip.update(set(http_tests))
140
141         return tests_to_skip
142
143     def split_into_chunks(self, test_names):
144         """split into a list to run and a set to skip, based on --run-chunk and --run-part."""
145         if not self._options.run_chunk and not self._options.run_part:
146             return test_names, set()
147
148         # If the user specifies they just want to run a subset of the tests,
149         # just grab a subset of the non-skipped tests.
150         chunk_value = self._options.run_chunk or self._options.run_part
151         try:
152             (chunk_num, chunk_len) = chunk_value.split(":")
153             chunk_num = int(chunk_num)
154             assert(chunk_num >= 0)
155             test_size = int(chunk_len)
156             assert(test_size > 0)
157         except AssertionError:
158             _log.critical("invalid chunk '%s'" % chunk_value)
159             return (None, None)
160
161         # Get the number of tests
162         num_tests = len(test_names)
163
164         # Get the start offset of the slice.
165         if self._options.run_chunk:
166             chunk_len = test_size
167             # In this case chunk_num can be really large. We need
168             # to make the slave fit in the current number of tests.
169             slice_start = (chunk_num * chunk_len) % num_tests
170         else:
171             # Validate the data.
172             assert(test_size <= num_tests)
173             assert(chunk_num <= test_size)
174
175             # To count the chunk_len, and make sure we don't skip
176             # some tests, we round to the next value that fits exactly
177             # all the parts.
178             rounded_tests = num_tests
179             if rounded_tests % test_size != 0:
180                 rounded_tests = (num_tests + test_size - (num_tests % test_size))
181
182             chunk_len = rounded_tests / test_size
183             slice_start = chunk_len * (chunk_num - 1)
184             # It does not mind if we go over test_size.
185
186         # Get the end offset of the slice.
187         slice_end = min(num_tests, slice_start + chunk_len)
188
189         tests_to_run = test_names[slice_start:slice_end]
190
191         _log.debug('chunk slice [%d:%d] of %d is %d tests' % (slice_start, slice_end, num_tests, (slice_end - slice_start)))
192
193         # If we reached the end and we don't have enough tests, we run some
194         # from the beginning.
195         if slice_end - slice_start < chunk_len:
196             extra = chunk_len - (slice_end - slice_start)
197             _log.debug('   last chunk is partial, appending [0:%d]' % extra)
198             tests_to_run.extend(test_names[0:extra])
199
200         return (tests_to_run, set(test_names) - set(tests_to_run))