b3acc6725a74e4a3f8b938a8d0df48f87d699b20
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 """A helper class for reading in and dealing with tests expectations
30 for layout tests.
31 """
32
33 import logging
34 import re
35
36 from webkitpy.layout_tests.models.test_configuration import TestConfigurationConverter
37
38 _log = logging.getLogger(__name__)
39
40
41 # Test expectation and modifier constants.
42 #
43 # FIXME: range() starts with 0 which makes if expectation checks harder
44 # as PASS is 0.
45 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, SKIP, WONTFIX,
46  SLOW, LEAK, DUMPJSCONSOLELOGINSTDERR, REBASELINE, MISSING, FLAKY, NOW, NONE) = range(18)
47
48 # FIXME: Perhas these two routines should be part of the Port instead?
49 BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt')
50
51
52 class ParseError(Exception):
53     def __init__(self, warnings):
54         super(ParseError, self).__init__()
55         self.warnings = warnings
56
57     def __str__(self):
58         return '\n'.join(map(str, self.warnings))
59
60     def __repr__(self):
61         return 'ParseError(warnings=%s)' % self.warnings
62
63
64 class TestExpectationWarning(object):
65     def __init__(self, filename, line_number, line, error, test=None):
66         self.filename = filename
67         self.line_number = line_number
68         self.line = line
69         self.error = error
70         self.test = test
71
72         self.related_files = {}
73
74     def __str__(self):
75         return '{}:{} {} {}'.format(self.filename, self.line_number, self.error, self.test if self.test else self.line)
76
77
78 class TestExpectationParser(object):
79     """Provides parsing facilities for lines in the test_expectation.txt file."""
80
81     DUMMY_BUG_MODIFIER = "bug_dummy"
82     BUG_MODIFIER_PREFIX = 'bug'
83     BUG_MODIFIER_REGEX = 'bug\d+'
84     REBASELINE_MODIFIER = 'rebaseline'
85     PASS_EXPECTATION = 'pass'
86     SKIP_MODIFIER = 'skip'
87     SLOW_MODIFIER = 'slow'
88     DUMPJSCONSOLELOGINSTDERR_MODIFIER = 'dumpjsconsoleloginstderr'
89     WONTFIX_MODIFIER = 'wontfix'
90
91     TIMEOUT_EXPECTATION = 'timeout'
92
93     MISSING_BUG_WARNING = 'Test lacks BUG modifier.'
94
95     def __init__(self, port, full_test_list, allow_rebaseline_modifier, shorten_filename=lambda x: x):
96         self._port = port
97         self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros())
98         if full_test_list is None:
99             self._full_test_list = None
100         else:
101             self._full_test_list = set(full_test_list)
102         self._allow_rebaseline_modifier = allow_rebaseline_modifier
103         self._shorten_filename = shorten_filename
104
105     def parse(self, filename, expectations_string):
106         expectation_lines = []
107         line_number = 0
108         for line in expectations_string.split("\n"):
109             line_number += 1
110             test_expectation = self._tokenize_line(filename, line, line_number)
111             self._parse_line(test_expectation)
112             expectation_lines.append(test_expectation)
113         return expectation_lines
114
115     def expectation_for_skipped_test(self, test_name):
116         if not self._port.test_exists(test_name):
117             _log.warning('The following test %s from the Skipped list doesn\'t exist' % test_name)
118         expectation_line = TestExpectationLine()
119         expectation_line.original_string = test_name
120         expectation_line.modifiers = [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER]
121         # FIXME: It's not clear what the expectations for a skipped test should be; the expectations
122         # might be different for different entries in a Skipped file, or from the command line, or from
123         # only running parts of the tests. It's also not clear if it matters much.
124         expectation_line.modifiers.append(TestExpectationParser.WONTFIX_MODIFIER)
125         expectation_line.name = test_name
126         # FIXME: we should pass in a more descriptive string here.
127         expectation_line.filename = '<Skipped file>'
128         expectation_line.line_number = 0
129         expectation_line.expectations = [TestExpectationParser.PASS_EXPECTATION]
130         expectation_line.not_applicable_to_current_platform = True
131         self._parse_line(expectation_line)
132         return expectation_line
133
134     def _parse_line(self, expectation_line):
135         if not expectation_line.name:
136             return
137
138         if not self._check_test_exists(expectation_line):
139             return
140
141         expectation_line.is_file = self._port.test_isfile(expectation_line.name)
142         if expectation_line.is_file:
143             expectation_line.path = expectation_line.name
144         else:
145             expectation_line.path = self._port.normalize_test_name(expectation_line.name)
146
147         self._collect_matching_tests(expectation_line)
148
149         self._parse_modifiers(expectation_line)
150         self._parse_expectations(expectation_line)
151
152     def _parse_modifiers(self, expectation_line):
153         has_wontfix = False
154         has_bugid = False
155         parsed_specifiers = set()
156
157         modifiers = [modifier.lower() for modifier in expectation_line.modifiers]
158         expectations = [expectation.lower() for expectation in expectation_line.expectations]
159
160         if self.SLOW_MODIFIER in modifiers and self.TIMEOUT_EXPECTATION in expectations:
161             expectation_line.warnings.append('A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT.')
162
163         for modifier in modifiers:
164             if modifier in TestExpectations.MODIFIERS:
165                 expectation_line.parsed_modifiers.append(modifier)
166                 if modifier == self.WONTFIX_MODIFIER:
167                     has_wontfix = True
168             elif modifier.startswith(self.BUG_MODIFIER_PREFIX):
169                 has_bugid = True
170                 if re.match(self.BUG_MODIFIER_REGEX, modifier):
171                     expectation_line.warnings.append('BUG\d+ is not allowed, must be one of BUGWK\d+, or a non-numeric bug identifier.')
172                 else:
173                     expectation_line.parsed_bug_modifiers.append(modifier)
174             else:
175                 parsed_specifiers.add(modifier)
176
177         if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid and self._port.warn_if_bug_missing_in_test_expectations():
178             expectation_line.warnings.append(self.MISSING_BUG_WARNING)
179
180         if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in modifiers:
181             expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.')
182
183         expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings)
184
185     def _parse_expectations(self, expectation_line):
186         result = set()
187         for part in expectation_line.expectations:
188             expectation = TestExpectations.expectation_from_string(part)
189             if expectation is None:  # Careful, PASS is currently 0.
190                 expectation_line.warnings.append('Unsupported expectation: %s' % part)
191                 continue
192             result.add(expectation)
193         expectation_line.parsed_expectations = result
194
195     def _check_test_exists(self, expectation_line):
196         # WebKit's way of skipping tests is to add a -disabled suffix.
197         # So we should consider the path existing if the path or the
198         # -disabled version exists.
199         if not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled'):
200             # Log a warning here since you hit this case any
201             # time you update TestExpectations without syncing
202             # the LayoutTests directory
203             expectation_line.warnings.append('Path does not exist.')
204             expected_path = self._shorten_filename(self._port.abspath_for_test(expectation_line.name))
205             expectation_line.related_files[expected_path] = None
206             return False
207         return True
208
209     def _collect_matching_tests(self, expectation_line):
210         """Convert the test specification to an absolute, normalized
211         path and make sure directories end with the OS path separator."""
212
213         if not self._full_test_list:
214             expectation_line.matching_tests = [expectation_line.path]
215             return
216
217         if not expectation_line.is_file:
218             # this is a test category, return all the tests of the category.
219             expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)]
220             return
221
222         # this is a test file, do a quick check if it's in the
223         # full test suite.
224         if expectation_line.path in self._full_test_list:
225             expectation_line.matching_tests.append(expectation_line.path)
226
227     # FIXME: Update the original modifiers and remove this once the old syntax is gone.
228     _configuration_tokens_list = [
229         'SnowLeopard', 'Lion', 'MountainLion', 'Mavericks', 'Yosemite', 'ElCapitan', # Legacy macOS
230         'Mac', 'Sierra', 'HighSierra', 'Mojave',
231         'Win', 'XP', 'Vista', 'Win7',
232         'Linux',
233         'Android',
234         'Release',
235         'Debug',
236     ]
237
238     _configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list)
239     _inverted_configuration_tokens = dict((value, name) for name, value in _configuration_tokens.iteritems())
240
241     # FIXME: Update the original modifiers list and remove this once the old syntax is gone.
242     _expectation_tokens = {
243         'Crash': 'CRASH',
244         'Failure': 'FAIL',
245         'ImageOnlyFailure': 'IMAGE',
246         'Leak': 'LEAK',
247         'Missing': 'MISSING',
248         'Pass': 'PASS',
249         'Rebaseline': 'REBASELINE',
250         'Skip': 'SKIP',
251         'Slow': 'SLOW',
252         'Timeout': 'TIMEOUT',
253         'WontFix': 'WONTFIX',
254     }
255
256     _inverted_expectation_tokens = dict([(value, name) for name, value in _expectation_tokens.iteritems()] +
257                                         [('TEXT', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')])
258
259     # FIXME: Seems like these should be classmethods on TestExpectationLine instead of TestExpectationParser.
260     @classmethod
261     def _tokenize_line(cls, filename, expectation_string, line_number):
262         """Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format.
263
264         The new format for a test expectation line is:
265
266         [[bugs] [ "[" <configuration modifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>]
267
268         Any errant whitespace is not preserved.
269
270         """
271         expectation_line = TestExpectationLine()
272         expectation_line.original_string = expectation_string
273         expectation_line.filename = filename
274         expectation_line.line_number = line_number
275
276         comment_index = expectation_string.find("#")
277         if comment_index == -1:
278             comment_index = len(expectation_string)
279         else:
280             expectation_line.comment = expectation_string[comment_index + 1:]
281
282         remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip())
283         if len(remaining_string) == 0:
284             return expectation_line
285
286         # special-case parsing this so that we fail immediately instead of treating this as a test name
287         if remaining_string.startswith('//'):
288             expectation_line.warnings = ['use "#" instead of "//" for comments']
289             return expectation_line
290
291         bugs = []
292         modifiers = []
293         name = None
294         expectations = []
295         warnings = []
296
297         WEBKIT_BUG_PREFIX = 'webkit.org/b/'
298
299         tokens = remaining_string.split()
300         state = 'start'
301         for token in tokens:
302             if token.startswith(WEBKIT_BUG_PREFIX) or token.startswith('Bug('):
303                 if state != 'start':
304                     warnings.append('"%s" is not at the start of the line.' % token)
305                     break
306                 if token.startswith(WEBKIT_BUG_PREFIX):
307                     bugs.append(token.replace(WEBKIT_BUG_PREFIX, 'BUGWK'))
308                 else:
309                     match = re.match('Bug\((\w+)\)$', token)
310                     if not match:
311                         warnings.append('unrecognized bug identifier "%s"' % token)
312                         break
313                     else:
314                         bugs.append('BUG' + match.group(1).upper())
315             elif token.startswith('BUG'):
316                 warnings.append('unrecognized old-style bug identifier "%s"' % token)
317                 break
318             elif token == '[':
319                 if state == 'start':
320                     state = 'configuration'
321                 elif state == 'name_found':
322                     state = 'expectations'
323                 else:
324                     warnings.append('unexpected "["')
325                     break
326             elif token == ']':
327                 if state == 'configuration':
328                     state = 'name'
329                 elif state == 'expectations':
330                     state = 'done'
331                 else:
332                     warnings.append('unexpected "]"')
333                     break
334             elif token in ('//', ':', '='):
335                 warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token)
336                 break
337             elif state == 'configuration':
338                 modifiers.append(cls._configuration_tokens.get(token, token))
339             elif state == 'expectations':
340                 if token in ('Rebaseline', 'Skip', 'Slow', 'WontFix', 'DumpJSConsoleLogInStdErr'):
341                     modifiers.append(token.upper())
342                 elif token not in cls._expectation_tokens:
343                     warnings.append('Unrecognized expectation "%s"' % token)
344                 else:
345                     expectations.append(cls._expectation_tokens.get(token, token))
346             elif state == 'name_found':
347                 warnings.append('expecting "[", "#", or end of line instead of "%s"' % token)
348                 break
349             else:
350                 name = token
351                 state = 'name_found'
352
353         if not warnings:
354             if not name:
355                 warnings.append('Did not find a test name.')
356             elif state not in ('name_found', 'done'):
357                 warnings.append('Missing a "]"')
358
359         if 'WONTFIX' in modifiers and 'SKIP' not in modifiers and not expectations:
360             modifiers.append('SKIP')
361
362         if 'SKIP' in modifiers and expectations:
363             # FIXME: This is really a semantic warning and shouldn't be here. Remove when we drop the old syntax.
364             warnings.append('A test marked Skip must not have other expectations.')
365         elif not expectations:
366             # FIXME: We can probably simplify this adding 'SKIP' if modifiers is empty
367             if 'SKIP' not in modifiers and 'REBASELINE' not in modifiers and 'SLOW' not in modifiers and 'DUMPJSCONSOLELOGINSTDERR' not in modifiers:
368                 modifiers.append('SKIP')
369             expectations = ['PASS']
370
371         # FIXME: expectation line should just store bugs and modifiers separately.
372         expectation_line.modifiers = bugs + modifiers
373         expectation_line.expectations = expectations
374         expectation_line.name = name
375         expectation_line.warnings = warnings
376         return expectation_line
377
378     @classmethod
379     def _split_space_separated(cls, space_separated_string):
380         """Splits a space-separated string into an array."""
381         return [part.strip() for part in space_separated_string.strip().split(' ')]
382
383
384 class TestExpectationLine(object):
385     """Represents a line in test expectations file."""
386
387     def __init__(self):
388         """Initializes a blank-line equivalent of an expectation."""
389         self.original_string = None
390         self.filename = None  # this is the path to the expectations file for this line
391         self.line_number = None
392         self.name = None  # this is the path in the line itself
393         self.path = None  # this is the normpath of self.name
394         self.modifiers = []
395         self.parsed_modifiers = []
396         self.parsed_bug_modifiers = []
397         self.matching_configurations = set()
398         self.expectations = []
399         self.parsed_expectations = set()
400         self.comment = None
401         self.matching_tests = []
402         self.warnings = []
403         self.related_files = {}  # Dictionary of files to lines number in that file which may have caused the list of warnings.
404         self.not_applicable_to_current_platform = False
405
406     def __str__(self):
407         return self.to_string(None)
408
409     def is_invalid(self):
410         return self.warnings and self.warnings != [TestExpectationParser.MISSING_BUG_WARNING]
411
412     def is_flaky(self):
413         return len(self.parsed_expectations) > 1
414
415     @property
416     def expected_behavior(self):
417         expectations = self.expectations[:]
418         if "SLOW" in self.modifiers:
419             expectations += ["SLOW"]
420
421         if "SKIP" in self.modifiers:
422             expectations = ["SKIP"]
423         elif "WONTFIX" in self.modifiers:
424             expectations = ["WONTFIX"]
425         elif "CRASH" in self.modifiers:
426             expectations += ["CRASH"]
427
428         return expectations
429
430     @staticmethod
431     def create_passing_expectation(test):
432         expectation_line = TestExpectationLine()
433         expectation_line.name = test
434         expectation_line.path = test
435         expectation_line.parsed_expectations = set([PASS])
436         expectation_line.expectations = ['PASS']
437         expectation_line.matching_tests = [test]
438         return expectation_line
439
440     def to_string(self, test_configuration_converter, include_modifiers=True, include_expectations=True, include_comment=True):
441         parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
442
443         if self.is_invalid():
444             return self.original_string or ''
445
446         if self.name is None:
447             return '' if self.comment is None else "#%s" % self.comment
448
449         if test_configuration_converter and self.parsed_bug_modifiers:
450             specifiers_list = test_configuration_converter.to_specifiers_list(self.matching_configurations)
451             result = []
452             for specifiers in specifiers_list:
453                 # FIXME: this is silly that we join the modifiers and then immediately split them.
454                 modifiers = self._serialize_parsed_modifiers(test_configuration_converter, specifiers).split()
455                 expectations = self._serialize_parsed_expectations(parsed_expectation_to_string).split()
456                 result.append(self._format_line(modifiers, self.name, expectations, self.comment))
457             return "\n".join(result) if result else None
458
459         return self._format_line(self.modifiers, self.name, self.expectations, self.comment,
460             include_modifiers, include_expectations, include_comment)
461
462     def to_csv(self):
463         # Note that this doesn't include the comments.
464         return '%s,%s,%s' % (self.name, ' '.join(self.modifiers), ' '.join(self.expectations))
465
466     def _serialize_parsed_expectations(self, parsed_expectation_to_string):
467         result = []
468         for index in TestExpectations.EXPECTATION_ORDER:
469             if index in self.parsed_expectations:
470                 result.append(parsed_expectation_to_string[index])
471         return ' '.join(result)
472
473     def _serialize_parsed_modifiers(self, test_configuration_converter, specifiers):
474         result = []
475         if self.parsed_bug_modifiers:
476             result.extend(sorted(self.parsed_bug_modifiers))
477         result.extend(sorted(self.parsed_modifiers))
478         result.extend(test_configuration_converter.specifier_sorter().sort_specifiers(specifiers))
479         return ' '.join(result)
480
481     @staticmethod
482     def _format_line(modifiers, name, expectations, comment, include_modifiers=True, include_expectations=True, include_comment=True):
483         bugs = []
484         new_modifiers = []
485         new_expectations = []
486         for modifier in modifiers:
487             modifier = modifier.upper()
488             if modifier.startswith('BUGWK'):
489                 bugs.append('webkit.org/b/' + modifier.replace('BUGWK', ''))
490             elif modifier.startswith('BUG'):
491                 # FIXME: we should preserve case once we can drop the old syntax.
492                 bugs.append('Bug(' + modifier[3:].lower() + ')')
493             elif modifier in ('SLOW', 'SKIP', 'REBASELINE', 'WONTFIX', 'DUMPJSCONSOLELOGINSTDERR'):
494                 new_expectations.append(TestExpectationParser._inverted_expectation_tokens.get(modifier))
495             else:
496                 new_modifiers.append(TestExpectationParser._inverted_configuration_tokens.get(modifier, modifier))
497
498         for expectation in expectations:
499             expectation = expectation.upper()
500             new_expectations.append(TestExpectationParser._inverted_expectation_tokens.get(expectation, expectation))
501
502         result = ''
503         if include_modifiers and (bugs or new_modifiers):
504             if bugs:
505                 result += ' '.join(bugs) + ' '
506             if new_modifiers:
507                 result += '[ %s ] ' % ' '.join(new_modifiers)
508         result += name
509         if include_expectations and new_expectations and set(new_expectations) != set(['Skip', 'Pass']):
510             result += ' [ %s ]' % ' '.join(sorted(set(new_expectations)))
511         if include_comment and comment is not None:
512             result += " #%s" % comment
513         return result
514
515
516 # FIXME: Refactor API to be a proper CRUD.
517 class TestExpectationsModel(object):
518     """Represents relational store of all expectations and provides CRUD semantics to manage it."""
519
520     def __init__(self, shorten_filename=lambda x: x):
521         # Maps a test to its list of expectations.
522         self._test_to_expectations = {}
523
524         # Maps a test to list of its modifiers (string values)
525         self._test_to_modifiers = {}
526
527         # Maps a test to a TestExpectationLine instance.
528         self._test_to_expectation_line = {}
529
530         self._modifier_to_tests = self._dict_of_sets(TestExpectations.MODIFIERS)
531         self._expectation_to_tests = self._dict_of_sets(TestExpectations.EXPECTATIONS)
532         self._timeline_to_tests = self._dict_of_sets(TestExpectations.TIMELINES)
533         self._result_type_to_tests = self._dict_of_sets(TestExpectations.RESULT_TYPES)
534
535         self._shorten_filename = shorten_filename
536
537     def _dict_of_sets(self, strings_to_constants):
538         """Takes a dict of strings->constants and returns a dict mapping
539         each constant to an empty set."""
540         d = {}
541         for c in strings_to_constants.values():
542             d[c] = set()
543         return d
544
545     def get_test_set(self, modifier, expectation=None, include_skips=True):
546         if expectation is None:
547             tests = self._modifier_to_tests[modifier]
548         else:
549             tests = (self._expectation_to_tests[expectation] &
550                 self._modifier_to_tests[modifier])
551
552         if not include_skips:
553             tests = tests - self.get_test_set(SKIP, expectation)
554
555         return tests
556
557     def get_test_set_for_keyword(self, keyword):
558         # FIXME: get_test_set() is an awkward public interface because it requires
559         # callers to know the difference between modifiers and expectations. We
560         # should replace that with this where possible.
561         expectation_enum = TestExpectations.EXPECTATIONS.get(keyword.lower(), None)
562         if expectation_enum is not None:
563             return self._expectation_to_tests[expectation_enum]
564         modifier_enum = TestExpectations.MODIFIERS.get(keyword.lower(), None)
565         if modifier_enum is not None:
566             return self._modifier_to_tests[modifier_enum]
567
568         # We must not have an index on this modifier.
569         matching_tests = set()
570         for test, modifiers in self._test_to_modifiers.iteritems():
571             if keyword.lower() in modifiers:
572                 matching_tests.add(test)
573         return matching_tests
574
575     def get_tests_with_result_type(self, result_type):
576         return self._result_type_to_tests[result_type]
577
578     def get_tests_with_timeline(self, timeline):
579         return self._timeline_to_tests[timeline]
580
581     def get_modifiers(self, test):
582         """This returns modifiers for the given test (the modifiers plus the BUGXXXX identifier). This is used by the LTTF dashboard."""
583         return self._test_to_modifiers[test]
584
585     def has_modifier(self, test, modifier):
586         return test in self._modifier_to_tests[modifier]
587
588     def has_keyword(self, test, keyword):
589         return (keyword.upper() in self.get_expectations_string(test) or
590                 keyword.lower() in self.get_modifiers(test))
591
592     def has_test(self, test):
593         return test in self._test_to_expectation_line
594
595     def get_expectation_line(self, test):
596         return self._test_to_expectation_line.get(test)
597
598     def get_expectations(self, test):
599         return self._test_to_expectations[test]
600
601     def get_expectations_string(self, test):
602         """Returns the expectatons for the given test as an uppercase string.
603         If there are no expectations for the test, then "PASS" is returned."""
604         try:
605             expectations = self.get_expectations(test)
606         except:
607             return "PASS"
608         retval = []
609
610         for expectation in expectations:
611             retval.append(self.expectation_to_string(expectation))
612
613         return " ".join(retval)
614
615     def expectation_to_string(self, expectation):
616         """Return the uppercased string equivalent of a given expectation."""
617         for item in TestExpectations.EXPECTATIONS.items():
618             if item[1] == expectation:
619                 return item[0].upper()
620         raise ValueError(expectation)
621
622     def add_expectation_line(self, expectation_line, in_skipped=False):
623         """Returns a list of warnings encountered while matching modifiers."""
624
625         if expectation_line.is_invalid():
626             return
627
628         for test in expectation_line.matching_tests:
629             if not in_skipped and self._already_seen_better_match(test, expectation_line):
630                 continue
631
632             self._clear_expectations_for_test(test)
633             self._test_to_expectation_line[test] = expectation_line
634             self._add_test(test, expectation_line)
635
636     def _add_test(self, test, expectation_line):
637         """Sets the expected state for a given test.
638
639         This routine assumes the test has not been added before. If it has,
640         use _clear_expectations_for_test() to reset the state prior to
641         calling this."""
642         self._test_to_expectations[test] = expectation_line.parsed_expectations
643         for expectation in expectation_line.parsed_expectations:
644             self._expectation_to_tests[expectation].add(test)
645
646         self._test_to_modifiers[test] = expectation_line.modifiers
647         for modifier in expectation_line.parsed_modifiers:
648             mod_value = TestExpectations.MODIFIERS[modifier]
649             self._modifier_to_tests[mod_value].add(test)
650
651         if TestExpectationParser.WONTFIX_MODIFIER in expectation_line.parsed_modifiers:
652             self._timeline_to_tests[WONTFIX].add(test)
653         else:
654             self._timeline_to_tests[NOW].add(test)
655
656         if TestExpectationParser.SKIP_MODIFIER in expectation_line.parsed_modifiers:
657             self._result_type_to_tests[SKIP].add(test)
658         elif expectation_line.parsed_expectations == set([PASS]):
659             self._result_type_to_tests[PASS].add(test)
660         elif expectation_line.is_flaky():
661             self._result_type_to_tests[FLAKY].add(test)
662         else:
663             # FIXME: What is this?
664             self._result_type_to_tests[FAIL].add(test)
665
666     def _clear_expectations_for_test(self, test):
667         """Remove prexisting expectations for this test.
668         This happens if we are seeing a more precise path
669         than a previous listing.
670         """
671         if self.has_test(test):
672             self._test_to_expectations.pop(test, '')
673             self._remove_from_sets(test, self._expectation_to_tests)
674             self._remove_from_sets(test, self._modifier_to_tests)
675             self._remove_from_sets(test, self._timeline_to_tests)
676             self._remove_from_sets(test, self._result_type_to_tests)
677
678     def _remove_from_sets(self, test, dict_of_sets_of_tests):
679         """Removes the given test from the sets in the dictionary.
680
681         Args:
682           test: test to look for
683           dict: dict of sets of files"""
684         for set_of_tests in dict_of_sets_of_tests.itervalues():
685             if test in set_of_tests:
686                 set_of_tests.remove(test)
687
688     def _already_seen_better_match(self, test, expectation_line):
689         """Returns whether we've seen a better match already in the file.
690
691         Returns True if we've already seen a expectation_line.name that matches more of the test
692             than this path does
693         """
694         # FIXME: See comment below about matching test configs and specificity.
695         if not self.has_test(test):
696             # We've never seen this test before.
697             return False
698
699         prev_expectation_line = self._test_to_expectation_line[test]
700
701         if prev_expectation_line.filename != expectation_line.filename:
702             # We've moved on to a new expectation file, which overrides older ones.
703             return False
704
705         if len(prev_expectation_line.path) > len(expectation_line.path):
706             # The previous path matched more of the test.
707             return True
708
709         if len(prev_expectation_line.path) < len(expectation_line.path):
710             # This path matches more of the test.
711             return False
712
713         # At this point we know we have seen a previous exact match on this
714         # base path, so we need to check the two sets of modifiers.
715
716         # FIXME: This code was originally designed to allow lines that matched
717         # more modifiers to override lines that matched fewer modifiers.
718         # However, we currently view these as errors.
719         #
720         # To use the "more modifiers wins" policy, change the errors for overrides
721         # to be warnings and return False".
722         shortened_expectation_filename = self._shorten_filename(expectation_line.filename)
723         shortened_previous_expectation_filename = self._shorten_filename(prev_expectation_line.filename)
724
725         if prev_expectation_line.matching_configurations == expectation_line.matching_configurations:
726             expectation_line.warnings.append('Duplicate or ambiguous entry lines %s:%d and %s:%d.' % (
727                 shortened_previous_expectation_filename, prev_expectation_line.line_number,
728                 shortened_expectation_filename, expectation_line.line_number))
729
730         elif prev_expectation_line.matching_configurations >= expectation_line.matching_configurations:
731             expectation_line.warnings.append('More specific entry for %s on line %s:%d overrides line %s:%d.' % (expectation_line.name,
732                 shortened_previous_expectation_filename, prev_expectation_line.line_number,
733                 shortened_expectation_filename, expectation_line.line_number))
734             # FIXME: return False if we want more specific to win.
735
736         elif prev_expectation_line.matching_configurations <= expectation_line.matching_configurations:
737             expectation_line.warnings.append('More specific entry for %s on line %s:%d overrides line %s:%d.' % (expectation_line.name,
738                 shortened_expectation_filename, expectation_line.line_number,
739                 shortened_previous_expectation_filename, prev_expectation_line.line_number))
740
741         elif prev_expectation_line.matching_configurations & expectation_line.matching_configurations:
742             expectation_line.warnings.append('Entries for %s on lines %s:%d and %s:%d match overlapping sets of configurations.' % (expectation_line.name,
743                 shortened_previous_expectation_filename, prev_expectation_line.line_number,
744                 shortened_expectation_filename, expectation_line.line_number))
745
746         else:
747             # Configuration sets are disjoint.
748             return False
749
750         # Missing files will be 'None'. It should be impossible to have a missing file which also has a line associated with it.
751         assert shortened_previous_expectation_filename not in expectation_line.related_files or expectation_line.related_files[shortened_previous_expectation_filename] is not None
752         expectation_line.related_files[shortened_previous_expectation_filename] = expectation_line.related_files.get(shortened_previous_expectation_filename, [])
753         expectation_line.related_files[shortened_previous_expectation_filename].append(prev_expectation_line.line_number)
754
755         assert shortened_expectation_filename not in prev_expectation_line.related_files or prev_expectation_line.related_files[shortened_expectation_filename] is not None
756         prev_expectation_line.related_files[shortened_expectation_filename] = prev_expectation_line.related_files.get(shortened_expectation_filename, [])
757         prev_expectation_line.related_files[shortened_expectation_filename].append(expectation_line.line_number)
758
759         return True
760
761
762 class TestExpectations(object):
763     """Test expectations consist of lines with specifications of what
764     to expect from layout test cases. The test cases can be directories
765     in which case the expectations apply to all test cases in that
766     directory and any subdirectory. The format is along the lines of:
767
768       LayoutTests/js/fixme.js [ Failure ]
769       LayoutTests/js/flaky.js [ Failure Pass ]
770       LayoutTests/js/crash.js [ Crash Failure Pass Timeout ]
771       ...
772
773     To add modifiers:
774       LayoutTests/js/no-good.js
775       [ Debug ] LayoutTests/js/no-good.js [ Pass Timeout ]
776       [ Debug ] LayoutTests/js/no-good.js [ Pass Skip Timeout ]
777       [ Linux Debug ] LayoutTests/js/no-good.js [ Pass Skip Timeout ]
778       [ Linux Win ] LayoutTests/js/no-good.js [ Pass Skip Timeout ]
779
780     Skip: Doesn't run the test.
781     Slow: The test takes a long time to run, but does not timeout indefinitely.
782     WontFix: For tests that we never intend to pass on a given platform (treated like Skip).
783
784     Notes:
785       -A test cannot be both SLOW and TIMEOUT
786       -A test can be included twice, but not via the same path.
787       -If a test is included twice, then the more precise path wins.
788       -CRASH tests cannot be WONTFIX
789     """
790
791     # FIXME: Update to new syntax once the old format is no longer supported.
792     EXPECTATIONS = {'pass': PASS,
793                     'audio': AUDIO,
794                     'fail': FAIL,
795                     'image': IMAGE,
796                     'image+text': IMAGE_PLUS_TEXT,
797                     'text': TEXT,
798                     'timeout': TIMEOUT,
799                     'crash': CRASH,
800                     'missing': MISSING,
801                     'leak': LEAK,
802                     'skip': SKIP}
803
804     # (aggregated by category, pass/fail/skip, type)
805     EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped',
806                                 PASS: 'passes',
807                                 FAIL: 'failures',
808                                 IMAGE: 'image-only failures',
809                                 TEXT: 'text-only failures',
810                                 IMAGE_PLUS_TEXT: 'image and text failures',
811                                 AUDIO: 'audio failures',
812                                 CRASH: 'crashes',
813                                 TIMEOUT: 'timeouts',
814                                 MISSING: 'missing results',
815                                 LEAK: 'leaks'}
816
817     EXPECTATION_ORDER = (PASS, CRASH, TIMEOUT, MISSING, FAIL, IMAGE, LEAK, SKIP)
818
819     BUILD_TYPES = ('debug', 'release')
820
821     MODIFIERS = {TestExpectationParser.SKIP_MODIFIER: SKIP,
822                  TestExpectationParser.WONTFIX_MODIFIER: WONTFIX,
823                  TestExpectationParser.SLOW_MODIFIER: SLOW,
824                  TestExpectationParser.DUMPJSCONSOLELOGINSTDERR_MODIFIER: DUMPJSCONSOLELOGINSTDERR,
825                  TestExpectationParser.REBASELINE_MODIFIER: REBASELINE,
826                  'none': NONE}
827
828     TIMELINES = {TestExpectationParser.WONTFIX_MODIFIER: WONTFIX,
829                  'now': NOW}
830
831     RESULT_TYPES = {'skip': SKIP,
832                     'pass': PASS,
833                     'fail': FAIL,
834                     'flaky': FLAKY}
835
836     @classmethod
837     def expectation_from_string(cls, string):
838         assert(' ' not in string)  # This only handles one expectation at a time.
839         return cls.EXPECTATIONS.get(string.lower())
840
841     @staticmethod
842     def result_was_expected(result, expected_results, test_needs_rebaselining, test_is_skipped):
843         """Returns whether we got a result we were expecting.
844         Args:
845             result: actual result of a test execution
846             expected_results: set of results listed in test_expectations
847             test_needs_rebaselining: whether test was marked as REBASELINE
848             test_is_skipped: whether test was marked as SKIP"""
849         if result in expected_results:
850             return True
851         if result in (TEXT, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected_results):
852             return True
853         if result == MISSING and test_needs_rebaselining:
854             return True
855         if result == SKIP and test_is_skipped:
856             return True
857         return False
858
859     @staticmethod
860     def remove_pixel_failures(expected_results):
861         """Returns a copy of the expected results for a test, except that we
862         drop any pixel failures and return the remaining expectations. For example,
863         if we're not running pixel tests, then tests expected to fail as IMAGE
864         will PASS."""
865         expected_results = expected_results.copy()
866         if IMAGE in expected_results:
867             expected_results.remove(IMAGE)
868             expected_results.add(PASS) # FIXME: does it always become a pass?
869         return expected_results
870
871     @staticmethod
872     def remove_leak_failures(expected_results):
873         """Returns a copy of the expected results for a test, except that we
874         drop any leak failures and return the remaining expectations. For example,
875         if we're not running with --world-leaks, then tests expected to fail as LEAK
876         will PASS."""
877         expected_results = expected_results.copy()
878         if LEAK in expected_results:
879             expected_results.remove(LEAK)
880             if not expected_results:
881                 expected_results.add(PASS)
882         return expected_results
883
884     @staticmethod
885     def has_pixel_failures(actual_results):
886         return IMAGE in actual_results or FAIL in actual_results
887
888     @staticmethod
889     def suffixes_for_expectations(expectations):
890         suffixes = set()
891         if IMAGE in expectations:
892             suffixes.add('png')
893         if FAIL in expectations:
894             suffixes.add('txt')
895             suffixes.add('png')
896             suffixes.add('wav')
897         return set(suffixes)
898
899     def __init__(self, port, tests=None, include_generic=True, include_overrides=True, expectations_to_lint=None, force_expectations_pass=False):
900         self._full_test_list = tests
901         self._test_config = port.test_configuration()
902         self._is_lint_mode = expectations_to_lint is not None
903         self._model = TestExpectationsModel(self._shorten_filename)
904         self._parser = TestExpectationParser(port, tests, self._is_lint_mode, self._shorten_filename)
905         self._port = port
906         self._skipped_tests_warnings = []
907         self._expectations = []
908         self._force_expectations_pass = force_expectations_pass
909         self._include_generic = include_generic
910         self._include_overrides = include_overrides
911         self._expectations_to_lint = expectations_to_lint
912
913     def readable_filename_and_line_number(self, line):
914         if line.not_applicable_to_current_platform:
915             return "(skipped for this platform)"
916         if not line.filename:
917             return ''
918         if line.filename.startswith(self._port.path_from_webkit_base()):
919             return '{}:{}'.format(self._port.host.filesystem.relpath(line.filename, self._port.path_from_webkit_base()), line.line_number)
920         return '{}:{}'.format(line.filename, line.line_number)
921
922     def parse_generic_expectations(self):
923         if self._port.path_to_generic_test_expectations_file() in self._expectations_dict:
924             if self._include_generic:
925                 expectations = self._parser.parse(self._expectations_dict.keys()[self._expectations_dict_index], self._expectations_dict.values()[self._expectations_dict_index])
926                 self._add_expectations(expectations)
927                 self._expectations += expectations
928             self._expectations_dict_index += 1
929
930     def parse_default_port_expectations(self):
931         if len(self._expectations_dict) > self._expectations_dict_index:
932             expectations = self._parser.parse(self._expectations_dict.keys()[self._expectations_dict_index], self._expectations_dict.values()[self._expectations_dict_index])
933             self._add_expectations(expectations)
934             self._expectations += expectations
935             self._expectations_dict_index += 1
936
937     def parse_override_expectations(self):
938         while len(self._expectations_dict) > self._expectations_dict_index and self._include_overrides:
939             expectations = self._parser.parse(self._expectations_dict.keys()[self._expectations_dict_index], self._expectations_dict.values()[self._expectations_dict_index])
940             self._add_expectations(expectations)
941             self._expectations += expectations
942             self._expectations_dict_index += 1
943
944     def parse_all_expectations(self):
945         self._expectations_dict = self._expectations_to_lint or self._port.expectations_dict()
946         self._expectations_dict_index = 0
947
948         self._has_warnings = False
949
950         self.parse_generic_expectations()
951         self.parse_default_port_expectations()
952         self.parse_override_expectations()
953
954         # FIXME: move ignore_tests into port.skipped_layout_tests()
955         self.add_skipped_tests(self._port.skipped_layout_tests(self._full_test_list).union(set(self._port.get_option('ignore_tests', []))))
956
957         self._report_warnings()
958         self._process_tests_without_expectations()
959
960     # TODO(ojan): Allow for removing skipped tests when getting the list of
961     # tests to run, but not when getting metrics.
962     def model(self):
963         return self._model
964
965     def get_rebaselining_failures(self):
966         return self._model.get_test_set(REBASELINE)
967
968     def matches_an_expected_result(self, test, result, pixel_tests_are_enabled, world_leaks_are_enabled):
969         expected_results = self._model.get_expectations(test)
970         if not pixel_tests_are_enabled:
971             expected_results = self.remove_pixel_failures(expected_results)
972
973         if not world_leaks_are_enabled:
974             expected_results = self.remove_leak_failures(expected_results)
975
976         return self.result_was_expected(result,
977                                    expected_results,
978                                    self.is_rebaselining(test),
979                                    self._model.has_modifier(test, SKIP))
980
981     def is_rebaselining(self, test):
982         return self._model.has_modifier(test, REBASELINE)
983
984     def _shorten_filename(self, filename):
985         if filename.startswith(self._port.path_from_webkit_base()):
986             return self._port.host.filesystem.relpath(filename, self._port.path_from_webkit_base())
987         return filename
988
989     def _report_warnings(self):
990         warnings = []
991         for expectation in self._expectations:
992             for warning in expectation.warnings:
993                 warning = TestExpectationWarning(
994                     self._shorten_filename(expectation.filename), expectation.line_number,
995                     expectation.original_string, warning, expectation.name if expectation.expectations else None)
996                 warning.related_files = expectation.related_files
997                 warnings.append(warning)
998
999         if warnings:
1000             self._has_warnings = True
1001             if self._is_lint_mode:
1002                 raise ParseError(warnings)
1003             _log.warning('--lint-test-files warnings:')
1004             for warning in warnings:
1005                 _log.warning(warning)
1006             _log.warning('')
1007
1008     def _process_tests_without_expectations(self):
1009         if self._full_test_list:
1010             for test in self._full_test_list:
1011                 if not self._model.has_test(test):
1012                     self._model.add_expectation_line(TestExpectationLine.create_passing_expectation(test))
1013
1014     def has_warnings(self):
1015         return self._has_warnings
1016
1017     def remove_configuration_from_test(self, test, test_configuration):
1018         expectations_to_remove = []
1019         modified_expectations = []
1020
1021         for expectation in self._expectations:
1022             if expectation.name != test or expectation.is_flaky() or not expectation.parsed_expectations:
1023                 continue
1024             if iter(expectation.parsed_expectations).next() not in (FAIL, IMAGE):
1025                 continue
1026             if test_configuration not in expectation.matching_configurations:
1027                 continue
1028
1029             expectation.matching_configurations.remove(test_configuration)
1030             if expectation.matching_configurations:
1031                 modified_expectations.append(expectation)
1032             else:
1033                 expectations_to_remove.append(expectation)
1034
1035         for expectation in expectations_to_remove:
1036             self._expectations.remove(expectation)
1037
1038         return self.list_to_string(self._expectations, self._parser._test_configuration_converter, modified_expectations)
1039
1040     def remove_rebaselined_tests(self, except_these_tests, filename):
1041         """Returns a copy of the expectations in the file with the tests removed."""
1042         def without_rebaseline_modifier(expectation):
1043             return (expectation.filename == filename and
1044                     not (not expectation.is_invalid() and
1045                          expectation.name in except_these_tests and
1046                          'rebaseline' in expectation.parsed_modifiers))
1047
1048         return self.list_to_string(filter(without_rebaseline_modifier, self._expectations), reconstitute_only_these=[])
1049
1050     def _add_expectations(self, expectation_list):
1051         for expectation_line in expectation_list:
1052             if self._force_expectations_pass:
1053                 expectation_line.expectations = ['PASS']
1054                 expectation_line.parsed_expectations = set([PASS])
1055
1056             elif not expectation_line.expectations:
1057                 continue
1058
1059             if self._is_lint_mode or self._test_config in expectation_line.matching_configurations:
1060                 self._model.add_expectation_line(expectation_line)
1061
1062     def add_skipped_tests(self, tests_to_skip):
1063         if not tests_to_skip:
1064             return
1065         for test in self._expectations:
1066             if test.name and test.name in tests_to_skip:
1067                 test.warnings.append('%s:%d %s is also in a Skipped file.' % (test.filename, test.line_number, test.name))
1068
1069         for test_name in tests_to_skip:
1070             expectation_line = self._parser.expectation_for_skipped_test(test_name)
1071             self._model.add_expectation_line(expectation_line, in_skipped=True)
1072
1073     @staticmethod
1074     def list_to_string(expectation_lines, test_configuration_converter=None, reconstitute_only_these=None):
1075         def serialize(expectation_line):
1076             # If reconstitute_only_these is an empty list, we want to return original_string.
1077             # So we need to compare reconstitute_only_these to None, not just check if it's falsey.
1078             if reconstitute_only_these is None or expectation_line in reconstitute_only_these:
1079                 return expectation_line.to_string(test_configuration_converter)
1080             return expectation_line.original_string
1081
1082         def nones_out(expectation_line):
1083             return expectation_line is not None
1084
1085         return "\n".join(filter(nones_out, map(serialize, expectation_lines)))