lint-test-expectations should be run during style checking
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 """A helper class for reading in and dealing with tests expectations
30 for layout tests.
31 """
32
33 import logging
34 import re
35
36 from webkitpy.layout_tests.models.test_configuration import TestConfigurationConverter
37
38 _log = logging.getLogger(__name__)
39
40
41 # Test expectation and modifier constants.
42 #
43 # FIXME: range() starts with 0 which makes if expectation checks harder
44 # as PASS is 0.
45 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, SKIP, WONTFIX,
46  SLOW, DUMPJSCONSOLELOGINSTDERR, REBASELINE, MISSING, FLAKY, NOW, NONE) = range(17)
47
48 # FIXME: Perhas these two routines should be part of the Port instead?
49 BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt')
50
51
52 class ParseError(Exception):
53     def __init__(self, warnings):
54         super(ParseError, self).__init__()
55         self.warnings = warnings
56
57     def __str__(self):
58         return '\n'.join(map(str, self.warnings))
59
60     def __repr__(self):
61         return 'ParseError(warnings=%s)' % self.warnings
62
63
64 class TestExpectationWarning(object):
65     def __init__(self, filename, line_number, line, error, test=None):
66         self.filename = filename
67         self.line_number = line_number
68         self.line = line
69         self.error = error
70         self.test = test
71
72         self.related_files = {}
73
74     def __str__(self):
75         return '{}:{} {} {}'.format(self.filename, self.line_number, self.error, self.test if self.test else self.line)
76
77
78 class TestExpectationParser(object):
79     """Provides parsing facilities for lines in the test_expectation.txt file."""
80
81     DUMMY_BUG_MODIFIER = "bug_dummy"
82     BUG_MODIFIER_PREFIX = 'bug'
83     BUG_MODIFIER_REGEX = 'bug\d+'
84     REBASELINE_MODIFIER = 'rebaseline'
85     PASS_EXPECTATION = 'pass'
86     SKIP_MODIFIER = 'skip'
87     SLOW_MODIFIER = 'slow'
88     DUMPJSCONSOLELOGINSTDERR_MODIFIER = 'dumpjsconsoleloginstderr'
89     WONTFIX_MODIFIER = 'wontfix'
90
91     TIMEOUT_EXPECTATION = 'timeout'
92
93     MISSING_BUG_WARNING = 'Test lacks BUG modifier.'
94
95     def __init__(self, port, full_test_list, allow_rebaseline_modifier, shorten_filename=lambda x: x):
96         self._port = port
97         self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros())
98         if full_test_list is None:
99             self._full_test_list = None
100         else:
101             self._full_test_list = set(full_test_list)
102         self._allow_rebaseline_modifier = allow_rebaseline_modifier
103         self._shorten_filename = shorten_filename
104
105     def parse(self, filename, expectations_string):
106         expectation_lines = []
107         line_number = 0
108         for line in expectations_string.split("\n"):
109             line_number += 1
110             test_expectation = self._tokenize_line(filename, line, line_number)
111             self._parse_line(test_expectation)
112             expectation_lines.append(test_expectation)
113         return expectation_lines
114
115     def expectation_for_skipped_test(self, test_name):
116         if not self._port.test_exists(test_name):
117             _log.warning('The following test %s from the Skipped list doesn\'t exist' % test_name)
118         expectation_line = TestExpectationLine()
119         expectation_line.original_string = test_name
120         expectation_line.modifiers = [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER]
121         # FIXME: It's not clear what the expectations for a skipped test should be; the expectations
122         # might be different for different entries in a Skipped file, or from the command line, or from
123         # only running parts of the tests. It's also not clear if it matters much.
124         expectation_line.modifiers.append(TestExpectationParser.WONTFIX_MODIFIER)
125         expectation_line.name = test_name
126         # FIXME: we should pass in a more descriptive string here.
127         expectation_line.filename = '<Skipped file>'
128         expectation_line.line_number = 0
129         expectation_line.expectations = [TestExpectationParser.PASS_EXPECTATION]
130         expectation_line.not_applicable_to_current_platform = True
131         self._parse_line(expectation_line)
132         return expectation_line
133
134     def _parse_line(self, expectation_line):
135         if not expectation_line.name:
136             return
137
138         if not self._check_test_exists(expectation_line):
139             return
140
141         expectation_line.is_file = self._port.test_isfile(expectation_line.name)
142         if expectation_line.is_file:
143             expectation_line.path = expectation_line.name
144         else:
145             expectation_line.path = self._port.normalize_test_name(expectation_line.name)
146
147         self._collect_matching_tests(expectation_line)
148
149         self._parse_modifiers(expectation_line)
150         self._parse_expectations(expectation_line)
151
152     def _parse_modifiers(self, expectation_line):
153         has_wontfix = False
154         has_bugid = False
155         parsed_specifiers = set()
156
157         modifiers = [modifier.lower() for modifier in expectation_line.modifiers]
158         expectations = [expectation.lower() for expectation in expectation_line.expectations]
159
160         if self.SLOW_MODIFIER in modifiers and self.TIMEOUT_EXPECTATION in expectations:
161             expectation_line.warnings.append('A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT.')
162
163         for modifier in modifiers:
164             if modifier in TestExpectations.MODIFIERS:
165                 expectation_line.parsed_modifiers.append(modifier)
166                 if modifier == self.WONTFIX_MODIFIER:
167                     has_wontfix = True
168             elif modifier.startswith(self.BUG_MODIFIER_PREFIX):
169                 has_bugid = True
170                 if re.match(self.BUG_MODIFIER_REGEX, modifier):
171                     expectation_line.warnings.append('BUG\d+ is not allowed, must be one of BUGWK\d+, or a non-numeric bug identifier.')
172                 else:
173                     expectation_line.parsed_bug_modifiers.append(modifier)
174             else:
175                 parsed_specifiers.add(modifier)
176
177         if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid and self._port.warn_if_bug_missing_in_test_expectations():
178             expectation_line.warnings.append(self.MISSING_BUG_WARNING)
179
180         if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in modifiers:
181             expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.')
182
183         expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings)
184
185     def _parse_expectations(self, expectation_line):
186         result = set()
187         for part in expectation_line.expectations:
188             expectation = TestExpectations.expectation_from_string(part)
189             if expectation is None:  # Careful, PASS is currently 0.
190                 expectation_line.warnings.append('Unsupported expectation: %s' % part)
191                 continue
192             result.add(expectation)
193         expectation_line.parsed_expectations = result
194
195     def _check_test_exists(self, expectation_line):
196         # WebKit's way of skipping tests is to add a -disabled suffix.
197         # So we should consider the path existing if the path or the
198         # -disabled version exists.
199         if not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled'):
200             # Log a warning here since you hit this case any
201             # time you update TestExpectations without syncing
202             # the LayoutTests directory
203             expectation_line.warnings.append('Path does not exist.')
204             expected_path = self._shorten_filename(self._port.abspath_for_test(expectation_line.name))
205             expectation_line.related_files[expected_path] = None
206             return False
207         return True
208
209     def _collect_matching_tests(self, expectation_line):
210         """Convert the test specification to an absolute, normalized
211         path and make sure directories end with the OS path separator."""
212
213         if not self._full_test_list:
214             expectation_line.matching_tests = [expectation_line.path]
215             return
216
217         if not expectation_line.is_file:
218             # this is a test category, return all the tests of the category.
219             expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)]
220             return
221
222         # this is a test file, do a quick check if it's in the
223         # full test suite.
224         if expectation_line.path in self._full_test_list:
225             expectation_line.matching_tests.append(expectation_line.path)
226
227     # FIXME: Update the original modifiers and remove this once the old syntax is gone.
228     _configuration_tokens_list = [
229         'Mac', 'SnowLeopard', 'Lion', 'MountainLion', 'Mavericks', 'Yosemite', 'ElCapitan', 'Sierra', 'HighSierra',
230         'Win', 'XP', 'Vista', 'Win7',
231         'Linux',
232         'Android',
233         'Release',
234         'Debug',
235     ]
236
237     _configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list)
238     _inverted_configuration_tokens = dict((value, name) for name, value in _configuration_tokens.iteritems())
239
240     # FIXME: Update the original modifiers list and remove this once the old syntax is gone.
241     _expectation_tokens = {
242         'Crash': 'CRASH',
243         'Failure': 'FAIL',
244         'ImageOnlyFailure': 'IMAGE',
245         'Missing': 'MISSING',
246         'Pass': 'PASS',
247         'Rebaseline': 'REBASELINE',
248         'Skip': 'SKIP',
249         'Slow': 'SLOW',
250         'Timeout': 'TIMEOUT',
251         'WontFix': 'WONTFIX',
252     }
253
254     _inverted_expectation_tokens = dict([(value, name) for name, value in _expectation_tokens.iteritems()] +
255                                         [('TEXT', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')])
256
257     # FIXME: Seems like these should be classmethods on TestExpectationLine instead of TestExpectationParser.
258     @classmethod
259     def _tokenize_line(cls, filename, expectation_string, line_number):
260         """Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format.
261
262         The new format for a test expectation line is:
263
264         [[bugs] [ "[" <configuration modifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>]
265
266         Any errant whitespace is not preserved.
267
268         """
269         expectation_line = TestExpectationLine()
270         expectation_line.original_string = expectation_string
271         expectation_line.filename = filename
272         expectation_line.line_number = line_number
273
274         comment_index = expectation_string.find("#")
275         if comment_index == -1:
276             comment_index = len(expectation_string)
277         else:
278             expectation_line.comment = expectation_string[comment_index + 1:]
279
280         remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip())
281         if len(remaining_string) == 0:
282             return expectation_line
283
284         # special-case parsing this so that we fail immediately instead of treating this as a test name
285         if remaining_string.startswith('//'):
286             expectation_line.warnings = ['use "#" instead of "//" for comments']
287             return expectation_line
288
289         bugs = []
290         modifiers = []
291         name = None
292         expectations = []
293         warnings = []
294
295         WEBKIT_BUG_PREFIX = 'webkit.org/b/'
296
297         tokens = remaining_string.split()
298         state = 'start'
299         for token in tokens:
300             if token.startswith(WEBKIT_BUG_PREFIX) or token.startswith('Bug('):
301                 if state != 'start':
302                     warnings.append('"%s" is not at the start of the line.' % token)
303                     break
304                 if token.startswith(WEBKIT_BUG_PREFIX):
305                     bugs.append(token.replace(WEBKIT_BUG_PREFIX, 'BUGWK'))
306                 else:
307                     match = re.match('Bug\((\w+)\)$', token)
308                     if not match:
309                         warnings.append('unrecognized bug identifier "%s"' % token)
310                         break
311                     else:
312                         bugs.append('BUG' + match.group(1).upper())
313             elif token.startswith('BUG'):
314                 warnings.append('unrecognized old-style bug identifier "%s"' % token)
315                 break
316             elif token == '[':
317                 if state == 'start':
318                     state = 'configuration'
319                 elif state == 'name_found':
320                     state = 'expectations'
321                 else:
322                     warnings.append('unexpected "["')
323                     break
324             elif token == ']':
325                 if state == 'configuration':
326                     state = 'name'
327                 elif state == 'expectations':
328                     state = 'done'
329                 else:
330                     warnings.append('unexpected "]"')
331                     break
332             elif token in ('//', ':', '='):
333                 warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token)
334                 break
335             elif state == 'configuration':
336                 modifiers.append(cls._configuration_tokens.get(token, token))
337             elif state == 'expectations':
338                 if token in ('Rebaseline', 'Skip', 'Slow', 'WontFix', 'DumpJSConsoleLogInStdErr'):
339                     modifiers.append(token.upper())
340                 elif token not in cls._expectation_tokens:
341                     warnings.append('Unrecognized expectation "%s"' % token)
342                 else:
343                     expectations.append(cls._expectation_tokens.get(token, token))
344             elif state == 'name_found':
345                 warnings.append('expecting "[", "#", or end of line instead of "%s"' % token)
346                 break
347             else:
348                 name = token
349                 state = 'name_found'
350
351         if not warnings:
352             if not name:
353                 warnings.append('Did not find a test name.')
354             elif state not in ('name_found', 'done'):
355                 warnings.append('Missing a "]"')
356
357         if 'WONTFIX' in modifiers and 'SKIP' not in modifiers and not expectations:
358             modifiers.append('SKIP')
359
360         if 'SKIP' in modifiers and expectations:
361             # FIXME: This is really a semantic warning and shouldn't be here. Remove when we drop the old syntax.
362             warnings.append('A test marked Skip must not have other expectations.')
363         elif not expectations:
364             # FIXME: We can probably simplify this adding 'SKIP' if modifiers is empty
365             if 'SKIP' not in modifiers and 'REBASELINE' not in modifiers and 'SLOW' not in modifiers and 'DUMPJSCONSOLELOGINSTDERR' not in modifiers:
366                 modifiers.append('SKIP')
367             expectations = ['PASS']
368
369         # FIXME: expectation line should just store bugs and modifiers separately.
370         expectation_line.modifiers = bugs + modifiers
371         expectation_line.expectations = expectations
372         expectation_line.name = name
373         expectation_line.warnings = warnings
374         return expectation_line
375
376     @classmethod
377     def _split_space_separated(cls, space_separated_string):
378         """Splits a space-separated string into an array."""
379         return [part.strip() for part in space_separated_string.strip().split(' ')]
380
381
382 class TestExpectationLine(object):
383     """Represents a line in test expectations file."""
384
385     def __init__(self):
386         """Initializes a blank-line equivalent of an expectation."""
387         self.original_string = None
388         self.filename = None  # this is the path to the expectations file for this line
389         self.line_number = None
390         self.name = None  # this is the path in the line itself
391         self.path = None  # this is the normpath of self.name
392         self.modifiers = []
393         self.parsed_modifiers = []
394         self.parsed_bug_modifiers = []
395         self.matching_configurations = set()
396         self.expectations = []
397         self.parsed_expectations = set()
398         self.comment = None
399         self.matching_tests = []
400         self.warnings = []
401         self.related_files = {}  # Dictionary of files to lines number in that file which may have caused the list of warnings.
402         self.not_applicable_to_current_platform = False
403
404     def is_invalid(self):
405         return self.warnings and self.warnings != [TestExpectationParser.MISSING_BUG_WARNING]
406
407     def is_flaky(self):
408         return len(self.parsed_expectations) > 1
409
410     @property
411     def expected_behavior(self):
412         expectations = self.expectations
413         if "SLOW" in self.modifiers:
414             expectations += ["SLOW"]
415
416         if "SKIP" in self.modifiers:
417             expectations = ["SKIP"]
418         elif "WONTFIX" in self.modifiers:
419             expectations = ["WONTFIX"]
420         elif "CRASH" in self.modifiers:
421             expectations += ["CRASH"]
422
423         return expectations
424
425     @staticmethod
426     def create_passing_expectation(test):
427         expectation_line = TestExpectationLine()
428         expectation_line.name = test
429         expectation_line.path = test
430         expectation_line.parsed_expectations = set([PASS])
431         expectation_line.expectations = ['PASS']
432         expectation_line.matching_tests = [test]
433         return expectation_line
434
435     def to_string(self, test_configuration_converter, include_modifiers=True, include_expectations=True, include_comment=True):
436         parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
437
438         if self.is_invalid():
439             return self.original_string or ''
440
441         if self.name is None:
442             return '' if self.comment is None else "#%s" % self.comment
443
444         if test_configuration_converter and self.parsed_bug_modifiers:
445             specifiers_list = test_configuration_converter.to_specifiers_list(self.matching_configurations)
446             result = []
447             for specifiers in specifiers_list:
448                 # FIXME: this is silly that we join the modifiers and then immediately split them.
449                 modifiers = self._serialize_parsed_modifiers(test_configuration_converter, specifiers).split()
450                 expectations = self._serialize_parsed_expectations(parsed_expectation_to_string).split()
451                 result.append(self._format_line(modifiers, self.name, expectations, self.comment))
452             return "\n".join(result) if result else None
453
454         return self._format_line(self.modifiers, self.name, self.expectations, self.comment,
455             include_modifiers, include_expectations, include_comment)
456
457     def to_csv(self):
458         # Note that this doesn't include the comments.
459         return '%s,%s,%s' % (self.name, ' '.join(self.modifiers), ' '.join(self.expectations))
460
461     def _serialize_parsed_expectations(self, parsed_expectation_to_string):
462         result = []
463         for index in TestExpectations.EXPECTATION_ORDER:
464             if index in self.parsed_expectations:
465                 result.append(parsed_expectation_to_string[index])
466         return ' '.join(result)
467
468     def _serialize_parsed_modifiers(self, test_configuration_converter, specifiers):
469         result = []
470         if self.parsed_bug_modifiers:
471             result.extend(sorted(self.parsed_bug_modifiers))
472         result.extend(sorted(self.parsed_modifiers))
473         result.extend(test_configuration_converter.specifier_sorter().sort_specifiers(specifiers))
474         return ' '.join(result)
475
476     @staticmethod
477     def _format_line(modifiers, name, expectations, comment, include_modifiers=True, include_expectations=True, include_comment=True):
478         bugs = []
479         new_modifiers = []
480         new_expectations = []
481         for modifier in modifiers:
482             modifier = modifier.upper()
483             if modifier.startswith('BUGWK'):
484                 bugs.append('webkit.org/b/' + modifier.replace('BUGWK', ''))
485             elif modifier.startswith('BUG'):
486                 # FIXME: we should preserve case once we can drop the old syntax.
487                 bugs.append('Bug(' + modifier[3:].lower() + ')')
488             elif modifier in ('SLOW', 'SKIP', 'REBASELINE', 'WONTFIX', 'DUMPJSCONSOLELOGINSTDERR'):
489                 new_expectations.append(TestExpectationParser._inverted_expectation_tokens.get(modifier))
490             else:
491                 new_modifiers.append(TestExpectationParser._inverted_configuration_tokens.get(modifier, modifier))
492
493         for expectation in expectations:
494             expectation = expectation.upper()
495             new_expectations.append(TestExpectationParser._inverted_expectation_tokens.get(expectation, expectation))
496
497         result = ''
498         if include_modifiers and (bugs or new_modifiers):
499             if bugs:
500                 result += ' '.join(bugs) + ' '
501             if new_modifiers:
502                 result += '[ %s ] ' % ' '.join(new_modifiers)
503         result += name
504         if include_expectations and new_expectations and set(new_expectations) != set(['Skip', 'Pass']):
505             result += ' [ %s ]' % ' '.join(sorted(set(new_expectations)))
506         if include_comment and comment is not None:
507             result += " #%s" % comment
508         return result
509
510
511 # FIXME: Refactor API to be a proper CRUD.
512 class TestExpectationsModel(object):
513     """Represents relational store of all expectations and provides CRUD semantics to manage it."""
514
515     def __init__(self, shorten_filename=lambda x: x):
516         # Maps a test to its list of expectations.
517         self._test_to_expectations = {}
518
519         # Maps a test to list of its modifiers (string values)
520         self._test_to_modifiers = {}
521
522         # Maps a test to a TestExpectationLine instance.
523         self._test_to_expectation_line = {}
524
525         self._modifier_to_tests = self._dict_of_sets(TestExpectations.MODIFIERS)
526         self._expectation_to_tests = self._dict_of_sets(TestExpectations.EXPECTATIONS)
527         self._timeline_to_tests = self._dict_of_sets(TestExpectations.TIMELINES)
528         self._result_type_to_tests = self._dict_of_sets(TestExpectations.RESULT_TYPES)
529
530         self._shorten_filename = shorten_filename
531
532     def _dict_of_sets(self, strings_to_constants):
533         """Takes a dict of strings->constants and returns a dict mapping
534         each constant to an empty set."""
535         d = {}
536         for c in strings_to_constants.values():
537             d[c] = set()
538         return d
539
540     def get_test_set(self, modifier, expectation=None, include_skips=True):
541         if expectation is None:
542             tests = self._modifier_to_tests[modifier]
543         else:
544             tests = (self._expectation_to_tests[expectation] &
545                 self._modifier_to_tests[modifier])
546
547         if not include_skips:
548             tests = tests - self.get_test_set(SKIP, expectation)
549
550         return tests
551
552     def get_test_set_for_keyword(self, keyword):
553         # FIXME: get_test_set() is an awkward public interface because it requires
554         # callers to know the difference between modifiers and expectations. We
555         # should replace that with this where possible.
556         expectation_enum = TestExpectations.EXPECTATIONS.get(keyword.lower(), None)
557         if expectation_enum is not None:
558             return self._expectation_to_tests[expectation_enum]
559         modifier_enum = TestExpectations.MODIFIERS.get(keyword.lower(), None)
560         if modifier_enum is not None:
561             return self._modifier_to_tests[modifier_enum]
562
563         # We must not have an index on this modifier.
564         matching_tests = set()
565         for test, modifiers in self._test_to_modifiers.iteritems():
566             if keyword.lower() in modifiers:
567                 matching_tests.add(test)
568         return matching_tests
569
570     def get_tests_with_result_type(self, result_type):
571         return self._result_type_to_tests[result_type]
572
573     def get_tests_with_timeline(self, timeline):
574         return self._timeline_to_tests[timeline]
575
576     def get_modifiers(self, test):
577         """This returns modifiers for the given test (the modifiers plus the BUGXXXX identifier). This is used by the LTTF dashboard."""
578         return self._test_to_modifiers[test]
579
580     def has_modifier(self, test, modifier):
581         return test in self._modifier_to_tests[modifier]
582
583     def has_keyword(self, test, keyword):
584         return (keyword.upper() in self.get_expectations_string(test) or
585                 keyword.lower() in self.get_modifiers(test))
586
587     def has_test(self, test):
588         return test in self._test_to_expectation_line
589
590     def get_expectation_line(self, test):
591         return self._test_to_expectation_line.get(test)
592
593     def get_expectations(self, test):
594         return self._test_to_expectations[test]
595
596     def get_expectations_string(self, test):
597         """Returns the expectatons for the given test as an uppercase string.
598         If there are no expectations for the test, then "PASS" is returned."""
599         try:
600             expectations = self.get_expectations(test)
601         except:
602             return "PASS"
603         retval = []
604
605         for expectation in expectations:
606             retval.append(self.expectation_to_string(expectation))
607
608         return " ".join(retval)
609
610     def expectation_to_string(self, expectation):
611         """Return the uppercased string equivalent of a given expectation."""
612         for item in TestExpectations.EXPECTATIONS.items():
613             if item[1] == expectation:
614                 return item[0].upper()
615         raise ValueError(expectation)
616
617     def add_expectation_line(self, expectation_line, in_skipped=False):
618         """Returns a list of warnings encountered while matching modifiers."""
619
620         if expectation_line.is_invalid():
621             return
622
623         for test in expectation_line.matching_tests:
624             if not in_skipped and self._already_seen_better_match(test, expectation_line):
625                 continue
626
627             self._clear_expectations_for_test(test)
628             self._test_to_expectation_line[test] = expectation_line
629             self._add_test(test, expectation_line)
630
631     def _add_test(self, test, expectation_line):
632         """Sets the expected state for a given test.
633
634         This routine assumes the test has not been added before. If it has,
635         use _clear_expectations_for_test() to reset the state prior to
636         calling this."""
637         self._test_to_expectations[test] = expectation_line.parsed_expectations
638         for expectation in expectation_line.parsed_expectations:
639             self._expectation_to_tests[expectation].add(test)
640
641         self._test_to_modifiers[test] = expectation_line.modifiers
642         for modifier in expectation_line.parsed_modifiers:
643             mod_value = TestExpectations.MODIFIERS[modifier]
644             self._modifier_to_tests[mod_value].add(test)
645
646         if TestExpectationParser.WONTFIX_MODIFIER in expectation_line.parsed_modifiers:
647             self._timeline_to_tests[WONTFIX].add(test)
648         else:
649             self._timeline_to_tests[NOW].add(test)
650
651         if TestExpectationParser.SKIP_MODIFIER in expectation_line.parsed_modifiers:
652             self._result_type_to_tests[SKIP].add(test)
653         elif expectation_line.parsed_expectations == set([PASS]):
654             self._result_type_to_tests[PASS].add(test)
655         elif expectation_line.is_flaky():
656             self._result_type_to_tests[FLAKY].add(test)
657         else:
658             # FIXME: What is this?
659             self._result_type_to_tests[FAIL].add(test)
660
661     def _clear_expectations_for_test(self, test):
662         """Remove prexisting expectations for this test.
663         This happens if we are seeing a more precise path
664         than a previous listing.
665         """
666         if self.has_test(test):
667             self._test_to_expectations.pop(test, '')
668             self._remove_from_sets(test, self._expectation_to_tests)
669             self._remove_from_sets(test, self._modifier_to_tests)
670             self._remove_from_sets(test, self._timeline_to_tests)
671             self._remove_from_sets(test, self._result_type_to_tests)
672
673     def _remove_from_sets(self, test, dict_of_sets_of_tests):
674         """Removes the given test from the sets in the dictionary.
675
676         Args:
677           test: test to look for
678           dict: dict of sets of files"""
679         for set_of_tests in dict_of_sets_of_tests.itervalues():
680             if test in set_of_tests:
681                 set_of_tests.remove(test)
682
683     def _already_seen_better_match(self, test, expectation_line):
684         """Returns whether we've seen a better match already in the file.
685
686         Returns True if we've already seen a expectation_line.name that matches more of the test
687             than this path does
688         """
689         # FIXME: See comment below about matching test configs and specificity.
690         if not self.has_test(test):
691             # We've never seen this test before.
692             return False
693
694         prev_expectation_line = self._test_to_expectation_line[test]
695
696         if prev_expectation_line.filename != expectation_line.filename:
697             # We've moved on to a new expectation file, which overrides older ones.
698             return False
699
700         if len(prev_expectation_line.path) > len(expectation_line.path):
701             # The previous path matched more of the test.
702             return True
703
704         if len(prev_expectation_line.path) < len(expectation_line.path):
705             # This path matches more of the test.
706             return False
707
708         # At this point we know we have seen a previous exact match on this
709         # base path, so we need to check the two sets of modifiers.
710
711         # FIXME: This code was originally designed to allow lines that matched
712         # more modifiers to override lines that matched fewer modifiers.
713         # However, we currently view these as errors.
714         #
715         # To use the "more modifiers wins" policy, change the errors for overrides
716         # to be warnings and return False".
717         shortened_expectation_filename = self._shorten_filename(expectation_line.filename)
718         shortened_previous_expectation_filename = self._shorten_filename(prev_expectation_line.filename)
719
720         if prev_expectation_line.matching_configurations == expectation_line.matching_configurations:
721             expectation_line.warnings.append('Duplicate or ambiguous entry lines %s:%d and %s:%d.' % (
722                 shortened_previous_expectation_filename, prev_expectation_line.line_number,
723                 shortened_expectation_filename, expectation_line.line_number))
724
725         elif prev_expectation_line.matching_configurations >= expectation_line.matching_configurations:
726             expectation_line.warnings.append('More specific entry for %s on line %s:%d overrides line %s:%d.' % (expectation_line.name,
727                 shortened_previous_expectation_filename, prev_expectation_line.line_number,
728                 shortened_expectation_filename, expectation_line.line_number))
729             # FIXME: return False if we want more specific to win.
730
731         elif prev_expectation_line.matching_configurations <= expectation_line.matching_configurations:
732             expectation_line.warnings.append('More specific entry for %s on line %s:%d overrides line %s:%d.' % (expectation_line.name,
733                 shortened_expectation_filename, expectation_line.line_number,
734                 shortened_previous_expectation_filename, prev_expectation_line.line_number))
735
736         elif prev_expectation_line.matching_configurations & expectation_line.matching_configurations:
737             expectation_line.warnings.append('Entries for %s on lines %s:%d and %s:%d match overlapping sets of configurations.' % (expectation_line.name,
738                 shortened_previous_expectation_filename, prev_expectation_line.line_number,
739                 shortened_expectation_filename, expectation_line.line_number))
740
741         else:
742             # Configuration sets are disjoint.
743             return False
744
745         # Missing files will be 'None'. It should be impossible to have a missing file which also has a line associated with it.
746         assert shortened_previous_expectation_filename not in expectation_line.related_files or expectation_line.related_files[shortened_previous_expectation_filename] is not None
747         expectation_line.related_files[shortened_previous_expectation_filename] = expectation_line.related_files.get(shortened_previous_expectation_filename, [])
748         expectation_line.related_files[shortened_previous_expectation_filename].append(prev_expectation_line.line_number)
749
750         assert shortened_expectation_filename not in prev_expectation_line.related_files or prev_expectation_line.related_files[shortened_expectation_filename] is not None
751         prev_expectation_line.related_files[shortened_expectation_filename] = prev_expectation_line.related_files.get(shortened_expectation_filename, [])
752         prev_expectation_line.related_files[shortened_expectation_filename].append(expectation_line.line_number)
753
754         return True
755
756
757 class TestExpectations(object):
758     """Test expectations consist of lines with specifications of what
759     to expect from layout test cases. The test cases can be directories
760     in which case the expectations apply to all test cases in that
761     directory and any subdirectory. The format is along the lines of:
762
763       LayoutTests/js/fixme.js [ Failure ]
764       LayoutTests/js/flaky.js [ Failure Pass ]
765       LayoutTests/js/crash.js [ Crash Failure Pass Timeout ]
766       ...
767
768     To add modifiers:
769       LayoutTests/js/no-good.js
770       [ Debug ] LayoutTests/js/no-good.js [ Pass Timeout ]
771       [ Debug ] LayoutTests/js/no-good.js [ Pass Skip Timeout ]
772       [ Linux Debug ] LayoutTests/js/no-good.js [ Pass Skip Timeout ]
773       [ Linux Win ] LayoutTests/js/no-good.js [ Pass Skip Timeout ]
774
775     Skip: Doesn't run the test.
776     Slow: The test takes a long time to run, but does not timeout indefinitely.
777     WontFix: For tests that we never intend to pass on a given platform (treated like Skip).
778
779     Notes:
780       -A test cannot be both SLOW and TIMEOUT
781       -A test can be included twice, but not via the same path.
782       -If a test is included twice, then the more precise path wins.
783       -CRASH tests cannot be WONTFIX
784     """
785
786     # FIXME: Update to new syntax once the old format is no longer supported.
787     EXPECTATIONS = {'pass': PASS,
788                     'audio': AUDIO,
789                     'fail': FAIL,
790                     'image': IMAGE,
791                     'image+text': IMAGE_PLUS_TEXT,
792                     'text': TEXT,
793                     'timeout': TIMEOUT,
794                     'crash': CRASH,
795                     'missing': MISSING,
796                     'skip': SKIP}
797
798     # (aggregated by category, pass/fail/skip, type)
799     EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped',
800                                 PASS: 'passes',
801                                 FAIL: 'failures',
802                                 IMAGE: 'image-only failures',
803                                 TEXT: 'text-only failures',
804                                 IMAGE_PLUS_TEXT: 'image and text failures',
805                                 AUDIO: 'audio failures',
806                                 CRASH: 'crashes',
807                                 TIMEOUT: 'timeouts',
808                                 MISSING: 'missing results'}
809
810     EXPECTATION_ORDER = (PASS, CRASH, TIMEOUT, MISSING, FAIL, IMAGE, SKIP)
811
812     BUILD_TYPES = ('debug', 'release')
813
814     MODIFIERS = {TestExpectationParser.SKIP_MODIFIER: SKIP,
815                  TestExpectationParser.WONTFIX_MODIFIER: WONTFIX,
816                  TestExpectationParser.SLOW_MODIFIER: SLOW,
817                  TestExpectationParser.DUMPJSCONSOLELOGINSTDERR_MODIFIER: DUMPJSCONSOLELOGINSTDERR,
818                  TestExpectationParser.REBASELINE_MODIFIER: REBASELINE,
819                  'none': NONE}
820
821     TIMELINES = {TestExpectationParser.WONTFIX_MODIFIER: WONTFIX,
822                  'now': NOW}
823
824     RESULT_TYPES = {'skip': SKIP,
825                     'pass': PASS,
826                     'fail': FAIL,
827                     'flaky': FLAKY}
828
829     @classmethod
830     def expectation_from_string(cls, string):
831         assert(' ' not in string)  # This only handles one expectation at a time.
832         return cls.EXPECTATIONS.get(string.lower())
833
834     @staticmethod
835     def result_was_expected(result, expected_results, test_needs_rebaselining, test_is_skipped):
836         """Returns whether we got a result we were expecting.
837         Args:
838             result: actual result of a test execution
839             expected_results: set of results listed in test_expectations
840             test_needs_rebaselining: whether test was marked as REBASELINE
841             test_is_skipped: whether test was marked as SKIP"""
842         if result in expected_results:
843             return True
844         if result in (TEXT, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected_results):
845             return True
846         if result == MISSING and test_needs_rebaselining:
847             return True
848         if result == SKIP and test_is_skipped:
849             return True
850         return False
851
852     @staticmethod
853     def remove_pixel_failures(expected_results):
854         """Returns a copy of the expected results for a test, except that we
855         drop any pixel failures and return the remaining expectations. For example,
856         if we're not running pixel tests, then tests expected to fail as IMAGE
857         will PASS."""
858         expected_results = expected_results.copy()
859         if IMAGE in expected_results:
860             expected_results.remove(IMAGE)
861             expected_results.add(PASS)
862         return expected_results
863
864     @staticmethod
865     def has_pixel_failures(actual_results):
866         return IMAGE in actual_results or FAIL in actual_results
867
868     @staticmethod
869     def suffixes_for_expectations(expectations):
870         suffixes = set()
871         if IMAGE in expectations:
872             suffixes.add('png')
873         if FAIL in expectations:
874             suffixes.add('txt')
875             suffixes.add('png')
876             suffixes.add('wav')
877         return set(suffixes)
878
879     def __init__(self, port, tests=None, include_generic=True, include_overrides=True, expectations_to_lint=None, force_expectations_pass=False):
880         self._full_test_list = tests
881         self._test_config = port.test_configuration()
882         self._is_lint_mode = expectations_to_lint is not None
883         self._model = TestExpectationsModel(self._shorten_filename)
884         self._parser = TestExpectationParser(port, tests, self._is_lint_mode, self._shorten_filename)
885         self._port = port
886         self._skipped_tests_warnings = []
887         self._expectations = []
888         self._force_expectations_pass = force_expectations_pass
889         self._include_generic = include_generic
890         self._include_overrides = include_overrides
891         self._expectations_to_lint = expectations_to_lint
892
893     def readable_filename_and_line_number(self, line):
894         if line.not_applicable_to_current_platform:
895             return "(skipped for this platform)"
896         if not line.filename:
897             return ''
898         if line.filename.startswith(self._port.path_from_webkit_base()):
899             return '{}:{}'.format(self._port.host.filesystem.relpath(line.filename, self._port.path_from_webkit_base()), line.line_number)
900         return '{}:{}'.format(line.filename, line.line_number)
901
902     def parse_generic_expectations(self):
903         if self._port.path_to_generic_test_expectations_file() in self._expectations_dict:
904             if self._include_generic:
905                 expectations = self._parser.parse(self._expectations_dict.keys()[self._expectations_dict_index], self._expectations_dict.values()[self._expectations_dict_index])
906                 self._add_expectations(expectations)
907                 self._expectations += expectations
908             self._expectations_dict_index += 1
909
910     def parse_default_port_expectations(self):
911         if len(self._expectations_dict) > self._expectations_dict_index:
912             expectations = self._parser.parse(self._expectations_dict.keys()[self._expectations_dict_index], self._expectations_dict.values()[self._expectations_dict_index])
913             self._add_expectations(expectations)
914             self._expectations += expectations
915             self._expectations_dict_index += 1
916
917     def parse_override_expectations(self):
918         while len(self._expectations_dict) > self._expectations_dict_index and self._include_overrides:
919             expectations = self._parser.parse(self._expectations_dict.keys()[self._expectations_dict_index], self._expectations_dict.values()[self._expectations_dict_index])
920             self._add_expectations(expectations)
921             self._expectations += expectations
922             self._expectations_dict_index += 1
923
924     def parse_all_expectations(self):
925         self._expectations_dict = self._expectations_to_lint or self._port.expectations_dict()
926         self._expectations_dict_index = 0
927
928         self._has_warnings = False
929
930         self.parse_generic_expectations()
931         self.parse_default_port_expectations()
932         self.parse_override_expectations()
933
934         # FIXME: move ignore_tests into port.skipped_layout_tests()
935         self.add_skipped_tests(self._port.skipped_layout_tests(self._full_test_list).union(set(self._port.get_option('ignore_tests', []))))
936
937         self._report_warnings()
938         self._process_tests_without_expectations()
939
940     # TODO(ojan): Allow for removing skipped tests when getting the list of
941     # tests to run, but not when getting metrics.
942     def model(self):
943         return self._model
944
945     def get_rebaselining_failures(self):
946         return self._model.get_test_set(REBASELINE)
947
948     def matches_an_expected_result(self, test, result, pixel_tests_are_enabled):
949         expected_results = self._model.get_expectations(test)
950         if not pixel_tests_are_enabled:
951             expected_results = self.remove_pixel_failures(expected_results)
952         return self.result_was_expected(result,
953                                    expected_results,
954                                    self.is_rebaselining(test),
955                                    self._model.has_modifier(test, SKIP))
956
957     def is_rebaselining(self, test):
958         return self._model.has_modifier(test, REBASELINE)
959
960     def _shorten_filename(self, filename):
961         if filename.startswith(self._port.path_from_webkit_base()):
962             return self._port.host.filesystem.relpath(filename, self._port.path_from_webkit_base())
963         return filename
964
965     def _report_warnings(self):
966         warnings = []
967         for expectation in self._expectations:
968             for warning in expectation.warnings:
969                 warning = TestExpectationWarning(
970                     self._shorten_filename(expectation.filename), expectation.line_number,
971                     expectation.original_string, warning, expectation.name if expectation.expectations else None)
972                 warning.related_files = expectation.related_files
973                 warnings.append(warning)
974
975         if warnings:
976             self._has_warnings = True
977             if self._is_lint_mode:
978                 raise ParseError(warnings)
979             _log.warning('--lint-test-files warnings:')
980             for warning in warnings:
981                 _log.warning(warning)
982             _log.warning('')
983
984     def _process_tests_without_expectations(self):
985         if self._full_test_list:
986             for test in self._full_test_list:
987                 if not self._model.has_test(test):
988                     self._model.add_expectation_line(TestExpectationLine.create_passing_expectation(test))
989
990     def has_warnings(self):
991         return self._has_warnings
992
993     def remove_configuration_from_test(self, test, test_configuration):
994         expectations_to_remove = []
995         modified_expectations = []
996
997         for expectation in self._expectations:
998             if expectation.name != test or expectation.is_flaky() or not expectation.parsed_expectations:
999                 continue
1000             if iter(expectation.parsed_expectations).next() not in (FAIL, IMAGE):
1001                 continue
1002             if test_configuration not in expectation.matching_configurations:
1003                 continue
1004
1005             expectation.matching_configurations.remove(test_configuration)
1006             if expectation.matching_configurations:
1007                 modified_expectations.append(expectation)
1008             else:
1009                 expectations_to_remove.append(expectation)
1010
1011         for expectation in expectations_to_remove:
1012             self._expectations.remove(expectation)
1013
1014         return self.list_to_string(self._expectations, self._parser._test_configuration_converter, modified_expectations)
1015
1016     def remove_rebaselined_tests(self, except_these_tests, filename):
1017         """Returns a copy of the expectations in the file with the tests removed."""
1018         def without_rebaseline_modifier(expectation):
1019             return (expectation.filename == filename and
1020                     not (not expectation.is_invalid() and
1021                          expectation.name in except_these_tests and
1022                          'rebaseline' in expectation.parsed_modifiers))
1023
1024         return self.list_to_string(filter(without_rebaseline_modifier, self._expectations), reconstitute_only_these=[])
1025
1026     def _add_expectations(self, expectation_list):
1027         for expectation_line in expectation_list:
1028             if self._force_expectations_pass:
1029                 expectation_line.expectations = ['PASS']
1030                 expectation_line.parsed_expectations = set([PASS])
1031
1032             elif not expectation_line.expectations:
1033                 continue
1034
1035             if self._is_lint_mode or self._test_config in expectation_line.matching_configurations:
1036                 self._model.add_expectation_line(expectation_line)
1037
1038     def add_skipped_tests(self, tests_to_skip):
1039         if not tests_to_skip:
1040             return
1041         for test in self._expectations:
1042             if test.name and test.name in tests_to_skip:
1043                 test.warnings.append('%s:%d %s is also in a Skipped file.' % (test.filename, test.line_number, test.name))
1044
1045         for test_name in tests_to_skip:
1046             expectation_line = self._parser.expectation_for_skipped_test(test_name)
1047             self._model.add_expectation_line(expectation_line, in_skipped=True)
1048
1049     @staticmethod
1050     def list_to_string(expectation_lines, test_configuration_converter=None, reconstitute_only_these=None):
1051         def serialize(expectation_line):
1052             # If reconstitute_only_these is an empty list, we want to return original_string.
1053             # So we need to compare reconstitute_only_these to None, not just check if it's falsey.
1054             if reconstitute_only_these is None or expectation_line in reconstitute_only_these:
1055                 return expectation_line.to_string(test_configuration_converter)
1056             return expectation_line.original_string
1057
1058         def nones_out(expectation_line):
1059             return expectation_line is not None
1060
1061         return "\n".join(filter(nones_out, map(serialize, expectation_lines)))