047e4ae1157ec94534bb125bd7692d99c3387ad1
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 """A helper class for reading in and dealing with tests expectations
30 for layout tests.
31 """
32
33 import logging
34 import re
35
36 from webkitpy.layout_tests.models.test_configuration import TestConfigurationConverter
37
38 _log = logging.getLogger(__name__)
39
40
41 # Test expectation and modifier constants.
42 #
43 # FIXME: range() starts with 0 which makes if expectation checks harder
44 # as PASS is 0.
45 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, SKIP, WONTFIX,
46  SLOW, DUMPJSCONSOLELOGINSTDERR, REBASELINE, MISSING, FLAKY, NOW, NONE) = range(17)
47
48 # FIXME: Perhas these two routines should be part of the Port instead?
49 BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt')
50
51
52 class ParseError(Exception):
53     def __init__(self, warnings):
54         super(ParseError, self).__init__()
55         self.warnings = warnings
56
57     def __str__(self):
58         return '\n'.join(map(str, self.warnings))
59
60     def __repr__(self):
61         return 'ParseError(warnings=%s)' % self.warnings
62
63
64 class TestExpectationWarning(object):
65     def __init__(self, filename, line_number, line, error, test=None):
66         self.filename = filename
67         self.line_number = line_number
68         self.line = line
69         self.error = error
70         self.test = test
71
72         self.related_files = {}
73
74     def __str__(self):
75         return '{}:{} {} {}'.format(self.filename, self.line_number, self.error, self.test if self.test else self.line)
76
77
78 class TestExpectationParser(object):
79     """Provides parsing facilities for lines in the test_expectation.txt file."""
80
81     DUMMY_BUG_MODIFIER = "bug_dummy"
82     BUG_MODIFIER_PREFIX = 'bug'
83     BUG_MODIFIER_REGEX = 'bug\d+'
84     REBASELINE_MODIFIER = 'rebaseline'
85     PASS_EXPECTATION = 'pass'
86     SKIP_MODIFIER = 'skip'
87     SLOW_MODIFIER = 'slow'
88     DUMPJSCONSOLELOGINSTDERR_MODIFIER = 'dumpjsconsoleloginstderr'
89     WONTFIX_MODIFIER = 'wontfix'
90
91     TIMEOUT_EXPECTATION = 'timeout'
92
93     MISSING_BUG_WARNING = 'Test lacks BUG modifier.'
94
95     def __init__(self, port, full_test_list, allow_rebaseline_modifier, shorten_filename=lambda x: x):
96         self._port = port
97         self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros())
98         if full_test_list is None:
99             self._full_test_list = None
100         else:
101             self._full_test_list = set(full_test_list)
102         self._allow_rebaseline_modifier = allow_rebaseline_modifier
103         self._shorten_filename = shorten_filename
104
105     def parse(self, filename, expectations_string):
106         expectation_lines = []
107         line_number = 0
108         for line in expectations_string.split("\n"):
109             line_number += 1
110             test_expectation = self._tokenize_line(filename, line, line_number)
111             self._parse_line(test_expectation)
112             expectation_lines.append(test_expectation)
113         return expectation_lines
114
115     def expectation_for_skipped_test(self, test_name):
116         if not self._port.test_exists(test_name):
117             _log.warning('The following test %s from the Skipped list doesn\'t exist' % test_name)
118         expectation_line = TestExpectationLine()
119         expectation_line.original_string = test_name
120         expectation_line.modifiers = [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER]
121         # FIXME: It's not clear what the expectations for a skipped test should be; the expectations
122         # might be different for different entries in a Skipped file, or from the command line, or from
123         # only running parts of the tests. It's also not clear if it matters much.
124         expectation_line.modifiers.append(TestExpectationParser.WONTFIX_MODIFIER)
125         expectation_line.name = test_name
126         # FIXME: we should pass in a more descriptive string here.
127         expectation_line.filename = '<Skipped file>'
128         expectation_line.line_number = 0
129         expectation_line.expectations = [TestExpectationParser.PASS_EXPECTATION]
130         expectation_line.not_applicable_to_current_platform = True
131         self._parse_line(expectation_line)
132         return expectation_line
133
134     def _parse_line(self, expectation_line):
135         if not expectation_line.name:
136             return
137
138         if not self._check_test_exists(expectation_line):
139             return
140
141         expectation_line.is_file = self._port.test_isfile(expectation_line.name)
142         if expectation_line.is_file:
143             expectation_line.path = expectation_line.name
144         else:
145             expectation_line.path = self._port.normalize_test_name(expectation_line.name)
146
147         self._collect_matching_tests(expectation_line)
148
149         self._parse_modifiers(expectation_line)
150         self._parse_expectations(expectation_line)
151
152     def _parse_modifiers(self, expectation_line):
153         has_wontfix = False
154         has_bugid = False
155         parsed_specifiers = set()
156
157         modifiers = [modifier.lower() for modifier in expectation_line.modifiers]
158         expectations = [expectation.lower() for expectation in expectation_line.expectations]
159
160         if self.SLOW_MODIFIER in modifiers and self.TIMEOUT_EXPECTATION in expectations:
161             expectation_line.warnings.append('A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT.')
162
163         for modifier in modifiers:
164             if modifier in TestExpectations.MODIFIERS:
165                 expectation_line.parsed_modifiers.append(modifier)
166                 if modifier == self.WONTFIX_MODIFIER:
167                     has_wontfix = True
168             elif modifier.startswith(self.BUG_MODIFIER_PREFIX):
169                 has_bugid = True
170                 if re.match(self.BUG_MODIFIER_REGEX, modifier):
171                     expectation_line.warnings.append('BUG\d+ is not allowed, must be one of BUGWK\d+, or a non-numeric bug identifier.')
172                 else:
173                     expectation_line.parsed_bug_modifiers.append(modifier)
174             else:
175                 parsed_specifiers.add(modifier)
176
177         if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid and self._port.warn_if_bug_missing_in_test_expectations():
178             expectation_line.warnings.append(self.MISSING_BUG_WARNING)
179
180         if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in modifiers:
181             expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.')
182
183         expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings)
184
185     def _parse_expectations(self, expectation_line):
186         result = set()
187         for part in expectation_line.expectations:
188             expectation = TestExpectations.expectation_from_string(part)
189             if expectation is None:  # Careful, PASS is currently 0.
190                 expectation_line.warnings.append('Unsupported expectation: %s' % part)
191                 continue
192             result.add(expectation)
193         expectation_line.parsed_expectations = result
194
195     def _check_test_exists(self, expectation_line):
196         # WebKit's way of skipping tests is to add a -disabled suffix.
197         # So we should consider the path existing if the path or the
198         # -disabled version exists.
199         if not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled'):
200             # Log a warning here since you hit this case any
201             # time you update TestExpectations without syncing
202             # the LayoutTests directory
203             expectation_line.warnings.append('Path does not exist.')
204             expected_path = self._shorten_filename(self._port.abspath_for_test(expectation_line.name))
205             expectation_line.related_files[expected_path] = None
206             return False
207         return True
208
209     def _collect_matching_tests(self, expectation_line):
210         """Convert the test specification to an absolute, normalized
211         path and make sure directories end with the OS path separator."""
212
213         if not self._full_test_list:
214             expectation_line.matching_tests = [expectation_line.path]
215             return
216
217         if not expectation_line.is_file:
218             # this is a test category, return all the tests of the category.
219             expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)]
220             return
221
222         # this is a test file, do a quick check if it's in the
223         # full test suite.
224         if expectation_line.path in self._full_test_list:
225             expectation_line.matching_tests.append(expectation_line.path)
226
227     # FIXME: Update the original modifiers and remove this once the old syntax is gone.
228     _configuration_tokens_list = [
229         'Mac', 'SnowLeopard', 'Lion', 'MountainLion', 'Mavericks', 'Yosemite', 'ElCapitan', 'Sierra', 'HighSierra',
230         'Win', 'XP', 'Vista', 'Win7',
231         'Linux',
232         'Android',
233         'Release',
234         'Debug',
235     ]
236
237     _configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list)
238     _inverted_configuration_tokens = dict((value, name) for name, value in _configuration_tokens.iteritems())
239
240     # FIXME: Update the original modifiers list and remove this once the old syntax is gone.
241     _expectation_tokens = {
242         'Crash': 'CRASH',
243         'Failure': 'FAIL',
244         'ImageOnlyFailure': 'IMAGE',
245         'Missing': 'MISSING',
246         'Pass': 'PASS',
247         'Rebaseline': 'REBASELINE',
248         'Skip': 'SKIP',
249         'Slow': 'SLOW',
250         'Timeout': 'TIMEOUT',
251         'WontFix': 'WONTFIX',
252     }
253
254     _inverted_expectation_tokens = dict([(value, name) for name, value in _expectation_tokens.iteritems()] +
255                                         [('TEXT', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')])
256
257     # FIXME: Seems like these should be classmethods on TestExpectationLine instead of TestExpectationParser.
258     @classmethod
259     def _tokenize_line(cls, filename, expectation_string, line_number):
260         """Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format.
261
262         The new format for a test expectation line is:
263
264         [[bugs] [ "[" <configuration modifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>]
265
266         Any errant whitespace is not preserved.
267
268         """
269         expectation_line = TestExpectationLine()
270         expectation_line.original_string = expectation_string
271         expectation_line.filename = filename
272         expectation_line.line_number = line_number
273
274         comment_index = expectation_string.find("#")
275         if comment_index == -1:
276             comment_index = len(expectation_string)
277         else:
278             expectation_line.comment = expectation_string[comment_index + 1:]
279
280         remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip())
281         if len(remaining_string) == 0:
282             return expectation_line
283
284         # special-case parsing this so that we fail immediately instead of treating this as a test name
285         if remaining_string.startswith('//'):
286             expectation_line.warnings = ['use "#" instead of "//" for comments']
287             return expectation_line
288
289         bugs = []
290         modifiers = []
291         name = None
292         expectations = []
293         warnings = []
294
295         WEBKIT_BUG_PREFIX = 'webkit.org/b/'
296
297         tokens = remaining_string.split()
298         state = 'start'
299         for token in tokens:
300             if token.startswith(WEBKIT_BUG_PREFIX) or token.startswith('Bug('):
301                 if state != 'start':
302                     warnings.append('"%s" is not at the start of the line.' % token)
303                     break
304                 if token.startswith(WEBKIT_BUG_PREFIX):
305                     bugs.append(token.replace(WEBKIT_BUG_PREFIX, 'BUGWK'))
306                 else:
307                     match = re.match('Bug\((\w+)\)$', token)
308                     if not match:
309                         warnings.append('unrecognized bug identifier "%s"' % token)
310                         break
311                     else:
312                         bugs.append('BUG' + match.group(1).upper())
313             elif token.startswith('BUG'):
314                 warnings.append('unrecognized old-style bug identifier "%s"' % token)
315                 break
316             elif token == '[':
317                 if state == 'start':
318                     state = 'configuration'
319                 elif state == 'name_found':
320                     state = 'expectations'
321                 else:
322                     warnings.append('unexpected "["')
323                     break
324             elif token == ']':
325                 if state == 'configuration':
326                     state = 'name'
327                 elif state == 'expectations':
328                     state = 'done'
329                 else:
330                     warnings.append('unexpected "]"')
331                     break
332             elif token in ('//', ':', '='):
333                 warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token)
334                 break
335             elif state == 'configuration':
336                 modifiers.append(cls._configuration_tokens.get(token, token))
337             elif state == 'expectations':
338                 if token in ('Rebaseline', 'Skip', 'Slow', 'WontFix', 'DumpJSConsoleLogInStdErr'):
339                     modifiers.append(token.upper())
340                 elif token not in cls._expectation_tokens:
341                     warnings.append('Unrecognized expectation "%s"' % token)
342                 else:
343                     expectations.append(cls._expectation_tokens.get(token, token))
344             elif state == 'name_found':
345                 warnings.append('expecting "[", "#", or end of line instead of "%s"' % token)
346                 break
347             else:
348                 name = token
349                 state = 'name_found'
350
351         if not warnings:
352             if not name:
353                 warnings.append('Did not find a test name.')
354             elif state not in ('name_found', 'done'):
355                 warnings.append('Missing a "]"')
356
357         if 'WONTFIX' in modifiers and 'SKIP' not in modifiers and not expectations:
358             modifiers.append('SKIP')
359
360         if 'SKIP' in modifiers and expectations:
361             # FIXME: This is really a semantic warning and shouldn't be here. Remove when we drop the old syntax.
362             warnings.append('A test marked Skip must not have other expectations.')
363         elif not expectations:
364             # FIXME: We can probably simplify this adding 'SKIP' if modifiers is empty
365             if 'SKIP' not in modifiers and 'REBASELINE' not in modifiers and 'SLOW' not in modifiers and 'DUMPJSCONSOLELOGINSTDERR' not in modifiers:
366                 modifiers.append('SKIP')
367             expectations = ['PASS']
368
369         # FIXME: expectation line should just store bugs and modifiers separately.
370         expectation_line.modifiers = bugs + modifiers
371         expectation_line.expectations = expectations
372         expectation_line.name = name
373         expectation_line.warnings = warnings
374         return expectation_line
375
376     @classmethod
377     def _split_space_separated(cls, space_separated_string):
378         """Splits a space-separated string into an array."""
379         return [part.strip() for part in space_separated_string.strip().split(' ')]
380
381
382 class TestExpectationLine(object):
383     """Represents a line in test expectations file."""
384
385     def __init__(self):
386         """Initializes a blank-line equivalent of an expectation."""
387         self.original_string = None
388         self.filename = None  # this is the path to the expectations file for this line
389         self.line_number = None
390         self.name = None  # this is the path in the line itself
391         self.path = None  # this is the normpath of self.name
392         self.modifiers = []
393         self.parsed_modifiers = []
394         self.parsed_bug_modifiers = []
395         self.matching_configurations = set()
396         self.expectations = []
397         self.parsed_expectations = set()
398         self.comment = None
399         self.matching_tests = []
400         self.warnings = []
401         self.related_files = {}  # Dictionary of files to lines number in that file which may have caused the list of warnings.
402         self.not_applicable_to_current_platform = False
403
404     def is_invalid(self):
405         return self.warnings and self.warnings != [TestExpectationParser.MISSING_BUG_WARNING]
406
407     def is_flaky(self):
408         return len(self.parsed_expectations) > 1
409
410     @property
411     def expected_behavior(self):
412         expectations = self.expectations
413         if "SLOW" in self.modifiers:
414             expectations += ["SLOW"]
415
416         if "SKIP" in self.modifiers:
417             expectations = ["SKIP"]
418         elif "WONTFIX" in self.modifiers:
419             expectations = ["WONTFIX"]
420         elif "CRASH" in self.modifiers:
421             expectations += ["CRASH"]
422
423         return expectations
424
425     @staticmethod
426     def create_passing_expectation(test):
427         expectation_line = TestExpectationLine()
428         expectation_line.name = test
429         expectation_line.path = test
430         expectation_line.parsed_expectations = set([PASS])
431         expectation_line.expectations = ['PASS']
432         expectation_line.matching_tests = [test]
433         return expectation_line
434
435     def to_string(self, test_configuration_converter, include_modifiers=True, include_expectations=True, include_comment=True):
436         parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
437
438         if self.is_invalid():
439             return self.original_string or ''
440
441         if self.name is None:
442             return '' if self.comment is None else "#%s" % self.comment
443
444         if test_configuration_converter and self.parsed_bug_modifiers:
445             specifiers_list = test_configuration_converter.to_specifiers_list(self.matching_configurations)
446             result = []
447             for specifiers in specifiers_list:
448                 # FIXME: this is silly that we join the modifiers and then immediately split them.
449                 modifiers = self._serialize_parsed_modifiers(test_configuration_converter, specifiers).split()
450                 expectations = self._serialize_parsed_expectations(parsed_expectation_to_string).split()
451                 result.append(self._format_line(modifiers, self.name, expectations, self.comment))
452             return "\n".join(result) if result else None
453
454         return self._format_line(self.modifiers, self.name, self.expectations, self.comment,
455             include_modifiers, include_expectations, include_comment)
456
457     def to_csv(self):
458         # Note that this doesn't include the comments.
459         return '%s,%s,%s' % (self.name, ' '.join(self.modifiers), ' '.join(self.expectations))
460
461     def _serialize_parsed_expectations(self, parsed_expectation_to_string):
462         result = []
463         for index in TestExpectations.EXPECTATION_ORDER:
464             if index in self.parsed_expectations:
465                 result.append(parsed_expectation_to_string[index])
466         return ' '.join(result)
467
468     def _serialize_parsed_modifiers(self, test_configuration_converter, specifiers):
469         result = []
470         if self.parsed_bug_modifiers:
471             result.extend(sorted(self.parsed_bug_modifiers))
472         result.extend(sorted(self.parsed_modifiers))
473         result.extend(test_configuration_converter.specifier_sorter().sort_specifiers(specifiers))
474         return ' '.join(result)
475
476     @staticmethod
477     def _format_line(modifiers, name, expectations, comment, include_modifiers=True, include_expectations=True, include_comment=True):
478         bugs = []
479         new_modifiers = []
480         new_expectations = []
481         for modifier in modifiers:
482             modifier = modifier.upper()
483             if modifier.startswith('BUGWK'):
484                 bugs.append('webkit.org/b/' + modifier.replace('BUGWK', ''))
485             elif modifier.startswith('BUG'):
486                 # FIXME: we should preserve case once we can drop the old syntax.
487                 bugs.append('Bug(' + modifier[3:].lower() + ')')
488             elif modifier in ('SLOW', 'SKIP', 'REBASELINE', 'WONTFIX', 'DUMPJSCONSOLELOGINSTDERR'):
489                 new_expectations.append(TestExpectationParser._inverted_expectation_tokens.get(modifier))
490             else:
491                 new_modifiers.append(TestExpectationParser._inverted_configuration_tokens.get(modifier, modifier))
492
493         for expectation in expectations:
494             expectation = expectation.upper()
495             new_expectations.append(TestExpectationParser._inverted_expectation_tokens.get(expectation, expectation))
496
497         result = ''
498         if include_modifiers and (bugs or new_modifiers):
499             if bugs:
500                 result += ' '.join(bugs) + ' '
501             if new_modifiers:
502                 result += '[ %s ] ' % ' '.join(new_modifiers)
503         result += name
504         if include_expectations and new_expectations and set(new_expectations) != set(['Skip', 'Pass']):
505             result += ' [ %s ]' % ' '.join(sorted(set(new_expectations)))
506         if include_comment and comment is not None:
507             result += " #%s" % comment
508         return result
509
510
511 # FIXME: Refactor API to be a proper CRUD.
512 class TestExpectationsModel(object):
513     """Represents relational store of all expectations and provides CRUD semantics to manage it."""
514
515     def __init__(self, shorten_filename=lambda x: x):
516         # Maps a test to its list of expectations.
517         self._test_to_expectations = {}
518
519         # Maps a test to list of its modifiers (string values)
520         self._test_to_modifiers = {}
521
522         # Maps a test to a TestExpectationLine instance.
523         self._test_to_expectation_line = {}
524
525         self._modifier_to_tests = self._dict_of_sets(TestExpectations.MODIFIERS)
526         self._expectation_to_tests = self._dict_of_sets(TestExpectations.EXPECTATIONS)
527         self._timeline_to_tests = self._dict_of_sets(TestExpectations.TIMELINES)
528         self._result_type_to_tests = self._dict_of_sets(TestExpectations.RESULT_TYPES)
529
530         self._shorten_filename = shorten_filename
531
532     def _dict_of_sets(self, strings_to_constants):
533         """Takes a dict of strings->constants and returns a dict mapping
534         each constant to an empty set."""
535         d = {}
536         for c in strings_to_constants.values():
537             d[c] = set()
538         return d
539
540     def get_test_set(self, modifier, expectation=None, include_skips=True):
541         if expectation is None:
542             tests = self._modifier_to_tests[modifier]
543         else:
544             tests = (self._expectation_to_tests[expectation] &
545                 self._modifier_to_tests[modifier])
546
547         if not include_skips:
548             tests = tests - self.get_test_set(SKIP, expectation)
549
550         return tests
551
552     def get_test_set_for_keyword(self, keyword):
553         # FIXME: get_test_set() is an awkward public interface because it requires
554         # callers to know the difference between modifiers and expectations. We
555         # should replace that with this where possible.
556         expectation_enum = TestExpectations.EXPECTATIONS.get(keyword.lower(), None)
557         if expectation_enum is not None:
558             return self._expectation_to_tests[expectation_enum]
559         modifier_enum = TestExpectations.MODIFIERS.get(keyword.lower(), None)
560         if modifier_enum is not None:
561             return self._modifier_to_tests[modifier_enum]
562
563         # We must not have an index on this modifier.
564         matching_tests = set()
565         for test, modifiers in self._test_to_modifiers.iteritems():
566             if keyword.lower() in modifiers:
567                 matching_tests.add(test)
568         return matching_tests
569
570     def get_tests_with_result_type(self, result_type):
571         return self._result_type_to_tests[result_type]
572
573     def get_tests_with_timeline(self, timeline):
574         return self._timeline_to_tests[timeline]
575
576     def get_modifiers(self, test):
577         """This returns modifiers for the given test (the modifiers plus the BUGXXXX identifier). This is used by the LTTF dashboard."""
578         return self._test_to_modifiers[test]
579
580     def has_modifier(self, test, modifier):
581         return test in self._modifier_to_tests[modifier]
582
583     def has_keyword(self, test, keyword):
584         return (keyword.upper() in self.get_expectations_string(test) or
585                 keyword.lower() in self.get_modifiers(test))
586
587     def has_test(self, test):
588         return test in self._test_to_expectation_line
589
590     def get_expectation_line(self, test):
591         return self._test_to_expectation_line.get(test)
592
593     def get_expectations(self, test):
594         return self._test_to_expectations[test]
595
596     def get_expectations_string(self, test):
597         """Returns the expectatons for the given test as an uppercase string.
598         If there are no expectations for the test, then "PASS" is returned."""
599         try:
600             expectations = self.get_expectations(test)
601         except:
602             return "PASS"
603         retval = []
604
605         for expectation in expectations:
606             retval.append(self.expectation_to_string(expectation))
607
608         return " ".join(retval)
609
610     def expectation_to_string(self, expectation):
611         """Return the uppercased string equivalent of a given expectation."""
612         for item in TestExpectations.EXPECTATIONS.items():
613             if item[1] == expectation:
614                 return item[0].upper()
615         raise ValueError(expectation)
616
617     def add_expectation_line(self, expectation_line, in_skipped=False):
618         """Returns a list of warnings encountered while matching modifiers."""
619
620         if expectation_line.is_invalid():
621             return
622
623         for test in expectation_line.matching_tests:
624             if not in_skipped and self._already_seen_better_match(test, expectation_line):
625                 continue
626
627             self._clear_expectations_for_test(test)
628             self._test_to_expectation_line[test] = expectation_line
629             self._add_test(test, expectation_line)
630
631     def _add_test(self, test, expectation_line):
632         """Sets the expected state for a given test.
633
634         This routine assumes the test has not been added before. If it has,
635         use _clear_expectations_for_test() to reset the state prior to
636         calling this."""
637         self._test_to_expectations[test] = expectation_line.parsed_expectations
638         for expectation in expectation_line.parsed_expectations:
639             self._expectation_to_tests[expectation].add(test)
640
641         self._test_to_modifiers[test] = expectation_line.modifiers
642         for modifier in expectation_line.parsed_modifiers:
643             mod_value = TestExpectations.MODIFIERS[modifier]
644             self._modifier_to_tests[mod_value].add(test)
645
646         if TestExpectationParser.WONTFIX_MODIFIER in expectation_line.parsed_modifiers:
647             self._timeline_to_tests[WONTFIX].add(test)
648         else:
649             self._timeline_to_tests[NOW].add(test)
650
651         if TestExpectationParser.SKIP_MODIFIER in expectation_line.parsed_modifiers:
652             self._result_type_to_tests[SKIP].add(test)
653         elif expectation_line.parsed_expectations == set([PASS]):
654             self._result_type_to_tests[PASS].add(test)
655         elif expectation_line.is_flaky():
656             self._result_type_to_tests[FLAKY].add(test)
657         else:
658             # FIXME: What is this?
659             self._result_type_to_tests[FAIL].add(test)
660
661     def _clear_expectations_for_test(self, test):
662         """Remove prexisting expectations for this test.
663         This happens if we are seeing a more precise path
664         than a previous listing.
665         """
666         if self.has_test(test):
667             self._test_to_expectations.pop(test, '')
668             self._remove_from_sets(test, self._expectation_to_tests)
669             self._remove_from_sets(test, self._modifier_to_tests)
670             self._remove_from_sets(test, self._timeline_to_tests)
671             self._remove_from_sets(test, self._result_type_to_tests)
672
673     def _remove_from_sets(self, test, dict_of_sets_of_tests):
674         """Removes the given test from the sets in the dictionary.
675
676         Args:
677           test: test to look for
678           dict: dict of sets of files"""
679         for set_of_tests in dict_of_sets_of_tests.itervalues():
680             if test in set_of_tests:
681                 set_of_tests.remove(test)
682
683     def _already_seen_better_match(self, test, expectation_line):
684         """Returns whether we've seen a better match already in the file.
685
686         Returns True if we've already seen a expectation_line.name that matches more of the test
687             than this path does
688         """
689         # FIXME: See comment below about matching test configs and specificity.
690         if not self.has_test(test):
691             # We've never seen this test before.
692             return False
693
694         prev_expectation_line = self._test_to_expectation_line[test]
695
696         if prev_expectation_line.filename != expectation_line.filename:
697             # We've moved on to a new expectation file, which overrides older ones.
698             return False
699
700         if len(prev_expectation_line.path) > len(expectation_line.path):
701             # The previous path matched more of the test.
702             return True
703
704         if len(prev_expectation_line.path) < len(expectation_line.path):
705             # This path matches more of the test.
706             return False
707
708         # At this point we know we have seen a previous exact match on this
709         # base path, so we need to check the two sets of modifiers.
710
711         # FIXME: This code was originally designed to allow lines that matched
712         # more modifiers to override lines that matched fewer modifiers.
713         # However, we currently view these as errors.
714         #
715         # To use the "more modifiers wins" policy, change the errors for overrides
716         # to be warnings and return False".
717         shortened_expectation_filename = self._shorten_filename(expectation_line.filename)
718         shortened_previous_expectation_filename = self._shorten_filename(prev_expectation_line.filename)
719
720         if prev_expectation_line.matching_configurations == expectation_line.matching_configurations:
721             expectation_line.warnings.append('Duplicate or ambiguous entry lines %s:%d and %s:%d.' % (
722                 shortened_previous_expectation_filename, prev_expectation_line.line_number,
723                 shortened_expectation_filename, expectation_line.line_number))
724
725         elif prev_expectation_line.matching_configurations >= expectation_line.matching_configurations:
726             expectation_line.warnings.append('More specific entry for %s on line %s:%d overrides line %s:%d.' % (expectation_line.name,
727                 shortened_previous_expectation_filename, prev_expectation_line.line_number,
728                 shortened_expectation_filename, expectation_line.line_number))
729             # FIXME: return False if we want more specific to win.
730
731         elif prev_expectation_line.matching_configurations <= expectation_line.matching_configurations:
732             expectation_line.warnings.append('More specific entry for %s on line %s:%d overrides line %s:%d.' % (expectation_line.name,
733                 shortened_expectation_filename, expectation_line.line_number,
734                 shortened_previous_expectation_filename, prev_expectation_line.line_number))
735
736         elif prev_expectation_line.matching_configurations & expectation_line.matching_configurations:
737             expectation_line.warnings.append('Entries for %s on lines %s:%d and %s:%d match overlapping sets of configurations.' % (expectation_line.name,
738                 shortened_previous_expectation_filename, prev_expectation_line.line_number,
739                 shortened_expectation_filename, expectation_line.line_number))
740
741         else:
742             # Configuration sets are disjoint.
743             return False
744
745         # Missing files will be 'None'. It should be impossible to have a missing file which also has a line associated with it.
746         assert shortened_previous_expectation_filename not in expectation_line.related_files or expectation_line.related_files[shortened_previous_expectation_filename] is not None
747         expectation_line.related_files[shortened_previous_expectation_filename] = expectation_line.related_files.get(shortened_previous_expectation_filename, []).append(prev_expectation_line.line_number)
748
749         assert shortened_expectation_filename not in prev_expectation_line.related_files or prev_expectation_line.related_files[shortened_expectation_filename] is not None
750         prev_expectation_line.related_files[shortened_expectation_filename] = prev_expectation_line.related_files.get(shortened_expectation_filename, []).append(expectation_line.line_number)
751
752         return True
753
754
755 class TestExpectations(object):
756     """Test expectations consist of lines with specifications of what
757     to expect from layout test cases. The test cases can be directories
758     in which case the expectations apply to all test cases in that
759     directory and any subdirectory. The format is along the lines of:
760
761       LayoutTests/js/fixme.js [ Failure ]
762       LayoutTests/js/flaky.js [ Failure Pass ]
763       LayoutTests/js/crash.js [ Crash Failure Pass Timeout ]
764       ...
765
766     To add modifiers:
767       LayoutTests/js/no-good.js
768       [ Debug ] LayoutTests/js/no-good.js [ Pass Timeout ]
769       [ Debug ] LayoutTests/js/no-good.js [ Pass Skip Timeout ]
770       [ Linux Debug ] LayoutTests/js/no-good.js [ Pass Skip Timeout ]
771       [ Linux Win ] LayoutTests/js/no-good.js [ Pass Skip Timeout ]
772
773     Skip: Doesn't run the test.
774     Slow: The test takes a long time to run, but does not timeout indefinitely.
775     WontFix: For tests that we never intend to pass on a given platform (treated like Skip).
776
777     Notes:
778       -A test cannot be both SLOW and TIMEOUT
779       -A test can be included twice, but not via the same path.
780       -If a test is included twice, then the more precise path wins.
781       -CRASH tests cannot be WONTFIX
782     """
783
784     # FIXME: Update to new syntax once the old format is no longer supported.
785     EXPECTATIONS = {'pass': PASS,
786                     'audio': AUDIO,
787                     'fail': FAIL,
788                     'image': IMAGE,
789                     'image+text': IMAGE_PLUS_TEXT,
790                     'text': TEXT,
791                     'timeout': TIMEOUT,
792                     'crash': CRASH,
793                     'missing': MISSING,
794                     'skip': SKIP}
795
796     # (aggregated by category, pass/fail/skip, type)
797     EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped',
798                                 PASS: 'passes',
799                                 FAIL: 'failures',
800                                 IMAGE: 'image-only failures',
801                                 TEXT: 'text-only failures',
802                                 IMAGE_PLUS_TEXT: 'image and text failures',
803                                 AUDIO: 'audio failures',
804                                 CRASH: 'crashes',
805                                 TIMEOUT: 'timeouts',
806                                 MISSING: 'missing results'}
807
808     EXPECTATION_ORDER = (PASS, CRASH, TIMEOUT, MISSING, FAIL, IMAGE, SKIP)
809
810     BUILD_TYPES = ('debug', 'release')
811
812     MODIFIERS = {TestExpectationParser.SKIP_MODIFIER: SKIP,
813                  TestExpectationParser.WONTFIX_MODIFIER: WONTFIX,
814                  TestExpectationParser.SLOW_MODIFIER: SLOW,
815                  TestExpectationParser.DUMPJSCONSOLELOGINSTDERR_MODIFIER: DUMPJSCONSOLELOGINSTDERR,
816                  TestExpectationParser.REBASELINE_MODIFIER: REBASELINE,
817                  'none': NONE}
818
819     TIMELINES = {TestExpectationParser.WONTFIX_MODIFIER: WONTFIX,
820                  'now': NOW}
821
822     RESULT_TYPES = {'skip': SKIP,
823                     'pass': PASS,
824                     'fail': FAIL,
825                     'flaky': FLAKY}
826
827     @classmethod
828     def expectation_from_string(cls, string):
829         assert(' ' not in string)  # This only handles one expectation at a time.
830         return cls.EXPECTATIONS.get(string.lower())
831
832     @staticmethod
833     def result_was_expected(result, expected_results, test_needs_rebaselining, test_is_skipped):
834         """Returns whether we got a result we were expecting.
835         Args:
836             result: actual result of a test execution
837             expected_results: set of results listed in test_expectations
838             test_needs_rebaselining: whether test was marked as REBASELINE
839             test_is_skipped: whether test was marked as SKIP"""
840         if result in expected_results:
841             return True
842         if result in (TEXT, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected_results):
843             return True
844         if result == MISSING and test_needs_rebaselining:
845             return True
846         if result == SKIP and test_is_skipped:
847             return True
848         return False
849
850     @staticmethod
851     def remove_pixel_failures(expected_results):
852         """Returns a copy of the expected results for a test, except that we
853         drop any pixel failures and return the remaining expectations. For example,
854         if we're not running pixel tests, then tests expected to fail as IMAGE
855         will PASS."""
856         expected_results = expected_results.copy()
857         if IMAGE in expected_results:
858             expected_results.remove(IMAGE)
859             expected_results.add(PASS)
860         return expected_results
861
862     @staticmethod
863     def has_pixel_failures(actual_results):
864         return IMAGE in actual_results or FAIL in actual_results
865
866     @staticmethod
867     def suffixes_for_expectations(expectations):
868         suffixes = set()
869         if IMAGE in expectations:
870             suffixes.add('png')
871         if FAIL in expectations:
872             suffixes.add('txt')
873             suffixes.add('png')
874             suffixes.add('wav')
875         return set(suffixes)
876
877     def __init__(self, port, tests=None, include_generic=True, include_overrides=True, expectations_to_lint=None, force_expectations_pass=False):
878         self._full_test_list = tests
879         self._test_config = port.test_configuration()
880         self._is_lint_mode = expectations_to_lint is not None
881         self._model = TestExpectationsModel(self._shorten_filename)
882         self._parser = TestExpectationParser(port, tests, self._is_lint_mode, self._shorten_filename)
883         self._port = port
884         self._skipped_tests_warnings = []
885         self._expectations = []
886         self._force_expectations_pass = force_expectations_pass
887         self._include_generic = include_generic
888         self._include_overrides = include_overrides
889         self._expectations_to_lint = expectations_to_lint
890
891     def readable_filename_and_line_number(self, line):
892         if line.not_applicable_to_current_platform:
893             return "(skipped for this platform)"
894         if not line.filename:
895             return ''
896         if line.filename.startswith(self._port.path_from_webkit_base()):
897             return '{}:{}'.format(self._port.host.filesystem.relpath(line.filename, self._port.path_from_webkit_base()), line.line_number)
898         return '{}:{}'.format(line.filename, line.line_number)
899
900     def parse_generic_expectations(self):
901         if self._port.path_to_generic_test_expectations_file() in self._expectations_dict:
902             if self._include_generic:
903                 expectations = self._parser.parse(self._expectations_dict.keys()[self._expectations_dict_index], self._expectations_dict.values()[self._expectations_dict_index])
904                 self._add_expectations(expectations)
905                 self._expectations += expectations
906             self._expectations_dict_index += 1
907
908     def parse_default_port_expectations(self):
909         if len(self._expectations_dict) > self._expectations_dict_index:
910             expectations = self._parser.parse(self._expectations_dict.keys()[self._expectations_dict_index], self._expectations_dict.values()[self._expectations_dict_index])
911             self._add_expectations(expectations)
912             self._expectations += expectations
913             self._expectations_dict_index += 1
914
915     def parse_override_expectations(self):
916         while len(self._expectations_dict) > self._expectations_dict_index and self._include_overrides:
917             expectations = self._parser.parse(self._expectations_dict.keys()[self._expectations_dict_index], self._expectations_dict.values()[self._expectations_dict_index])
918             self._add_expectations(expectations)
919             self._expectations += expectations
920             self._expectations_dict_index += 1
921
922     def parse_all_expectations(self):
923         self._expectations_dict = self._expectations_to_lint or self._port.expectations_dict()
924         self._expectations_dict_index = 0
925
926         self._has_warnings = False
927
928         self.parse_generic_expectations()
929         self.parse_default_port_expectations()
930         self.parse_override_expectations()
931
932         # FIXME: move ignore_tests into port.skipped_layout_tests()
933         self.add_skipped_tests(self._port.skipped_layout_tests(self._full_test_list).union(set(self._port.get_option('ignore_tests', []))))
934
935         self._report_warnings()
936         self._process_tests_without_expectations()
937
938     # TODO(ojan): Allow for removing skipped tests when getting the list of
939     # tests to run, but not when getting metrics.
940     def model(self):
941         return self._model
942
943     def get_rebaselining_failures(self):
944         return self._model.get_test_set(REBASELINE)
945
946     def matches_an_expected_result(self, test, result, pixel_tests_are_enabled):
947         expected_results = self._model.get_expectations(test)
948         if not pixel_tests_are_enabled:
949             expected_results = self.remove_pixel_failures(expected_results)
950         return self.result_was_expected(result,
951                                    expected_results,
952                                    self.is_rebaselining(test),
953                                    self._model.has_modifier(test, SKIP))
954
955     def is_rebaselining(self, test):
956         return self._model.has_modifier(test, REBASELINE)
957
958     def _shorten_filename(self, filename):
959         if filename.startswith(self._port.path_from_webkit_base()):
960             return self._port.host.filesystem.relpath(filename, self._port.path_from_webkit_base())
961         return filename
962
963     def _report_warnings(self):
964         warnings = []
965         for expectation in self._expectations:
966             for warning in expectation.warnings:
967                 warning = TestExpectationWarning(
968                     self._shorten_filename(expectation.filename), expectation.line_number,
969                     expectation.original_string, warning, expectation.name if expectation.expectations else None)
970                 warning.related_files = expectation.related_files
971                 warnings.append(warning)
972
973         if warnings:
974             self._has_warnings = True
975             if self._is_lint_mode:
976                 raise ParseError(warnings)
977             _log.warning('--lint-test-files warnings:')
978             for warning in warnings:
979                 _log.warning(warning)
980             _log.warning('')
981
982     def _process_tests_without_expectations(self):
983         if self._full_test_list:
984             for test in self._full_test_list:
985                 if not self._model.has_test(test):
986                     self._model.add_expectation_line(TestExpectationLine.create_passing_expectation(test))
987
988     def has_warnings(self):
989         return self._has_warnings
990
991     def remove_configuration_from_test(self, test, test_configuration):
992         expectations_to_remove = []
993         modified_expectations = []
994
995         for expectation in self._expectations:
996             if expectation.name != test or expectation.is_flaky() or not expectation.parsed_expectations:
997                 continue
998             if iter(expectation.parsed_expectations).next() not in (FAIL, IMAGE):
999                 continue
1000             if test_configuration not in expectation.matching_configurations:
1001                 continue
1002
1003             expectation.matching_configurations.remove(test_configuration)
1004             if expectation.matching_configurations:
1005                 modified_expectations.append(expectation)
1006             else:
1007                 expectations_to_remove.append(expectation)
1008
1009         for expectation in expectations_to_remove:
1010             self._expectations.remove(expectation)
1011
1012         return self.list_to_string(self._expectations, self._parser._test_configuration_converter, modified_expectations)
1013
1014     def remove_rebaselined_tests(self, except_these_tests, filename):
1015         """Returns a copy of the expectations in the file with the tests removed."""
1016         def without_rebaseline_modifier(expectation):
1017             return (expectation.filename == filename and
1018                     not (not expectation.is_invalid() and
1019                          expectation.name in except_these_tests and
1020                          'rebaseline' in expectation.parsed_modifiers))
1021
1022         return self.list_to_string(filter(without_rebaseline_modifier, self._expectations), reconstitute_only_these=[])
1023
1024     def _add_expectations(self, expectation_list):
1025         for expectation_line in expectation_list:
1026             if self._force_expectations_pass:
1027                 expectation_line.expectations = ['PASS']
1028                 expectation_line.parsed_expectations = set([PASS])
1029
1030             elif not expectation_line.expectations:
1031                 continue
1032
1033             if self._is_lint_mode or self._test_config in expectation_line.matching_configurations:
1034                 self._model.add_expectation_line(expectation_line)
1035
1036     def add_skipped_tests(self, tests_to_skip):
1037         if not tests_to_skip:
1038             return
1039         for test in self._expectations:
1040             if test.name and test.name in tests_to_skip:
1041                 test.warnings.append('%s:%d %s is also in a Skipped file.' % (test.filename, test.line_number, test.name))
1042
1043         for test_name in tests_to_skip:
1044             expectation_line = self._parser.expectation_for_skipped_test(test_name)
1045             self._model.add_expectation_line(expectation_line, in_skipped=True)
1046
1047     @staticmethod
1048     def list_to_string(expectation_lines, test_configuration_converter=None, reconstitute_only_these=None):
1049         def serialize(expectation_line):
1050             # If reconstitute_only_these is an empty list, we want to return original_string.
1051             # So we need to compare reconstitute_only_these to None, not just check if it's falsey.
1052             if reconstitute_only_these is None or expectation_line in reconstitute_only_these:
1053                 return expectation_line.to_string(test_configuration_converter)
1054             return expectation_line.original_string
1055
1056         def nones_out(expectation_line):
1057             return expectation_line is not None
1058
1059         return "\n".join(filter(nones_out, map(serialize, expectation_lines)))