run-webkit-tests prints confusing messages when test expectations list results that...
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 """A helper class for reading in and dealing with tests expectations
30 for layout tests.
31 """
32
33 import logging
34 import re
35
36 from webkitpy.layout_tests.models.test_configuration import TestConfigurationConverter
37
38 _log = logging.getLogger(__name__)
39
40
41 # Test expectation and modifier constants.
42 #
43 # FIXME: range() starts with 0 which makes if expectation checks harder
44 # as PASS is 0.
45 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, SKIP, WONTFIX,
46  SLOW, LEAK, DUMPJSCONSOLELOGINSTDERR, REBASELINE, MISSING, FLAKY, NOW, NONE) = range(18)
47
48 # FIXME: Perhas these two routines should be part of the Port instead?
49 BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt')
50
51
52 class ParseError(Exception):
53     def __init__(self, warnings):
54         super(ParseError, self).__init__()
55         self.warnings = warnings
56
57     def __str__(self):
58         return '\n'.join(map(str, self.warnings))
59
60     def __repr__(self):
61         return 'ParseError(warnings=%s)' % self.warnings
62
63
64 class TestExpectationWarning(object):
65     def __init__(self, filename, line_number, line, error, test=None):
66         self.filename = filename
67         self.line_number = line_number
68         self.line = line
69         self.error = error
70         self.test = test
71
72         self.related_files = {}
73
74     def __str__(self):
75         return '{}:{} {} {}'.format(self.filename, self.line_number, self.error, self.test if self.test else self.line)
76
77
78 class TestExpectationParser(object):
79     """Provides parsing facilities for lines in the test_expectation.txt file."""
80
81     DUMMY_BUG_MODIFIER = "bug_dummy"
82     BUG_MODIFIER_PREFIX = 'bug'
83     BUG_MODIFIER_REGEX = 'bug\d+'
84     REBASELINE_MODIFIER = 'rebaseline'
85     PASS_EXPECTATION = 'pass'
86     SKIP_MODIFIER = 'skip'
87     SLOW_MODIFIER = 'slow'
88     DUMPJSCONSOLELOGINSTDERR_MODIFIER = 'dumpjsconsoleloginstderr'
89     WONTFIX_MODIFIER = 'wontfix'
90
91     TIMEOUT_EXPECTATION = 'timeout'
92
93     MISSING_BUG_WARNING = 'Test lacks BUG modifier.'
94
95     def __init__(self, port, full_test_list, allow_rebaseline_modifier, shorten_filename=lambda x: x):
96         self._port = port
97         self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros())
98         if full_test_list is None:
99             self._full_test_list = None
100         else:
101             self._full_test_list = set(full_test_list)
102         self._allow_rebaseline_modifier = allow_rebaseline_modifier
103         self._shorten_filename = shorten_filename
104
105     def parse(self, filename, expectations_string):
106         expectation_lines = []
107         line_number = 0
108         for line in expectations_string.split("\n"):
109             line_number += 1
110             test_expectation = self._tokenize_line(filename, line, line_number)
111             self._parse_line(test_expectation)
112             expectation_lines.append(test_expectation)
113         return expectation_lines
114
115     def expectation_for_skipped_test(self, test_name):
116         if not self._port.test_exists(test_name):
117             _log.warning('The following test %s from the Skipped list doesn\'t exist' % test_name)
118         expectation_line = TestExpectationLine()
119         expectation_line.original_string = test_name
120         expectation_line.modifiers = [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER]
121         # FIXME: It's not clear what the expectations for a skipped test should be; the expectations
122         # might be different for different entries in a Skipped file, or from the command line, or from
123         # only running parts of the tests. It's also not clear if it matters much.
124         expectation_line.modifiers.append(TestExpectationParser.WONTFIX_MODIFIER)
125         expectation_line.name = test_name
126         # FIXME: we should pass in a more descriptive string here.
127         expectation_line.filename = '<Skipped file>'
128         expectation_line.line_number = 0
129         expectation_line.expectations = [TestExpectationParser.PASS_EXPECTATION]
130         expectation_line.not_applicable_to_current_platform = True
131         self._parse_line(expectation_line)
132         return expectation_line
133
134     def _parse_line(self, expectation_line):
135         if not expectation_line.name:
136             return
137
138         if not self._check_test_exists(expectation_line):
139             return
140
141         expectation_line.is_file = self._port.test_isfile(expectation_line.name)
142         if expectation_line.is_file:
143             expectation_line.path = expectation_line.name
144         else:
145             expectation_line.path = self._port.normalize_test_name(expectation_line.name)
146
147         self._collect_matching_tests(expectation_line)
148
149         self._parse_modifiers(expectation_line)
150         self._parse_expectations(expectation_line)
151
152     def _parse_modifiers(self, expectation_line):
153         has_wontfix = False
154         has_bugid = False
155         parsed_specifiers = set()
156
157         modifiers = [modifier.lower() for modifier in expectation_line.modifiers]
158         expectations = [expectation.lower() for expectation in expectation_line.expectations]
159
160         if self.SLOW_MODIFIER in modifiers and self.TIMEOUT_EXPECTATION in expectations:
161             expectation_line.warnings.append('A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT.')
162
163         for modifier in modifiers:
164             if modifier in TestExpectations.MODIFIERS:
165                 expectation_line.parsed_modifiers.append(modifier)
166                 if modifier == self.WONTFIX_MODIFIER:
167                     has_wontfix = True
168             elif modifier.startswith(self.BUG_MODIFIER_PREFIX):
169                 has_bugid = True
170                 if re.match(self.BUG_MODIFIER_REGEX, modifier):
171                     expectation_line.warnings.append('BUG\d+ is not allowed, must be one of BUGWK\d+, or a non-numeric bug identifier.')
172                 else:
173                     expectation_line.parsed_bug_modifiers.append(modifier)
174             else:
175                 parsed_specifiers.add(modifier)
176
177         if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid and self._port.warn_if_bug_missing_in_test_expectations():
178             expectation_line.warnings.append(self.MISSING_BUG_WARNING)
179
180         if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in modifiers:
181             expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.')
182
183         expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings)
184
185     def _parse_expectations(self, expectation_line):
186         result = set()
187         for part in expectation_line.expectations:
188             expectation = TestExpectations.expectation_from_string(part)
189             if expectation is None:  # Careful, PASS is currently 0.
190                 expectation_line.warnings.append('Unsupported expectation: %s' % part)
191                 continue
192             result.add(expectation)
193         expectation_line.parsed_expectations = result
194
195     def _check_test_exists(self, expectation_line):
196         # WebKit's way of skipping tests is to add a -disabled suffix.
197         # So we should consider the path existing if the path or the
198         # -disabled version exists.
199         if not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled'):
200             # Log a warning here since you hit this case any
201             # time you update TestExpectations without syncing
202             # the LayoutTests directory
203             expectation_line.warnings.append('Path does not exist.')
204             expected_path = self._shorten_filename(self._port.abspath_for_test(expectation_line.name))
205             expectation_line.related_files[expected_path] = None
206             return False
207         return True
208
209     def _collect_matching_tests(self, expectation_line):
210         """Convert the test specification to an absolute, normalized
211         path and make sure directories end with the OS path separator."""
212
213         if not self._full_test_list:
214             expectation_line.matching_tests = [expectation_line.path]
215             return
216
217         if not expectation_line.is_file:
218             # this is a test category, return all the tests of the category.
219             expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)]
220             return
221
222         # this is a test file, do a quick check if it's in the
223         # full test suite.
224         if expectation_line.path in self._full_test_list:
225             expectation_line.matching_tests.append(expectation_line.path)
226
227     # FIXME: Update the original modifiers and remove this once the old syntax is gone.
228     _configuration_tokens_list = [
229         'SnowLeopard', 'Lion', 'MountainLion', 'Mavericks', 'Yosemite', 'ElCapitan', # Legacy macOS
230         'Mac', 'Sierra', 'HighSierra', 'Mojave',
231         'Win', 'XP', 'Vista', 'Win7',
232         'Linux',
233         'Android',
234         'Release',
235         'Debug',
236     ]
237
238     _configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list)
239     _inverted_configuration_tokens = dict((value, name) for name, value in _configuration_tokens.iteritems())
240
241     # FIXME: Update the original modifiers list and remove this once the old syntax is gone.
242     _expectation_tokens = {
243         'Crash': 'CRASH',
244         'Failure': 'FAIL',
245         'ImageOnlyFailure': 'IMAGE',
246         'Leak': 'LEAK',
247         'Missing': 'MISSING',
248         'Pass': 'PASS',
249         'Rebaseline': 'REBASELINE',
250         'Skip': 'SKIP',
251         'Slow': 'SLOW',
252         'Timeout': 'TIMEOUT',
253         'WontFix': 'WONTFIX',
254     }
255
256     _inverted_expectation_tokens = dict([(value, name) for name, value in _expectation_tokens.iteritems()] +
257                                         [('TEXT', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')])
258
259     # FIXME: Seems like these should be classmethods on TestExpectationLine instead of TestExpectationParser.
260     @classmethod
261     def _tokenize_line(cls, filename, expectation_string, line_number):
262         """Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format.
263
264         The new format for a test expectation line is:
265
266         [[bugs] [ "[" <configuration modifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>]
267
268         Any errant whitespace is not preserved.
269
270         """
271         expectation_line = TestExpectationLine()
272         expectation_line.original_string = expectation_string
273         expectation_line.filename = filename
274         expectation_line.line_number = line_number
275
276         comment_index = expectation_string.find("#")
277         if comment_index == -1:
278             comment_index = len(expectation_string)
279         else:
280             expectation_line.comment = expectation_string[comment_index + 1:]
281
282         remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip())
283         if len(remaining_string) == 0:
284             return expectation_line
285
286         # special-case parsing this so that we fail immediately instead of treating this as a test name
287         if remaining_string.startswith('//'):
288             expectation_line.warnings = ['use "#" instead of "//" for comments']
289             return expectation_line
290
291         bugs = []
292         modifiers = []
293         name = None
294         expectations = []
295         warnings = []
296
297         WEBKIT_BUG_PREFIX = 'webkit.org/b/'
298
299         tokens = remaining_string.split()
300         state = 'start'
301         for token in tokens:
302             if token.startswith(WEBKIT_BUG_PREFIX) or token.startswith('Bug('):
303                 if state != 'start':
304                     warnings.append('"%s" is not at the start of the line.' % token)
305                     break
306                 if token.startswith(WEBKIT_BUG_PREFIX):
307                     bugs.append(token.replace(WEBKIT_BUG_PREFIX, 'BUGWK'))
308                 else:
309                     match = re.match('Bug\((\w+)\)$', token)
310                     if not match:
311                         warnings.append('unrecognized bug identifier "%s"' % token)
312                         break
313                     else:
314                         bugs.append('BUG' + match.group(1).upper())
315             elif token.startswith('BUG'):
316                 warnings.append('unrecognized old-style bug identifier "%s"' % token)
317                 break
318             elif token == '[':
319                 if state == 'start':
320                     state = 'configuration'
321                 elif state == 'name_found':
322                     state = 'expectations'
323                 else:
324                     warnings.append('unexpected "["')
325                     break
326             elif token == ']':
327                 if state == 'configuration':
328                     state = 'name'
329                 elif state == 'expectations':
330                     state = 'done'
331                 else:
332                     warnings.append('unexpected "]"')
333                     break
334             elif token in ('//', ':', '='):
335                 warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token)
336                 break
337             elif state == 'configuration':
338                 modifiers.append(cls._configuration_tokens.get(token, token))
339             elif state == 'expectations':
340                 if token in ('Rebaseline', 'Skip', 'Slow', 'WontFix', 'DumpJSConsoleLogInStdErr'):
341                     modifiers.append(token.upper())
342                 elif token not in cls._expectation_tokens:
343                     warnings.append('Unrecognized expectation "%s"' % token)
344                 else:
345                     expectations.append(cls._expectation_tokens.get(token, token))
346             elif state == 'name_found':
347                 warnings.append('expecting "[", "#", or end of line instead of "%s"' % token)
348                 break
349             else:
350                 name = token
351                 state = 'name_found'
352
353         if not warnings:
354             if not name:
355                 warnings.append('Did not find a test name.')
356             elif state not in ('name_found', 'done'):
357                 warnings.append('Missing a "]"')
358
359         if 'WONTFIX' in modifiers and 'SKIP' not in modifiers and not expectations:
360             modifiers.append('SKIP')
361
362         if 'SKIP' in modifiers and expectations:
363             # FIXME: This is really a semantic warning and shouldn't be here. Remove when we drop the old syntax.
364             warnings.append('A test marked Skip must not have other expectations.')
365         elif not expectations:
366             # FIXME: We can probably simplify this adding 'SKIP' if modifiers is empty
367             if 'SKIP' not in modifiers and 'REBASELINE' not in modifiers and 'SLOW' not in modifiers and 'DUMPJSCONSOLELOGINSTDERR' not in modifiers:
368                 modifiers.append('SKIP')
369             expectations = ['PASS']
370
371         # FIXME: expectation line should just store bugs and modifiers separately.
372         expectation_line.modifiers = bugs + modifiers
373         expectation_line.expectations = expectations
374         expectation_line.name = name
375         expectation_line.warnings = warnings
376         return expectation_line
377
378     @classmethod
379     def _split_space_separated(cls, space_separated_string):
380         """Splits a space-separated string into an array."""
381         return [part.strip() for part in space_separated_string.strip().split(' ')]
382
383
384 class TestExpectationLine(object):
385     """Represents a line in test expectations file."""
386
387     def __init__(self):
388         """Initializes a blank-line equivalent of an expectation."""
389         self.original_string = None
390         self.filename = None  # this is the path to the expectations file for this line
391         self.line_number = None
392         self.name = None  # this is the path in the line itself
393         self.path = None  # this is the normpath of self.name
394         self.modifiers = []
395         self.parsed_modifiers = []
396         self.parsed_bug_modifiers = []
397         self.matching_configurations = set()
398         self.expectations = []
399         self.parsed_expectations = set()
400         self.comment = None
401         self.matching_tests = []
402         self.warnings = []
403         self.related_files = {}  # Dictionary of files to lines number in that file which may have caused the list of warnings.
404         self.not_applicable_to_current_platform = False
405
406     def __str__(self):
407         return self.to_string(None)
408
409     def is_invalid(self):
410         return self.warnings and self.warnings != [TestExpectationParser.MISSING_BUG_WARNING]
411
412     def is_flaky(self):
413         return len(self.parsed_expectations) > 1
414
415     @property
416     def expected_behavior(self):
417         expectations = self.expectations[:]
418         if "SLOW" in self.modifiers:
419             expectations += ["SLOW"]
420
421         if "SKIP" in self.modifiers:
422             expectations = ["SKIP"]
423         elif "WONTFIX" in self.modifiers:
424             expectations = ["WONTFIX"]
425         elif "CRASH" in self.modifiers:
426             expectations += ["CRASH"]
427
428         return expectations
429
430     @staticmethod
431     def create_passing_expectation(test):
432         expectation_line = TestExpectationLine()
433         expectation_line.name = test
434         expectation_line.path = test
435         expectation_line.parsed_expectations = set([PASS])
436         expectation_line.expectations = ['PASS']
437         expectation_line.matching_tests = [test]
438         return expectation_line
439
440     def to_string(self, test_configuration_converter, include_modifiers=True, include_expectations=True, include_comment=True):
441         parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
442
443         if self.is_invalid():
444             return self.original_string or ''
445
446         if self.name is None:
447             return '' if self.comment is None else "#%s" % self.comment
448
449         if test_configuration_converter and self.parsed_bug_modifiers:
450             specifiers_list = test_configuration_converter.to_specifiers_list(self.matching_configurations)
451             result = []
452             for specifiers in specifiers_list:
453                 # FIXME: this is silly that we join the modifiers and then immediately split them.
454                 modifiers = self._serialize_parsed_modifiers(test_configuration_converter, specifiers).split()
455                 expectations = self._serialize_parsed_expectations(parsed_expectation_to_string).split()
456                 result.append(self._format_line(modifiers, self.name, expectations, self.comment))
457             return "\n".join(result) if result else None
458
459         return self._format_line(self.modifiers, self.name, self.expectations, self.comment,
460             include_modifiers, include_expectations, include_comment)
461
462     def to_csv(self):
463         # Note that this doesn't include the comments.
464         return '%s,%s,%s' % (self.name, ' '.join(self.modifiers), ' '.join(self.expectations))
465
466     def _serialize_parsed_expectations(self, parsed_expectation_to_string):
467         result = []
468         for index in TestExpectations.EXPECTATION_ORDER:
469             if index in self.parsed_expectations:
470                 result.append(parsed_expectation_to_string[index])
471         return ' '.join(result)
472
473     def _serialize_parsed_modifiers(self, test_configuration_converter, specifiers):
474         result = []
475         if self.parsed_bug_modifiers:
476             result.extend(sorted(self.parsed_bug_modifiers))
477         result.extend(sorted(self.parsed_modifiers))
478         result.extend(test_configuration_converter.specifier_sorter().sort_specifiers(specifiers))
479         return ' '.join(result)
480
481     @staticmethod
482     def _format_line(modifiers, name, expectations, comment, include_modifiers=True, include_expectations=True, include_comment=True):
483         bugs = []
484         new_modifiers = []
485         new_expectations = []
486         for modifier in modifiers:
487             modifier = modifier.upper()
488             if modifier.startswith('BUGWK'):
489                 bugs.append('webkit.org/b/' + modifier.replace('BUGWK', ''))
490             elif modifier.startswith('BUG'):
491                 # FIXME: we should preserve case once we can drop the old syntax.
492                 bugs.append('Bug(' + modifier[3:].lower() + ')')
493             elif modifier in ('SLOW', 'SKIP', 'REBASELINE', 'WONTFIX', 'DUMPJSCONSOLELOGINSTDERR'):
494                 new_expectations.append(TestExpectationParser._inverted_expectation_tokens.get(modifier))
495             else:
496                 new_modifiers.append(TestExpectationParser._inverted_configuration_tokens.get(modifier, modifier))
497
498         for expectation in expectations:
499             expectation = expectation.upper()
500             new_expectations.append(TestExpectationParser._inverted_expectation_tokens.get(expectation, expectation))
501
502         result = ''
503         if include_modifiers and (bugs or new_modifiers):
504             if bugs:
505                 result += ' '.join(bugs) + ' '
506             if new_modifiers:
507                 result += '[ %s ] ' % ' '.join(new_modifiers)
508         result += name
509         if include_expectations and new_expectations and set(new_expectations) != set(['Skip', 'Pass']):
510             result += ' [ %s ]' % ' '.join(sorted(set(new_expectations)))
511         if include_comment and comment is not None:
512             result += " #%s" % comment
513         return result
514
515
516 # FIXME: Refactor API to be a proper CRUD.
517 class TestExpectationsModel(object):
518     """Represents relational store of all expectations and provides CRUD semantics to manage it."""
519
520     def __init__(self, shorten_filename=lambda x: x):
521         # Maps a test to its list of expectations.
522         self._test_to_expectations = {}
523
524         # Maps a test to list of its modifiers (string values)
525         self._test_to_modifiers = {}
526
527         # Maps a test to a TestExpectationLine instance.
528         self._test_to_expectation_line = {}
529
530         self._modifier_to_tests = self._dict_of_sets(TestExpectations.MODIFIERS)
531         self._expectation_to_tests = self._dict_of_sets(TestExpectations.EXPECTATIONS)
532         self._timeline_to_tests = self._dict_of_sets(TestExpectations.TIMELINES)
533         self._result_type_to_tests = self._dict_of_sets(TestExpectations.RESULT_TYPES)
534
535         self._shorten_filename = shorten_filename
536
537     def _dict_of_sets(self, strings_to_constants):
538         """Takes a dict of strings->constants and returns a dict mapping
539         each constant to an empty set."""
540         d = {}
541         for c in strings_to_constants.values():
542             d[c] = set()
543         return d
544
545     def get_test_set(self, modifier, expectation=None, include_skips=True):
546         if expectation is None:
547             tests = self._modifier_to_tests[modifier]
548         else:
549             tests = (self._expectation_to_tests[expectation] &
550                 self._modifier_to_tests[modifier])
551
552         if not include_skips:
553             tests = tests - self.get_test_set(SKIP, expectation)
554
555         return tests
556
557     def get_test_set_for_keyword(self, keyword):
558         # FIXME: get_test_set() is an awkward public interface because it requires
559         # callers to know the difference between modifiers and expectations. We
560         # should replace that with this where possible.
561         expectation_enum = TestExpectations.EXPECTATIONS.get(keyword.lower(), None)
562         if expectation_enum is not None:
563             return self._expectation_to_tests[expectation_enum]
564         modifier_enum = TestExpectations.MODIFIERS.get(keyword.lower(), None)
565         if modifier_enum is not None:
566             return self._modifier_to_tests[modifier_enum]
567
568         # We must not have an index on this modifier.
569         matching_tests = set()
570         for test, modifiers in self._test_to_modifiers.iteritems():
571             if keyword.lower() in modifiers:
572                 matching_tests.add(test)
573         return matching_tests
574
575     def get_tests_with_result_type(self, result_type):
576         return self._result_type_to_tests[result_type]
577
578     def get_tests_with_timeline(self, timeline):
579         return self._timeline_to_tests[timeline]
580
581     def get_modifiers(self, test):
582         """This returns modifiers for the given test (the modifiers plus the BUGXXXX identifier). This is used by the LTTF dashboard."""
583         return self._test_to_modifiers[test]
584
585     def has_modifier(self, test, modifier):
586         return test in self._modifier_to_tests[modifier]
587
588     def has_keyword(self, test, keyword):
589         return (keyword.upper() in self.get_expectations_string(test) or
590                 keyword.lower() in self.get_modifiers(test))
591
592     def has_test(self, test):
593         return test in self._test_to_expectation_line
594
595     def get_expectation_line(self, test):
596         return self._test_to_expectation_line.get(test)
597
598     def get_expectations(self, test):
599         return self._test_to_expectations[test]
600
601     def get_expectations_or_pass(self, test):
602         try:
603             return self.get_expectations(test)
604         except:
605             return set([PASS])
606
607     def expectations_to_string(self, expectations):
608         retval = []
609
610         for expectation in expectations:
611             retval.append(self.expectation_to_string(expectation))
612
613         return " ".join(retval)
614
615     def get_expectations_string(self, test):
616         """Returns the expectations for the given test as an uppercase string.
617         If there are no expectations for the test, then "PASS" is returned."""
618         try:
619             expectations = self.get_expectations(test)
620         except:
621             return "PASS"
622         retval = []
623
624         for expectation in expectations:
625             retval.append(self.expectation_to_string(expectation))
626
627         return " ".join(retval)
628
629     def expectation_to_string(self, expectation):
630         """Return the uppercased string equivalent of a given expectation."""
631         for item in TestExpectations.EXPECTATIONS.items():
632             if item[1] == expectation:
633                 return item[0].upper()
634         raise ValueError(expectation)
635
636     def add_expectation_line(self, expectation_line, in_skipped=False):
637         """Returns a list of warnings encountered while matching modifiers."""
638
639         if expectation_line.is_invalid():
640             return
641
642         for test in expectation_line.matching_tests:
643             if not in_skipped and self._already_seen_better_match(test, expectation_line):
644                 continue
645
646             self._clear_expectations_for_test(test)
647             self._test_to_expectation_line[test] = expectation_line
648             self._add_test(test, expectation_line)
649
650     def _add_test(self, test, expectation_line):
651         """Sets the expected state for a given test.
652
653         This routine assumes the test has not been added before. If it has,
654         use _clear_expectations_for_test() to reset the state prior to
655         calling this."""
656         self._test_to_expectations[test] = expectation_line.parsed_expectations
657         for expectation in expectation_line.parsed_expectations:
658             self._expectation_to_tests[expectation].add(test)
659
660         self._test_to_modifiers[test] = expectation_line.modifiers
661         for modifier in expectation_line.parsed_modifiers:
662             mod_value = TestExpectations.MODIFIERS[modifier]
663             self._modifier_to_tests[mod_value].add(test)
664
665         if TestExpectationParser.WONTFIX_MODIFIER in expectation_line.parsed_modifiers:
666             self._timeline_to_tests[WONTFIX].add(test)
667         else:
668             self._timeline_to_tests[NOW].add(test)
669
670         if TestExpectationParser.SKIP_MODIFIER in expectation_line.parsed_modifiers:
671             self._result_type_to_tests[SKIP].add(test)
672         elif expectation_line.parsed_expectations == set([PASS]):
673             self._result_type_to_tests[PASS].add(test)
674         elif expectation_line.is_flaky():
675             self._result_type_to_tests[FLAKY].add(test)
676         else:
677             # FIXME: What is this?
678             self._result_type_to_tests[FAIL].add(test)
679
680     def _clear_expectations_for_test(self, test):
681         """Remove prexisting expectations for this test.
682         This happens if we are seeing a more precise path
683         than a previous listing.
684         """
685         if self.has_test(test):
686             self._test_to_expectations.pop(test, '')
687             self._remove_from_sets(test, self._expectation_to_tests)
688             self._remove_from_sets(test, self._modifier_to_tests)
689             self._remove_from_sets(test, self._timeline_to_tests)
690             self._remove_from_sets(test, self._result_type_to_tests)
691
692     def _remove_from_sets(self, test, dict_of_sets_of_tests):
693         """Removes the given test from the sets in the dictionary.
694
695         Args:
696           test: test to look for
697           dict: dict of sets of files"""
698         for set_of_tests in dict_of_sets_of_tests.itervalues():
699             if test in set_of_tests:
700                 set_of_tests.remove(test)
701
702     def _already_seen_better_match(self, test, expectation_line):
703         """Returns whether we've seen a better match already in the file.
704
705         Returns True if we've already seen a expectation_line.name that matches more of the test
706             than this path does
707         """
708         # FIXME: See comment below about matching test configs and specificity.
709         if not self.has_test(test):
710             # We've never seen this test before.
711             return False
712
713         prev_expectation_line = self._test_to_expectation_line[test]
714
715         if prev_expectation_line.filename != expectation_line.filename:
716             # We've moved on to a new expectation file, which overrides older ones.
717             return False
718
719         if len(prev_expectation_line.path) > len(expectation_line.path):
720             # The previous path matched more of the test.
721             return True
722
723         if len(prev_expectation_line.path) < len(expectation_line.path):
724             # This path matches more of the test.
725             return False
726
727         # At this point we know we have seen a previous exact match on this
728         # base path, so we need to check the two sets of modifiers.
729
730         # FIXME: This code was originally designed to allow lines that matched
731         # more modifiers to override lines that matched fewer modifiers.
732         # However, we currently view these as errors.
733         #
734         # To use the "more modifiers wins" policy, change the errors for overrides
735         # to be warnings and return False".
736         shortened_expectation_filename = self._shorten_filename(expectation_line.filename)
737         shortened_previous_expectation_filename = self._shorten_filename(prev_expectation_line.filename)
738
739         if prev_expectation_line.matching_configurations == expectation_line.matching_configurations:
740             expectation_line.warnings.append('Duplicate or ambiguous entry lines %s:%d and %s:%d.' % (
741                 shortened_previous_expectation_filename, prev_expectation_line.line_number,
742                 shortened_expectation_filename, expectation_line.line_number))
743
744         elif prev_expectation_line.matching_configurations >= expectation_line.matching_configurations:
745             expectation_line.warnings.append('More specific entry for %s on line %s:%d overrides line %s:%d.' % (expectation_line.name,
746                 shortened_previous_expectation_filename, prev_expectation_line.line_number,
747                 shortened_expectation_filename, expectation_line.line_number))
748             # FIXME: return False if we want more specific to win.
749
750         elif prev_expectation_line.matching_configurations <= expectation_line.matching_configurations:
751             expectation_line.warnings.append('More specific entry for %s on line %s:%d overrides line %s:%d.' % (expectation_line.name,
752                 shortened_expectation_filename, expectation_line.line_number,
753                 shortened_previous_expectation_filename, prev_expectation_line.line_number))
754
755         elif prev_expectation_line.matching_configurations & expectation_line.matching_configurations:
756             expectation_line.warnings.append('Entries for %s on lines %s:%d and %s:%d match overlapping sets of configurations.' % (expectation_line.name,
757                 shortened_previous_expectation_filename, prev_expectation_line.line_number,
758                 shortened_expectation_filename, expectation_line.line_number))
759
760         else:
761             # Configuration sets are disjoint.
762             return False
763
764         # Missing files will be 'None'. It should be impossible to have a missing file which also has a line associated with it.
765         assert shortened_previous_expectation_filename not in expectation_line.related_files or expectation_line.related_files[shortened_previous_expectation_filename] is not None
766         expectation_line.related_files[shortened_previous_expectation_filename] = expectation_line.related_files.get(shortened_previous_expectation_filename, [])
767         expectation_line.related_files[shortened_previous_expectation_filename].append(prev_expectation_line.line_number)
768
769         assert shortened_expectation_filename not in prev_expectation_line.related_files or prev_expectation_line.related_files[shortened_expectation_filename] is not None
770         prev_expectation_line.related_files[shortened_expectation_filename] = prev_expectation_line.related_files.get(shortened_expectation_filename, [])
771         prev_expectation_line.related_files[shortened_expectation_filename].append(expectation_line.line_number)
772
773         return True
774
775
776 class TestExpectations(object):
777     """Test expectations consist of lines with specifications of what
778     to expect from layout test cases. The test cases can be directories
779     in which case the expectations apply to all test cases in that
780     directory and any subdirectory. The format is along the lines of:
781
782       LayoutTests/js/fixme.js [ Failure ]
783       LayoutTests/js/flaky.js [ Failure Pass ]
784       LayoutTests/js/crash.js [ Crash Failure Pass Timeout ]
785       ...
786
787     To add modifiers:
788       LayoutTests/js/no-good.js
789       [ Debug ] LayoutTests/js/no-good.js [ Pass Timeout ]
790       [ Debug ] LayoutTests/js/no-good.js [ Pass Skip Timeout ]
791       [ Linux Debug ] LayoutTests/js/no-good.js [ Pass Skip Timeout ]
792       [ Linux Win ] LayoutTests/js/no-good.js [ Pass Skip Timeout ]
793
794     Skip: Doesn't run the test.
795     Slow: The test takes a long time to run, but does not timeout indefinitely.
796     WontFix: For tests that we never intend to pass on a given platform (treated like Skip).
797
798     Notes:
799       -A test cannot be both SLOW and TIMEOUT
800       -A test can be included twice, but not via the same path.
801       -If a test is included twice, then the more precise path wins.
802       -CRASH tests cannot be WONTFIX
803     """
804
805     # FIXME: Update to new syntax once the old format is no longer supported.
806     EXPECTATIONS = {'pass': PASS,
807                     'audio': AUDIO,
808                     'fail': FAIL,
809                     'image': IMAGE,
810                     'image+text': IMAGE_PLUS_TEXT,
811                     'text': TEXT,
812                     'timeout': TIMEOUT,
813                     'crash': CRASH,
814                     'missing': MISSING,
815                     'leak': LEAK,
816                     'skip': SKIP}
817
818     # (aggregated by category, pass/fail/skip, type)
819     EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped',
820                                 PASS: 'passes',
821                                 FAIL: 'failures',
822                                 IMAGE: 'image-only failures',
823                                 TEXT: 'text-only failures',
824                                 IMAGE_PLUS_TEXT: 'image and text failures',
825                                 AUDIO: 'audio failures',
826                                 CRASH: 'crashes',
827                                 TIMEOUT: 'timeouts',
828                                 MISSING: 'missing results',
829                                 LEAK: 'leaks'}
830
831     EXPECTATION_ORDER = (PASS, CRASH, TIMEOUT, MISSING, FAIL, IMAGE, LEAK, SKIP)
832
833     BUILD_TYPES = ('debug', 'release')
834
835     MODIFIERS = {TestExpectationParser.SKIP_MODIFIER: SKIP,
836                  TestExpectationParser.WONTFIX_MODIFIER: WONTFIX,
837                  TestExpectationParser.SLOW_MODIFIER: SLOW,
838                  TestExpectationParser.DUMPJSCONSOLELOGINSTDERR_MODIFIER: DUMPJSCONSOLELOGINSTDERR,
839                  TestExpectationParser.REBASELINE_MODIFIER: REBASELINE,
840                  'none': NONE}
841
842     TIMELINES = {TestExpectationParser.WONTFIX_MODIFIER: WONTFIX,
843                  'now': NOW}
844
845     RESULT_TYPES = {'skip': SKIP,
846                     'pass': PASS,
847                     'fail': FAIL,
848                     'flaky': FLAKY}
849
850     @classmethod
851     def expectation_from_string(cls, string):
852         assert(' ' not in string)  # This only handles one expectation at a time.
853         return cls.EXPECTATIONS.get(string.lower())
854
855     @staticmethod
856     def result_was_expected(result, expected_results, test_needs_rebaselining, test_is_skipped):
857         """Returns whether we got a result we were expecting.
858         Args:
859             result: actual result of a test execution
860             expected_results: set of results listed in test_expectations
861             test_needs_rebaselining: whether test was marked as REBASELINE
862             test_is_skipped: whether test was marked as SKIP"""
863         if result in expected_results:
864             return True
865         if result in (TEXT, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected_results):
866             return True
867         if result == MISSING and test_needs_rebaselining:
868             return True
869         if result == SKIP and test_is_skipped:
870             return True
871         return False
872
873     @staticmethod
874     def remove_pixel_failures(expected_results):
875         """Returns a copy of the expected results for a test, except that we
876         drop any pixel failures and return the remaining expectations. For example,
877         if we're not running pixel tests, then tests expected to fail as IMAGE
878         will PASS."""
879         expected_results = expected_results.copy()
880         if IMAGE in expected_results:
881             expected_results.remove(IMAGE)
882             expected_results.add(PASS) # FIXME: does it always become a pass?
883         return expected_results
884
885     @staticmethod
886     def remove_leak_failures(expected_results):
887         """Returns a copy of the expected results for a test, except that we
888         drop any leak failures and return the remaining expectations. For example,
889         if we're not running with --world-leaks, then tests expected to fail as LEAK
890         will PASS."""
891         expected_results = expected_results.copy()
892         if LEAK in expected_results:
893             expected_results.remove(LEAK)
894             if not expected_results:
895                 expected_results.add(PASS)
896         return expected_results
897
898     @staticmethod
899     def has_pixel_failures(actual_results):
900         return IMAGE in actual_results or FAIL in actual_results
901
902     @staticmethod
903     def suffixes_for_expectations(expectations):
904         suffixes = set()
905         if IMAGE in expectations:
906             suffixes.add('png')
907         if FAIL in expectations:
908             suffixes.add('txt')
909             suffixes.add('png')
910             suffixes.add('wav')
911         return set(suffixes)
912
913     def __init__(self, port, tests=None, include_generic=True, include_overrides=True, expectations_to_lint=None, force_expectations_pass=False):
914         self._full_test_list = tests
915         self._test_config = port.test_configuration()
916         self._is_lint_mode = expectations_to_lint is not None
917         self._model = TestExpectationsModel(self._shorten_filename)
918         self._parser = TestExpectationParser(port, tests, self._is_lint_mode, self._shorten_filename)
919         self._port = port
920         self._skipped_tests_warnings = []
921         self._expectations = []
922         self._force_expectations_pass = force_expectations_pass
923         self._include_generic = include_generic
924         self._include_overrides = include_overrides
925         self._expectations_to_lint = expectations_to_lint
926
927     def readable_filename_and_line_number(self, line):
928         if line.not_applicable_to_current_platform:
929             return "(skipped for this platform)"
930         if not line.filename:
931             return ''
932         if line.filename.startswith(self._port.path_from_webkit_base()):
933             return '{}:{}'.format(self._port.host.filesystem.relpath(line.filename, self._port.path_from_webkit_base()), line.line_number)
934         return '{}:{}'.format(line.filename, line.line_number)
935
936     def parse_generic_expectations(self):
937         if self._port.path_to_generic_test_expectations_file() in self._expectations_dict:
938             if self._include_generic:
939                 expectations = self._parser.parse(self._expectations_dict.keys()[self._expectations_dict_index], self._expectations_dict.values()[self._expectations_dict_index])
940                 self._add_expectations(expectations)
941                 self._expectations += expectations
942             self._expectations_dict_index += 1
943
944     def parse_default_port_expectations(self):
945         if len(self._expectations_dict) > self._expectations_dict_index:
946             expectations = self._parser.parse(self._expectations_dict.keys()[self._expectations_dict_index], self._expectations_dict.values()[self._expectations_dict_index])
947             self._add_expectations(expectations)
948             self._expectations += expectations
949             self._expectations_dict_index += 1
950
951     def parse_override_expectations(self):
952         while len(self._expectations_dict) > self._expectations_dict_index and self._include_overrides:
953             expectations = self._parser.parse(self._expectations_dict.keys()[self._expectations_dict_index], self._expectations_dict.values()[self._expectations_dict_index])
954             self._add_expectations(expectations)
955             self._expectations += expectations
956             self._expectations_dict_index += 1
957
958     def parse_all_expectations(self):
959         self._expectations_dict = self._expectations_to_lint or self._port.expectations_dict()
960         self._expectations_dict_index = 0
961
962         self._has_warnings = False
963
964         self.parse_generic_expectations()
965         self.parse_default_port_expectations()
966         self.parse_override_expectations()
967
968         # FIXME: move ignore_tests into port.skipped_layout_tests()
969         self.add_skipped_tests(self._port.skipped_layout_tests(self._full_test_list).union(set(self._port.get_option('ignore_tests', []))))
970
971         self._report_warnings()
972         self._process_tests_without_expectations()
973
974     # TODO(ojan): Allow for removing skipped tests when getting the list of
975     # tests to run, but not when getting metrics.
976     def model(self):
977         return self._model
978
979     def get_rebaselining_failures(self):
980         return self._model.get_test_set(REBASELINE)
981
982     def filtered_expectations_for_test(self, test, pixel_tests_are_enabled, world_leaks_are_enabled):
983         expected_results = self._model.get_expectations_or_pass(test)
984         if not pixel_tests_are_enabled:
985             expected_results = self.remove_pixel_failures(expected_results)
986         if not world_leaks_are_enabled:
987             expected_results = self.remove_leak_failures(expected_results)
988
989         return expected_results
990
991     def matches_an_expected_result(self, test, result, expected_results):
992         return self.result_was_expected(result,
993                                    expected_results,
994                                    self.is_rebaselining(test),
995                                    self._model.has_modifier(test, SKIP))
996
997     def is_rebaselining(self, test):
998         return self._model.has_modifier(test, REBASELINE)
999
1000     def _shorten_filename(self, filename):
1001         if filename.startswith(self._port.path_from_webkit_base()):
1002             return self._port.host.filesystem.relpath(filename, self._port.path_from_webkit_base())
1003         return filename
1004
1005     def _report_warnings(self):
1006         warnings = []
1007         for expectation in self._expectations:
1008             for warning in expectation.warnings:
1009                 warning = TestExpectationWarning(
1010                     self._shorten_filename(expectation.filename), expectation.line_number,
1011                     expectation.original_string, warning, expectation.name if expectation.expectations else None)
1012                 warning.related_files = expectation.related_files
1013                 warnings.append(warning)
1014
1015         if warnings:
1016             self._has_warnings = True
1017             if self._is_lint_mode:
1018                 raise ParseError(warnings)
1019             _log.warning('--lint-test-files warnings:')
1020             for warning in warnings:
1021                 _log.warning(warning)
1022             _log.warning('')
1023
1024     def _process_tests_without_expectations(self):
1025         if self._full_test_list:
1026             for test in self._full_test_list:
1027                 if not self._model.has_test(test):
1028                     self._model.add_expectation_line(TestExpectationLine.create_passing_expectation(test))
1029
1030     def has_warnings(self):
1031         return self._has_warnings
1032
1033     def remove_configuration_from_test(self, test, test_configuration):
1034         expectations_to_remove = []
1035         modified_expectations = []
1036
1037         for expectation in self._expectations:
1038             if expectation.name != test or expectation.is_flaky() or not expectation.parsed_expectations:
1039                 continue
1040             if iter(expectation.parsed_expectations).next() not in (FAIL, IMAGE):
1041                 continue
1042             if test_configuration not in expectation.matching_configurations:
1043                 continue
1044
1045             expectation.matching_configurations.remove(test_configuration)
1046             if expectation.matching_configurations:
1047                 modified_expectations.append(expectation)
1048             else:
1049                 expectations_to_remove.append(expectation)
1050
1051         for expectation in expectations_to_remove:
1052             self._expectations.remove(expectation)
1053
1054         return self.list_to_string(self._expectations, self._parser._test_configuration_converter, modified_expectations)
1055
1056     def remove_rebaselined_tests(self, except_these_tests, filename):
1057         """Returns a copy of the expectations in the file with the tests removed."""
1058         def without_rebaseline_modifier(expectation):
1059             return (expectation.filename == filename and
1060                     not (not expectation.is_invalid() and
1061                          expectation.name in except_these_tests and
1062                          'rebaseline' in expectation.parsed_modifiers))
1063
1064         return self.list_to_string(filter(without_rebaseline_modifier, self._expectations), reconstitute_only_these=[])
1065
1066     def _add_expectations(self, expectation_list):
1067         for expectation_line in expectation_list:
1068             if self._force_expectations_pass:
1069                 expectation_line.expectations = ['PASS']
1070                 expectation_line.parsed_expectations = set([PASS])
1071
1072             elif not expectation_line.expectations:
1073                 continue
1074
1075             if self._is_lint_mode or self._test_config in expectation_line.matching_configurations:
1076                 self._model.add_expectation_line(expectation_line)
1077
1078     def add_skipped_tests(self, tests_to_skip):
1079         if not tests_to_skip:
1080             return
1081         for test in self._expectations:
1082             if test.name and test.name in tests_to_skip:
1083                 test.warnings.append('%s:%d %s is also in a Skipped file.' % (test.filename, test.line_number, test.name))
1084
1085         for test_name in tests_to_skip:
1086             expectation_line = self._parser.expectation_for_skipped_test(test_name)
1087             self._model.add_expectation_line(expectation_line, in_skipped=True)
1088
1089     @staticmethod
1090     def list_to_string(expectation_lines, test_configuration_converter=None, reconstitute_only_these=None):
1091         def serialize(expectation_line):
1092             # If reconstitute_only_these is an empty list, we want to return original_string.
1093             # So we need to compare reconstitute_only_these to None, not just check if it's falsey.
1094             if reconstitute_only_these is None or expectation_line in reconstitute_only_these:
1095                 return expectation_line.to_string(test_configuration_converter)
1096             return expectation_line.original_string
1097
1098         def nones_out(expectation_line):
1099             return expectation_line is not None
1100
1101         return "\n".join(filter(nones_out, map(serialize, expectation_lines)))