webkitpy: Implement device type specific expected results (Part 2)
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_expectations.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 """A helper class for reading in and dealing with tests expectations
30 for layout tests.
31 """
32
33 import logging
34 import re
35
36 from webkitpy.layout_tests.models.test_configuration import TestConfigurationConverter
37
38 _log = logging.getLogger(__name__)
39
40
41 # Test expectation and modifier constants.
42 #
43 # FIXME: range() starts with 0 which makes if expectation checks harder
44 # as PASS is 0.
45 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, SKIP, WONTFIX,
46  SLOW, LEAK, DUMPJSCONSOLELOGINSTDERR, REBASELINE, MISSING, FLAKY, NOW, NONE) = range(18)
47
48 # FIXME: Perhas these two routines should be part of the Port instead?
49 BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt')
50
51
52 class ParseError(Exception):
53     def __init__(self, warnings):
54         super(ParseError, self).__init__()
55         self.warnings = warnings
56
57     def __str__(self):
58         return '\n'.join(map(str, self.warnings))
59
60     def __repr__(self):
61         return 'ParseError(warnings=%s)' % self.warnings
62
63
64 class TestExpectationWarning(object):
65     def __init__(self, filename, line_number, line, error, test=None):
66         self.filename = filename
67         self.line_number = line_number
68         self.line = line
69         self.error = error
70         self.test = test
71
72         self.related_files = {}
73
74     def __str__(self):
75         return '{}:{} {} {}'.format(self.filename, self.line_number, self.error, self.test if self.test else self.line)
76
77
78 class TestExpectationParser(object):
79     """Provides parsing facilities for lines in the test_expectation.txt file."""
80
81     DUMMY_BUG_MODIFIER = "bug_dummy"
82     BUG_MODIFIER_PREFIX = 'bug'
83     BUG_MODIFIER_REGEX = 'bug\d+'
84     REBASELINE_MODIFIER = 'rebaseline'
85     PASS_EXPECTATION = 'pass'
86     SKIP_MODIFIER = 'skip'
87     SLOW_MODIFIER = 'slow'
88     DUMPJSCONSOLELOGINSTDERR_MODIFIER = 'dumpjsconsoleloginstderr'
89     WONTFIX_MODIFIER = 'wontfix'
90
91     TIMEOUT_EXPECTATION = 'timeout'
92
93     MISSING_BUG_WARNING = 'Test lacks BUG modifier.'
94
95     def __init__(self, port, full_test_list, allow_rebaseline_modifier, shorten_filename=lambda x: x):
96         self._port = port
97         self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros())
98         if full_test_list is None:
99             self._full_test_list = None
100         else:
101             self._full_test_list = set(full_test_list)
102         self._allow_rebaseline_modifier = allow_rebaseline_modifier
103         self._shorten_filename = shorten_filename
104
105     def parse(self, filename, expectations_string):
106         expectation_lines = []
107         line_number = 0
108         for line in expectations_string.split("\n"):
109             line_number += 1
110             test_expectation = self._tokenize_line(filename, line, line_number)
111             self._parse_line(test_expectation)
112             expectation_lines.append(test_expectation)
113         return expectation_lines
114
115     def expectation_for_skipped_test(self, test_name):
116         if not self._port.test_exists(test_name):
117             _log.warning('The following test %s from the Skipped list doesn\'t exist' % test_name)
118         expectation_line = TestExpectationLine()
119         expectation_line.original_string = test_name
120         expectation_line.modifiers = [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER]
121         # FIXME: It's not clear what the expectations for a skipped test should be; the expectations
122         # might be different for different entries in a Skipped file, or from the command line, or from
123         # only running parts of the tests. It's also not clear if it matters much.
124         expectation_line.modifiers.append(TestExpectationParser.WONTFIX_MODIFIER)
125         expectation_line.name = test_name
126         # FIXME: we should pass in a more descriptive string here.
127         expectation_line.filename = '<Skipped file>'
128         expectation_line.line_number = 0
129         expectation_line.expectations = [TestExpectationParser.PASS_EXPECTATION]
130         expectation_line.not_applicable_to_current_platform = True
131         self._parse_line(expectation_line)
132         return expectation_line
133
134     def _parse_line(self, expectation_line):
135         if not expectation_line.name:
136             return
137
138         if not self._check_test_exists(expectation_line):
139             return
140
141         expectation_line.is_file = self._port.test_isfile(expectation_line.name)
142         if expectation_line.is_file:
143             expectation_line.path = expectation_line.name
144         else:
145             expectation_line.path = self._port.normalize_test_name(expectation_line.name)
146
147         self._collect_matching_tests(expectation_line)
148
149         self._parse_modifiers(expectation_line)
150         self._parse_expectations(expectation_line)
151
152     def _parse_modifiers(self, expectation_line):
153         has_wontfix = False
154         has_bugid = False
155         parsed_specifiers = set()
156
157         modifiers = [modifier.lower() for modifier in expectation_line.modifiers]
158         expectations = [expectation.lower() for expectation in expectation_line.expectations]
159
160         if self.SLOW_MODIFIER in modifiers and self.TIMEOUT_EXPECTATION in expectations:
161             expectation_line.warnings.append('A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT.')
162
163         for modifier in modifiers:
164             if modifier in TestExpectations.MODIFIERS:
165                 expectation_line.parsed_modifiers.append(modifier)
166                 if modifier == self.WONTFIX_MODIFIER:
167                     has_wontfix = True
168             elif modifier.startswith(self.BUG_MODIFIER_PREFIX):
169                 has_bugid = True
170                 if re.match(self.BUG_MODIFIER_REGEX, modifier):
171                     expectation_line.warnings.append('BUG\d+ is not allowed, must be one of BUGWK\d+, or a non-numeric bug identifier.')
172                 else:
173                     expectation_line.parsed_bug_modifiers.append(modifier)
174             else:
175                 parsed_specifiers.add(modifier)
176
177         if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid and self._port.warn_if_bug_missing_in_test_expectations():
178             expectation_line.warnings.append(self.MISSING_BUG_WARNING)
179
180         if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in modifiers:
181             expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.')
182
183         expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings)
184
185     def _parse_expectations(self, expectation_line):
186         result = set()
187         for part in expectation_line.expectations:
188             expectation = TestExpectations.expectation_from_string(part)
189             if expectation is None:  # Careful, PASS is currently 0.
190                 expectation_line.warnings.append('Unsupported expectation: %s' % part)
191                 continue
192             result.add(expectation)
193         expectation_line.parsed_expectations = result
194
195     def _check_test_exists(self, expectation_line):
196         # WebKit's way of skipping tests is to add a -disabled suffix.
197         # So we should consider the path existing if the path or the
198         # -disabled version exists.
199         if not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled'):
200             # Log a warning here since you hit this case any
201             # time you update TestExpectations without syncing
202             # the LayoutTests directory
203             expectation_line.warnings.append('Path does not exist.')
204             expected_path = self._shorten_filename(self._port.abspath_for_test(expectation_line.name))
205             expectation_line.related_files[expected_path] = None
206             return False
207         return True
208
209     def _collect_matching_tests(self, expectation_line):
210         """Convert the test specification to an absolute, normalized
211         path and make sure directories end with the OS path separator."""
212
213         if not self._full_test_list:
214             expectation_line.matching_tests = [expectation_line.path]
215             return
216
217         if not expectation_line.is_file:
218             # this is a test category, return all the tests of the category.
219             expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)]
220             return
221
222         # this is a test file, do a quick check if it's in the
223         # full test suite.
224         if expectation_line.path in self._full_test_list:
225             expectation_line.matching_tests.append(expectation_line.path)
226
227     # FIXME: Update the original modifiers and remove this once the old syntax is gone.
228     _configuration_tokens_list = [
229         'SnowLeopard', 'Lion', 'MountainLion', 'Mavericks', 'Yosemite', 'ElCapitan', # Legacy macOS
230         'Mac', 'Sierra', 'HighSierra', 'Mojave',
231         'Win', 'XP', 'Vista', 'Win7',
232         'Linux',
233         'Android',
234         'Release',
235         'Debug',
236     ]
237
238     _configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list)
239     _inverted_configuration_tokens = dict((value, name) for name, value in _configuration_tokens.iteritems())
240
241     # FIXME: Update the original modifiers list and remove this once the old syntax is gone.
242     _expectation_tokens = {
243         'Crash': 'CRASH',
244         'Failure': 'FAIL',
245         'ImageOnlyFailure': 'IMAGE',
246         'Leak': 'LEAK',
247         'Missing': 'MISSING',
248         'Pass': 'PASS',
249         'Rebaseline': 'REBASELINE',
250         'Skip': 'SKIP',
251         'Slow': 'SLOW',
252         'Timeout': 'TIMEOUT',
253         'WontFix': 'WONTFIX',
254     }
255
256     _inverted_expectation_tokens = dict([(value, name) for name, value in _expectation_tokens.iteritems()] +
257                                         [('TEXT', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')])
258
259     # FIXME: Seems like these should be classmethods on TestExpectationLine instead of TestExpectationParser.
260     @classmethod
261     def _tokenize_line(cls, filename, expectation_string, line_number):
262         """Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format.
263
264         The new format for a test expectation line is:
265
266         [[bugs] [ "[" <configuration modifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>]
267
268         Any errant whitespace is not preserved.
269
270         """
271         expectation_line = TestExpectationLine()
272         expectation_line.original_string = expectation_string
273         expectation_line.filename = filename
274         expectation_line.line_number = line_number
275
276         comment_index = expectation_string.find("#")
277         if comment_index == -1:
278             comment_index = len(expectation_string)
279         else:
280             expectation_line.comment = expectation_string[comment_index + 1:]
281
282         remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip())
283         if len(remaining_string) == 0:
284             return expectation_line
285
286         # special-case parsing this so that we fail immediately instead of treating this as a test name
287         if remaining_string.startswith('//'):
288             expectation_line.warnings = ['use "#" instead of "//" for comments']
289             return expectation_line
290
291         bugs = []
292         modifiers = []
293         name = None
294         expectations = []
295         warnings = []
296
297         WEBKIT_BUG_PREFIX = 'webkit.org/b/'
298
299         tokens = remaining_string.split()
300         state = 'start'
301         for token in tokens:
302             if token.startswith(WEBKIT_BUG_PREFIX) or token.startswith('Bug('):
303                 if state != 'start':
304                     warnings.append('"%s" is not at the start of the line.' % token)
305                     break
306                 if token.startswith(WEBKIT_BUG_PREFIX):
307                     bugs.append(token.replace(WEBKIT_BUG_PREFIX, 'BUGWK'))
308                 else:
309                     match = re.match('Bug\((\w+)\)$', token)
310                     if not match:
311                         warnings.append('unrecognized bug identifier "%s"' % token)
312                         break
313                     else:
314                         bugs.append('BUG' + match.group(1).upper())
315             elif token.startswith('BUG'):
316                 warnings.append('unrecognized old-style bug identifier "%s"' % token)
317                 break
318             elif token == '[':
319                 if state == 'start':
320                     state = 'configuration'
321                 elif state == 'name_found':
322                     state = 'expectations'
323                 else:
324                     warnings.append('unexpected "["')
325                     break
326             elif token == ']':
327                 if state == 'configuration':
328                     state = 'name'
329                 elif state == 'expectations':
330                     state = 'done'
331                 else:
332                     warnings.append('unexpected "]"')
333                     break
334             elif token in ('//', ':', '='):
335                 warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token)
336                 break
337             elif state == 'configuration':
338                 modifiers.append(cls._configuration_tokens.get(token, token))
339             elif state == 'expectations':
340                 if token in ('Rebaseline', 'Skip', 'Slow', 'WontFix', 'DumpJSConsoleLogInStdErr'):
341                     modifiers.append(token.upper())
342                 elif token not in cls._expectation_tokens:
343                     warnings.append('Unrecognized expectation "%s"' % token)
344                 else:
345                     expectations.append(cls._expectation_tokens.get(token, token))
346             elif state == 'name_found':
347                 warnings.append('expecting "[", "#", or end of line instead of "%s"' % token)
348                 break
349             else:
350                 name = token
351                 state = 'name_found'
352
353         if not warnings:
354             if not name:
355                 warnings.append('Did not find a test name.')
356             elif state not in ('name_found', 'done'):
357                 warnings.append('Missing a "]"')
358
359         if 'WONTFIX' in modifiers and 'SKIP' not in modifiers and not expectations:
360             modifiers.append('SKIP')
361
362         if 'SKIP' in modifiers and expectations:
363             # FIXME: This is really a semantic warning and shouldn't be here. Remove when we drop the old syntax.
364             warnings.append('A test marked Skip must not have other expectations.')
365         elif not expectations:
366             # FIXME: We can probably simplify this adding 'SKIP' if modifiers is empty
367             if 'SKIP' not in modifiers and 'REBASELINE' not in modifiers and 'SLOW' not in modifiers and 'DUMPJSCONSOLELOGINSTDERR' not in modifiers:
368                 modifiers.append('SKIP')
369             expectations = ['PASS']
370
371         # FIXME: expectation line should just store bugs and modifiers separately.
372         expectation_line.modifiers = bugs + modifiers
373         expectation_line.expectations = expectations
374         expectation_line.name = name
375         expectation_line.warnings = warnings
376         return expectation_line
377
378     @classmethod
379     def _split_space_separated(cls, space_separated_string):
380         """Splits a space-separated string into an array."""
381         return [part.strip() for part in space_separated_string.strip().split(' ')]
382
383
384 class TestExpectationLine(object):
385     """Represents a line in test expectations file."""
386
387     def __init__(self):
388         """Initializes a blank-line equivalent of an expectation."""
389         self.original_string = None
390         self.filename = None  # this is the path to the expectations file for this line
391         self.line_number = None
392         self.name = None  # this is the path in the line itself
393         self.path = None  # this is the normpath of self.name
394         self.modifiers = []
395         self.parsed_modifiers = []
396         self.parsed_bug_modifiers = []
397         self.matching_configurations = set()
398         self.expectations = []
399         self.parsed_expectations = set()
400         self.comment = None
401         self.matching_tests = []
402         self.warnings = []
403         self.related_files = {}  # Dictionary of files to lines number in that file which may have caused the list of warnings.
404         self.not_applicable_to_current_platform = False
405
406     def __str__(self):
407         return self.to_string(None)
408
409     def is_invalid(self):
410         return self.warnings and self.warnings != [TestExpectationParser.MISSING_BUG_WARNING]
411
412     def is_flaky(self):
413         return len(self.parsed_expectations) > 1
414
415     @property
416     def expected_behavior(self):
417         expectations = self.expectations[:]
418         if "SLOW" in self.modifiers:
419             expectations += ["SLOW"]
420
421         if "SKIP" in self.modifiers:
422             expectations = ["SKIP"]
423         elif "WONTFIX" in self.modifiers:
424             expectations = ["WONTFIX"]
425         elif "CRASH" in self.modifiers:
426             expectations += ["CRASH"]
427
428         return expectations
429
430     @staticmethod
431     def create_passing_expectation(test):
432         expectation_line = TestExpectationLine()
433         expectation_line.name = test
434         expectation_line.path = test
435         expectation_line.parsed_expectations = set([PASS])
436         expectation_line.expectations = ['PASS']
437         expectation_line.matching_tests = [test]
438         return expectation_line
439
440     def to_string(self, test_configuration_converter, include_modifiers=True, include_expectations=True, include_comment=True):
441         parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
442
443         if self.is_invalid():
444             return self.original_string or ''
445
446         if self.name is None:
447             return '' if self.comment is None else "#%s" % self.comment
448
449         if test_configuration_converter and self.parsed_bug_modifiers:
450             specifiers_list = test_configuration_converter.to_specifiers_list(self.matching_configurations)
451             result = []
452             for specifiers in specifiers_list:
453                 # FIXME: this is silly that we join the modifiers and then immediately split them.
454                 modifiers = self._serialize_parsed_modifiers(test_configuration_converter, specifiers).split()
455                 expectations = self._serialize_parsed_expectations(parsed_expectation_to_string).split()
456                 result.append(self._format_line(modifiers, self.name, expectations, self.comment))
457             return "\n".join(result) if result else None
458
459         return self._format_line(self.modifiers, self.name, self.expectations, self.comment,
460             include_modifiers, include_expectations, include_comment)
461
462     def to_csv(self):
463         # Note that this doesn't include the comments.
464         return '%s,%s,%s' % (self.name, ' '.join(self.modifiers), ' '.join(self.expectations))
465
466     def _serialize_parsed_expectations(self, parsed_expectation_to_string):
467         result = []
468         for index in TestExpectations.EXPECTATION_ORDER:
469             if index in self.parsed_expectations:
470                 result.append(parsed_expectation_to_string[index])
471         return ' '.join(result)
472
473     def _serialize_parsed_modifiers(self, test_configuration_converter, specifiers):
474         result = []
475         if self.parsed_bug_modifiers:
476             result.extend(sorted(self.parsed_bug_modifiers))
477         result.extend(sorted(self.parsed_modifiers))
478         result.extend(test_configuration_converter.specifier_sorter().sort_specifiers(specifiers))
479         return ' '.join(result)
480
481     @staticmethod
482     def _format_line(modifiers, name, expectations, comment, include_modifiers=True, include_expectations=True, include_comment=True):
483         bugs = []
484         new_modifiers = []
485         new_expectations = []
486         for modifier in modifiers:
487             modifier = modifier.upper()
488             if modifier.startswith('BUGWK'):
489                 bugs.append('webkit.org/b/' + modifier.replace('BUGWK', ''))
490             elif modifier.startswith('BUG'):
491                 # FIXME: we should preserve case once we can drop the old syntax.
492                 bugs.append('Bug(' + modifier[3:].lower() + ')')
493             elif modifier in ('SLOW', 'SKIP', 'REBASELINE', 'WONTFIX', 'DUMPJSCONSOLELOGINSTDERR'):
494                 new_expectations.append(TestExpectationParser._inverted_expectation_tokens.get(modifier))
495             else:
496                 new_modifiers.append(TestExpectationParser._inverted_configuration_tokens.get(modifier, modifier))
497
498         for expectation in expectations:
499             expectation = expectation.upper()
500             new_expectations.append(TestExpectationParser._inverted_expectation_tokens.get(expectation, expectation))
501
502         result = ''
503         if include_modifiers and (bugs or new_modifiers):
504             if bugs:
505                 result += ' '.join(bugs) + ' '
506             if new_modifiers:
507                 result += '[ %s ] ' % ' '.join(new_modifiers)
508         result += name
509         if include_expectations and new_expectations and set(new_expectations) != set(['Skip', 'Pass']):
510             result += ' [ %s ]' % ' '.join(sorted(set(new_expectations)))
511         if include_comment and comment is not None:
512             result += " #%s" % comment
513         return result
514
515
516 # FIXME: Refactor API to be a proper CRUD.
517 class TestExpectationsModel(object):
518     """Represents relational store of all expectations and provides CRUD semantics to manage it."""
519
520     def __init__(self, shorten_filename=lambda x: x):
521         # Maps a test to its list of expectations.
522         self._test_to_expectations = {}
523
524         # Maps a test to list of its modifiers (string values)
525         self._test_to_modifiers = {}
526
527         # Maps a test to a TestExpectationLine instance.
528         self._test_to_expectation_line = {}
529
530         self._modifier_to_tests = self._dict_of_sets(TestExpectations.MODIFIERS)
531         self._expectation_to_tests = self._dict_of_sets(TestExpectations.EXPECTATIONS)
532         self._timeline_to_tests = self._dict_of_sets(TestExpectations.TIMELINES)
533         self._result_type_to_tests = self._dict_of_sets(TestExpectations.RESULT_TYPES)
534
535         self._shorten_filename = shorten_filename
536
537     def _dict_of_sets(self, strings_to_constants):
538         """Takes a dict of strings->constants and returns a dict mapping
539         each constant to an empty set."""
540         d = {}
541         for c in strings_to_constants.values():
542             d[c] = set()
543         return d
544
545     def get_test_set(self, modifier, expectation=None, include_skips=True):
546         if expectation is None:
547             tests = self._modifier_to_tests[modifier]
548         else:
549             tests = (self._expectation_to_tests[expectation] &
550                 self._modifier_to_tests[modifier])
551
552         if not include_skips:
553             tests = tests - self.get_test_set(SKIP, expectation)
554
555         return tests
556
557     def get_test_set_for_keyword(self, keyword):
558         # FIXME: get_test_set() is an awkward public interface because it requires
559         # callers to know the difference between modifiers and expectations. We
560         # should replace that with this where possible.
561         expectation_enum = TestExpectations.EXPECTATIONS.get(keyword.lower(), None)
562         if expectation_enum is not None:
563             return self._expectation_to_tests[expectation_enum]
564         modifier_enum = TestExpectations.MODIFIERS.get(keyword.lower(), None)
565         if modifier_enum is not None:
566             return self._modifier_to_tests[modifier_enum]
567
568         # We must not have an index on this modifier.
569         matching_tests = set()
570         for test, modifiers in self._test_to_modifiers.iteritems():
571             if keyword.lower() in modifiers:
572                 matching_tests.add(test)
573         return matching_tests
574
575     def get_tests_with_result_type(self, result_type):
576         return self._result_type_to_tests[result_type]
577
578     def get_tests_with_timeline(self, timeline):
579         return self._timeline_to_tests[timeline]
580
581     def get_modifiers(self, test):
582         """This returns modifiers for the given test (the modifiers plus the BUGXXXX identifier). This is used by the LTTF dashboard."""
583         return self._test_to_modifiers[test]
584
585     def has_modifier(self, test, modifier):
586         return test in self._modifier_to_tests[modifier]
587
588     def has_keyword(self, test, keyword):
589         return (keyword.upper() in self.get_expectations_string(test) or
590                 keyword.lower() in self.get_modifiers(test))
591
592     def has_test(self, test):
593         return test in self._test_to_expectation_line
594
595     def get_expectation_line(self, test):
596         return self._test_to_expectation_line.get(test)
597
598     def get_expectations(self, test):
599         return self._test_to_expectations[test]
600
601     def get_expectations_or_pass(self, test):
602         try:
603             return self.get_expectations(test)
604         except:
605             return set([PASS])
606
607     def expectations_to_string(self, expectations):
608         retval = []
609
610         for expectation in expectations:
611             retval.append(self.expectation_to_string(expectation))
612
613         return " ".join(retval)
614
615     def get_expectations_string(self, test):
616         """Returns the expectations for the given test as an uppercase string.
617         If there are no expectations for the test, then "PASS" is returned."""
618         try:
619             expectations = self.get_expectations(test)
620         except:
621             return "PASS"
622         retval = []
623
624         for expectation in expectations:
625             retval.append(self.expectation_to_string(expectation))
626
627         return " ".join(retval)
628
629     def expectation_to_string(self, expectation):
630         """Return the uppercased string equivalent of a given expectation."""
631         for item in TestExpectations.EXPECTATIONS.items():
632             if item[1] == expectation:
633                 return item[0].upper()
634         raise ValueError(expectation)
635
636     def add_expectation_line(self, expectation_line, in_skipped=False):
637         """Returns a list of warnings encountered while matching modifiers."""
638
639         if expectation_line.is_invalid():
640             return
641
642         for test in expectation_line.matching_tests:
643             if not in_skipped and self._already_seen_better_match(test, expectation_line):
644                 continue
645
646             self._clear_expectations_for_test(test)
647             self._test_to_expectation_line[test] = expectation_line
648             self._add_test(test, expectation_line)
649
650     def _add_test(self, test, expectation_line):
651         """Sets the expected state for a given test.
652
653         This routine assumes the test has not been added before. If it has,
654         use _clear_expectations_for_test() to reset the state prior to
655         calling this."""
656         self._test_to_expectations[test] = expectation_line.parsed_expectations
657         for expectation in expectation_line.parsed_expectations:
658             self._expectation_to_tests[expectation].add(test)
659
660         self._test_to_modifiers[test] = expectation_line.modifiers
661         for modifier in expectation_line.parsed_modifiers:
662             mod_value = TestExpectations.MODIFIERS[modifier]
663             self._modifier_to_tests[mod_value].add(test)
664
665         if TestExpectationParser.WONTFIX_MODIFIER in expectation_line.parsed_modifiers:
666             self._timeline_to_tests[WONTFIX].add(test)
667         else:
668             self._timeline_to_tests[NOW].add(test)
669
670         if TestExpectationParser.SKIP_MODIFIER in expectation_line.parsed_modifiers:
671             self._result_type_to_tests[SKIP].add(test)
672         elif expectation_line.parsed_expectations == set([PASS]):
673             self._result_type_to_tests[PASS].add(test)
674         elif expectation_line.is_flaky():
675             self._result_type_to_tests[FLAKY].add(test)
676         else:
677             # FIXME: What is this?
678             self._result_type_to_tests[FAIL].add(test)
679
680     def _clear_expectations_for_test(self, test):
681         """Remove prexisting expectations for this test.
682         This happens if we are seeing a more precise path
683         than a previous listing.
684         """
685         if self.has_test(test):
686             self._test_to_expectations.pop(test, '')
687             self._remove_from_sets(test, self._expectation_to_tests)
688             self._remove_from_sets(test, self._modifier_to_tests)
689             self._remove_from_sets(test, self._timeline_to_tests)
690             self._remove_from_sets(test, self._result_type_to_tests)
691
692     def _remove_from_sets(self, test, dict_of_sets_of_tests):
693         """Removes the given test from the sets in the dictionary.
694
695         Args:
696           test: test to look for
697           dict: dict of sets of files"""
698         for set_of_tests in dict_of_sets_of_tests.itervalues():
699             if test in set_of_tests:
700                 set_of_tests.remove(test)
701
702     def _already_seen_better_match(self, test, expectation_line):
703         """Returns whether we've seen a better match already in the file.
704
705         Returns True if we've already seen a expectation_line.name that matches more of the test
706             than this path does
707         """
708         # FIXME: See comment below about matching test configs and specificity.
709         if not self.has_test(test):
710             # We've never seen this test before.
711             return False
712
713         prev_expectation_line = self._test_to_expectation_line[test]
714
715         if prev_expectation_line.filename != expectation_line.filename:
716             # We've moved on to a new expectation file, which overrides older ones.
717             return False
718
719         if len(prev_expectation_line.path) > len(expectation_line.path):
720             # The previous path matched more of the test.
721             return True
722
723         if len(prev_expectation_line.path) < len(expectation_line.path):
724             # This path matches more of the test.
725             return False
726
727         # At this point we know we have seen a previous exact match on this
728         # base path, so we need to check the two sets of modifiers.
729
730         # FIXME: This code was originally designed to allow lines that matched
731         # more modifiers to override lines that matched fewer modifiers.
732         # However, we currently view these as errors.
733         #
734         # To use the "more modifiers wins" policy, change the errors for overrides
735         # to be warnings and return False".
736         shortened_expectation_filename = self._shorten_filename(expectation_line.filename)
737         shortened_previous_expectation_filename = self._shorten_filename(prev_expectation_line.filename)
738
739         if prev_expectation_line.matching_configurations == expectation_line.matching_configurations:
740             expectation_line.warnings.append('Duplicate or ambiguous entry lines %s:%d and %s:%d.' % (
741                 shortened_previous_expectation_filename, prev_expectation_line.line_number,
742                 shortened_expectation_filename, expectation_line.line_number))
743
744         elif prev_expectation_line.matching_configurations >= expectation_line.matching_configurations:
745             expectation_line.warnings.append('More specific entry for %s on line %s:%d overrides line %s:%d.' % (expectation_line.name,
746                 shortened_previous_expectation_filename, prev_expectation_line.line_number,
747                 shortened_expectation_filename, expectation_line.line_number))
748             # FIXME: return False if we want more specific to win.
749
750         elif prev_expectation_line.matching_configurations <= expectation_line.matching_configurations:
751             expectation_line.warnings.append('More specific entry for %s on line %s:%d overrides line %s:%d.' % (expectation_line.name,
752                 shortened_expectation_filename, expectation_line.line_number,
753                 shortened_previous_expectation_filename, prev_expectation_line.line_number))
754
755         elif prev_expectation_line.matching_configurations & expectation_line.matching_configurations:
756             expectation_line.warnings.append('Entries for %s on lines %s:%d and %s:%d match overlapping sets of configurations.' % (expectation_line.name,
757                 shortened_previous_expectation_filename, prev_expectation_line.line_number,
758                 shortened_expectation_filename, expectation_line.line_number))
759
760         else:
761             # Configuration sets are disjoint.
762             return False
763
764         # Missing files will be 'None'. It should be impossible to have a missing file which also has a line associated with it.
765         assert shortened_previous_expectation_filename not in expectation_line.related_files or expectation_line.related_files[shortened_previous_expectation_filename] is not None
766         expectation_line.related_files[shortened_previous_expectation_filename] = expectation_line.related_files.get(shortened_previous_expectation_filename, [])
767         expectation_line.related_files[shortened_previous_expectation_filename].append(prev_expectation_line.line_number)
768
769         assert shortened_expectation_filename not in prev_expectation_line.related_files or prev_expectation_line.related_files[shortened_expectation_filename] is not None
770         prev_expectation_line.related_files[shortened_expectation_filename] = prev_expectation_line.related_files.get(shortened_expectation_filename, [])
771         prev_expectation_line.related_files[shortened_expectation_filename].append(expectation_line.line_number)
772
773         return True
774
775
776 class TestExpectations(object):
777     """Test expectations consist of lines with specifications of what
778     to expect from layout test cases. The test cases can be directories
779     in which case the expectations apply to all test cases in that
780     directory and any subdirectory. The format is along the lines of:
781
782       LayoutTests/js/fixme.js [ Failure ]
783       LayoutTests/js/flaky.js [ Failure Pass ]
784       LayoutTests/js/crash.js [ Crash Failure Pass Timeout ]
785       ...
786
787     To add modifiers:
788       LayoutTests/js/no-good.js
789       [ Debug ] LayoutTests/js/no-good.js [ Pass Timeout ]
790       [ Debug ] LayoutTests/js/no-good.js [ Pass Skip Timeout ]
791       [ Linux Debug ] LayoutTests/js/no-good.js [ Pass Skip Timeout ]
792       [ Linux Win ] LayoutTests/js/no-good.js [ Pass Skip Timeout ]
793
794     Skip: Doesn't run the test.
795     Slow: The test takes a long time to run, but does not timeout indefinitely.
796     WontFix: For tests that we never intend to pass on a given platform (treated like Skip).
797
798     Notes:
799       -A test cannot be both SLOW and TIMEOUT
800       -A test can be included twice, but not via the same path.
801       -If a test is included twice, then the more precise path wins.
802       -CRASH tests cannot be WONTFIX
803     """
804
805     # FIXME: Update to new syntax once the old format is no longer supported.
806     EXPECTATIONS = {'pass': PASS,
807                     'audio': AUDIO,
808                     'fail': FAIL,
809                     'image': IMAGE,
810                     'image+text': IMAGE_PLUS_TEXT,
811                     'text': TEXT,
812                     'timeout': TIMEOUT,
813                     'crash': CRASH,
814                     'missing': MISSING,
815                     'leak': LEAK,
816                     'skip': SKIP}
817
818     # Singulars
819     EXPECTATION_DESCRIPTION = {SKIP: 'skipped',
820                                 PASS: 'pass',
821                                 FAIL: 'failure',
822                                 IMAGE: 'image-only failure',
823                                 TEXT: 'text-only failure',
824                                 IMAGE_PLUS_TEXT: 'image and text failure',
825                                 AUDIO: 'audio failure',
826                                 CRASH: 'crash',
827                                 TIMEOUT: 'timeout',
828                                 MISSING: 'missing',
829                                 LEAK: 'leak'}
830
831     # (aggregated by category, pass/fail/skip, type)
832     EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped',
833                                 PASS: 'passes',
834                                 FAIL: 'failures',
835                                 IMAGE: 'image-only failures',
836                                 TEXT: 'text-only failures',
837                                 IMAGE_PLUS_TEXT: 'image and text failures',
838                                 AUDIO: 'audio failures',
839                                 CRASH: 'crashes',
840                                 TIMEOUT: 'timeouts',
841                                 MISSING: 'missing results',
842                                 LEAK: 'leaks'}
843
844     EXPECTATION_ORDER = (PASS, CRASH, TIMEOUT, MISSING, FAIL, IMAGE, LEAK, SKIP)
845
846     BUILD_TYPES = ('debug', 'release')
847
848     MODIFIERS = {TestExpectationParser.SKIP_MODIFIER: SKIP,
849                  TestExpectationParser.WONTFIX_MODIFIER: WONTFIX,
850                  TestExpectationParser.SLOW_MODIFIER: SLOW,
851                  TestExpectationParser.DUMPJSCONSOLELOGINSTDERR_MODIFIER: DUMPJSCONSOLELOGINSTDERR,
852                  TestExpectationParser.REBASELINE_MODIFIER: REBASELINE,
853                  'none': NONE}
854
855     TIMELINES = {TestExpectationParser.WONTFIX_MODIFIER: WONTFIX,
856                  'now': NOW}
857
858     RESULT_TYPES = {'skip': SKIP,
859                     'pass': PASS,
860                     'fail': FAIL,
861                     'flaky': FLAKY}
862
863     @classmethod
864     def expectation_from_string(cls, string):
865         assert(' ' not in string)  # This only handles one expectation at a time.
866         return cls.EXPECTATIONS.get(string.lower())
867
868     @staticmethod
869     def result_was_expected(result, expected_results, test_needs_rebaselining, test_is_skipped):
870         """Returns whether we got a result we were expecting.
871         Args:
872             result: actual result of a test execution
873             expected_results: set of results listed in test_expectations
874             test_needs_rebaselining: whether test was marked as REBASELINE
875             test_is_skipped: whether test was marked as SKIP"""
876         if result in expected_results:
877             return True
878         if result in (TEXT, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected_results):
879             return True
880         if result == MISSING and test_needs_rebaselining:
881             return True
882         if result == SKIP and test_is_skipped:
883             return True
884         return False
885
886     @staticmethod
887     def remove_pixel_failures(expected_results):
888         """Returns a copy of the expected results for a test, except that we
889         drop any pixel failures and return the remaining expectations. For example,
890         if we're not running pixel tests, then tests expected to fail as IMAGE
891         will PASS."""
892         expected_results = expected_results.copy()
893         if IMAGE in expected_results:
894             expected_results.remove(IMAGE)
895             expected_results.add(PASS) # FIXME: does it always become a pass?
896         return expected_results
897
898     @staticmethod
899     def remove_leak_failures(expected_results):
900         """Returns a copy of the expected results for a test, except that we
901         drop any leak failures and return the remaining expectations. For example,
902         if we're not running with --world-leaks, then tests expected to fail as LEAK
903         will PASS."""
904         expected_results = expected_results.copy()
905         if LEAK in expected_results:
906             expected_results.remove(LEAK)
907             if not expected_results:
908                 expected_results.add(PASS)
909         return expected_results
910
911     @staticmethod
912     def has_pixel_failures(actual_results):
913         return IMAGE in actual_results or FAIL in actual_results
914
915     @staticmethod
916     def suffixes_for_expectations(expectations):
917         suffixes = set()
918         if IMAGE in expectations:
919             suffixes.add('png')
920         if FAIL in expectations:
921             suffixes.add('txt')
922             suffixes.add('png')
923             suffixes.add('wav')
924         return set(suffixes)
925
926     def __init__(self, port, tests=None, include_generic=True, include_overrides=True, expectations_to_lint=None, force_expectations_pass=False, device_type=None):
927         self._full_test_list = tests
928         self._test_config = port.test_configuration()
929         self._is_lint_mode = expectations_to_lint is not None
930         self._model = TestExpectationsModel(self._shorten_filename)
931         self._parser = TestExpectationParser(port, tests, self._is_lint_mode, self._shorten_filename)
932         self._port = port
933         self._skipped_tests_warnings = []
934         self._expectations = []
935         self._force_expectations_pass = force_expectations_pass
936         self._device_type = device_type
937         self._include_generic = include_generic
938         self._include_overrides = include_overrides
939         self._expectations_to_lint = expectations_to_lint
940
941     def readable_filename_and_line_number(self, line):
942         if line.not_applicable_to_current_platform:
943             return "(skipped for this platform)"
944         if not line.filename:
945             return ''
946         if line.filename.startswith(self._port.path_from_webkit_base()):
947             return '{}:{}'.format(self._port.host.filesystem.relpath(line.filename, self._port.path_from_webkit_base()), line.line_number)
948         return '{}:{}'.format(line.filename, line.line_number)
949
950     def parse_generic_expectations(self):
951         if self._port.path_to_generic_test_expectations_file() in self._expectations_dict:
952             if self._include_generic:
953                 expectations = self._parser.parse(self._expectations_dict.keys()[self._expectations_dict_index], self._expectations_dict.values()[self._expectations_dict_index])
954                 self._add_expectations(expectations)
955                 self._expectations += expectations
956             self._expectations_dict_index += 1
957
958     def parse_default_port_expectations(self):
959         if len(self._expectations_dict) > self._expectations_dict_index:
960             expectations = self._parser.parse(self._expectations_dict.keys()[self._expectations_dict_index], self._expectations_dict.values()[self._expectations_dict_index])
961             self._add_expectations(expectations)
962             self._expectations += expectations
963             self._expectations_dict_index += 1
964
965     def parse_override_expectations(self):
966         while len(self._expectations_dict) > self._expectations_dict_index and self._include_overrides:
967             expectations = self._parser.parse(self._expectations_dict.keys()[self._expectations_dict_index], self._expectations_dict.values()[self._expectations_dict_index])
968             self._add_expectations(expectations)
969             self._expectations += expectations
970             self._expectations_dict_index += 1
971
972     def parse_all_expectations(self):
973         self._expectations_dict = self._expectations_to_lint or self._port.expectations_dict(device_type=self._device_type)
974         self._expectations_dict_index = 0
975
976         self._has_warnings = False
977
978         self.parse_generic_expectations()
979         self.parse_default_port_expectations()
980         self.parse_override_expectations()
981
982         # FIXME: move ignore_tests into port.skipped_layout_tests()
983         self.add_skipped_tests(self._port.skipped_layout_tests(self._full_test_list, device_type=self._device_type).union(set(self._port.get_option('ignore_tests', []))))
984
985         self._report_warnings()
986         self._process_tests_without_expectations()
987
988     # TODO(ojan): Allow for removing skipped tests when getting the list of
989     # tests to run, but not when getting metrics.
990     def model(self):
991         return self._model
992
993     def get_rebaselining_failures(self):
994         return self._model.get_test_set(REBASELINE)
995
996     def filtered_expectations_for_test(self, test, pixel_tests_are_enabled, world_leaks_are_enabled):
997         expected_results = self._model.get_expectations_or_pass(test)
998         if not pixel_tests_are_enabled:
999             expected_results = self.remove_pixel_failures(expected_results)
1000         if not world_leaks_are_enabled:
1001             expected_results = self.remove_leak_failures(expected_results)
1002
1003         return expected_results
1004
1005     def matches_an_expected_result(self, test, result, expected_results):
1006         return self.result_was_expected(result,
1007                                    expected_results,
1008                                    self.is_rebaselining(test),
1009                                    self._model.has_modifier(test, SKIP))
1010
1011     def is_rebaselining(self, test):
1012         return self._model.has_modifier(test, REBASELINE)
1013
1014     def _shorten_filename(self, filename):
1015         if filename.startswith(self._port.path_from_webkit_base()):
1016             return self._port.host.filesystem.relpath(filename, self._port.path_from_webkit_base())
1017         return filename
1018
1019     def _report_warnings(self):
1020         warnings = []
1021         for expectation in self._expectations:
1022             for warning in expectation.warnings:
1023                 warning = TestExpectationWarning(
1024                     self._shorten_filename(expectation.filename), expectation.line_number,
1025                     expectation.original_string, warning, expectation.name if expectation.expectations else None)
1026                 warning.related_files = expectation.related_files
1027                 warnings.append(warning)
1028
1029         if warnings:
1030             self._has_warnings = True
1031             if self._is_lint_mode:
1032                 raise ParseError(warnings)
1033             _log.warning('--lint-test-files warnings:')
1034             for warning in warnings:
1035                 _log.warning(warning)
1036             _log.warning('')
1037
1038     def _process_tests_without_expectations(self):
1039         if self._full_test_list:
1040             for test in self._full_test_list:
1041                 if not self._model.has_test(test):
1042                     self._model.add_expectation_line(TestExpectationLine.create_passing_expectation(test))
1043
1044     def has_warnings(self):
1045         return self._has_warnings
1046
1047     def remove_configuration_from_test(self, test, test_configuration):
1048         expectations_to_remove = []
1049         modified_expectations = []
1050
1051         for expectation in self._expectations:
1052             if expectation.name != test or expectation.is_flaky() or not expectation.parsed_expectations:
1053                 continue
1054             if iter(expectation.parsed_expectations).next() not in (FAIL, IMAGE):
1055                 continue
1056             if test_configuration not in expectation.matching_configurations:
1057                 continue
1058
1059             expectation.matching_configurations.remove(test_configuration)
1060             if expectation.matching_configurations:
1061                 modified_expectations.append(expectation)
1062             else:
1063                 expectations_to_remove.append(expectation)
1064
1065         for expectation in expectations_to_remove:
1066             self._expectations.remove(expectation)
1067
1068         return self.list_to_string(self._expectations, self._parser._test_configuration_converter, modified_expectations)
1069
1070     def remove_rebaselined_tests(self, except_these_tests, filename):
1071         """Returns a copy of the expectations in the file with the tests removed."""
1072         def without_rebaseline_modifier(expectation):
1073             return (expectation.filename == filename and
1074                     not (not expectation.is_invalid() and
1075                          expectation.name in except_these_tests and
1076                          'rebaseline' in expectation.parsed_modifiers))
1077
1078         return self.list_to_string(filter(without_rebaseline_modifier, self._expectations), reconstitute_only_these=[])
1079
1080     def _add_expectations(self, expectation_list):
1081         for expectation_line in expectation_list:
1082             if self._force_expectations_pass:
1083                 expectation_line.expectations = ['PASS']
1084                 expectation_line.parsed_expectations = set([PASS])
1085
1086             elif not expectation_line.expectations:
1087                 continue
1088
1089             if self._is_lint_mode or self._test_config in expectation_line.matching_configurations:
1090                 self._model.add_expectation_line(expectation_line)
1091
1092     def add_skipped_tests(self, tests_to_skip):
1093         if not tests_to_skip:
1094             return
1095         for test in self._expectations:
1096             if test.name and test.name in tests_to_skip:
1097                 test.warnings.append('%s:%d %s is also in a Skipped file.' % (test.filename, test.line_number, test.name))
1098
1099         for test_name in tests_to_skip:
1100             expectation_line = self._parser.expectation_for_skipped_test(test_name)
1101             self._model.add_expectation_line(expectation_line, in_skipped=True)
1102
1103     @staticmethod
1104     def list_to_string(expectation_lines, test_configuration_converter=None, reconstitute_only_these=None):
1105         def serialize(expectation_line):
1106             # If reconstitute_only_these is an empty list, we want to return original_string.
1107             # So we need to compare reconstitute_only_these to None, not just check if it's falsey.
1108             if reconstitute_only_these is None or expectation_line in reconstitute_only_these:
1109                 return expectation_line.to_string(test_configuration_converter)
1110             return expectation_line.original_string
1111
1112         def nones_out(expectation_line):
1113             return expectation_line is not None
1114
1115         return "\n".join(filter(nones_out, map(serialize, expectation_lines)))