webkitpy: Implement device type specific expected results (Part 2)
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / layout_package / json_layout_results_generator.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import logging
30
31 from webkitpy.layout_tests.layout_package import json_results_generator
32 from webkitpy.layout_tests.models import test_expectations
33 from webkitpy.layout_tests.models import test_failures
34
35
36 class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGenerator):
37     """A JSON results generator for layout tests."""
38
39     LAYOUT_TESTS_PATH = "LayoutTests"
40
41     # Additional JSON fields.
42     WONTFIX = "wontfixCounts"
43
44     FAILURE_TO_CHAR = {test_expectations.PASS: json_results_generator.JSONResultsGenerator.PASS_RESULT,
45                        test_expectations.SKIP: json_results_generator.JSONResultsGenerator.SKIP_RESULT,
46                        test_expectations.CRASH: "C",
47                        test_expectations.TIMEOUT: "T",
48                        test_expectations.IMAGE: "I",
49                        test_expectations.TEXT: "F",
50                        test_expectations.AUDIO: "A",
51                        test_expectations.MISSING: "O",
52                        test_expectations.LEAK: "L",
53                        test_expectations.IMAGE_PLUS_TEXT: "Z"}
54
55     def __init__(self, port, builder_name, build_name, build_number,
56         results_file_base_path,
57         expectations_by_type, run_results,
58         test_results_servers=[], test_type="", master_name=""):
59         """Modifies the results.json file. Grabs it off the archive directory
60         if it is not found locally.
61
62         Args:
63           run_results: TestRunResults object storing the details of the test run.
64         """
65         super(JSONLayoutResultsGenerator, self).__init__(
66             port, builder_name, build_name, build_number, results_file_base_path,
67             {}, port.repository_paths(),
68             test_results_servers, test_type, master_name)
69
70         self._expectations = expectations_by_type
71
72         self._run_results = run_results
73         self._failures = dict((test_name, run_results.results_by_name[test_name].type) for test_name in run_results.failures_by_name)
74         self._test_timings = run_results.results_by_name
75
76     def _get_path_relative_to_layout_test_root(self, test):
77         """Returns the path of the test relative to the layout test root.
78         For example, for:
79           src/third_party/WebKit/LayoutTests/fast/forms/foo.html
80         We would return
81           fast/forms/foo.html
82         """
83         index = test.find(self.LAYOUT_TESTS_PATH)
84         if index is not -1:
85             index += len(self.LAYOUT_TESTS_PATH)
86
87         if index is -1:
88             # Already a relative path.
89             relativePath = test
90         else:
91             relativePath = test[index + 1:]
92
93         # Make sure all paths are unix-style.
94         return relativePath.replace('\\', '/')
95
96     # override
97     def _get_test_timing(self, test_name):
98         if test_name in self._test_timings:
99             # Floor for now to get time in seconds.
100             return int(self._test_timings[test_name].test_run_time)
101         return 0
102
103     # override
104     def _get_failed_test_names(self):
105         return set(self._failures.keys())
106
107     # override
108     def _get_modifier_char(self, test_name):
109         if test_name not in self._run_results.results_by_name:
110             return self.NO_DATA_RESULT
111
112         if test_name in self._failures:
113             return self.FAILURE_TO_CHAR[self._failures[test_name]]
114
115         return self.PASS_RESULT
116
117     # override
118     def _get_result_char(self, test_name):
119         return self._get_modifier_char(test_name)
120
121     # override
122     def _insert_failure_summaries(self, results_for_builder):
123         run_results = self._run_results
124
125         self._insert_item_into_raw_list(results_for_builder,
126             len((set(run_results.failures_by_name.keys()) |
127                 run_results.tests_by_expectation[test_expectations.SKIP]) &
128                 run_results.tests_by_timeline[test_expectations.NOW]),
129             self.FIXABLE_COUNT)
130         self._insert_item_into_raw_list(results_for_builder,
131             self._get_failure_summary_entry(test_expectations.NOW),
132             self.FIXABLE)
133
134         num_fixable = 0
135         for expectation in self._expectations.itervalues():
136             num_fixable += len(expectation.model().get_tests_with_timeline(test_expectations.NOW))
137
138         self._insert_item_into_raw_list(results_for_builder, num_fixable, self.ALL_FIXABLE_COUNT)
139         self._insert_item_into_raw_list(results_for_builder,
140             self._get_failure_summary_entry(test_expectations.WONTFIX),
141             self.WONTFIX)
142
143     # override
144     def _normalize_results_json(self, test, test_name, tests):
145         super(JSONLayoutResultsGenerator, self)._normalize_results_json(
146             test, test_name, tests)
147
148         # Remove tests that don't exist anymore.
149         full_path = self._filesystem.join(self._port.layout_tests_dir(), test_name)
150         full_path = self._filesystem.normpath(full_path)
151         if not self._filesystem.exists(full_path):
152             del tests[test_name]
153
154     def _get_failure_summary_entry(self, timeline):
155         """Creates a summary object to insert into the JSON.
156
157         Args:
158           timeline  current test_expectations timeline to build entry for
159                     (e.g., test_expectations.NOW, etc.)
160         """
161         entry = {}
162         run_results = self._run_results
163         timeline_tests = run_results.tests_by_timeline[timeline]
164         entry[self.SKIP_RESULT] = len(
165             run_results.tests_by_expectation[test_expectations.SKIP] &
166             timeline_tests)
167         entry[self.PASS_RESULT] = len(
168             run_results.tests_by_expectation[test_expectations.PASS] &
169             timeline_tests)
170         for failure_type in run_results.tests_by_expectation.keys():
171             if failure_type not in self.FAILURE_TO_CHAR:
172                 continue
173             count = len(run_results.tests_by_expectation[failure_type] &
174                         timeline_tests)
175             entry[self.FAILURE_TO_CHAR[failure_type]] = count
176         return entry