Fix a bug where optimize-baselines would incorrectly fail to optimize
[WebKit-https.git] / Tools / Scripts / webkitpy / common / checkout / baselineoptimizer_unittest.py
1 # Copyright (C) 2011 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #    * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #    * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #    * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import sys
30 import unittest
31
32 from webkitpy.common.checkout.baselineoptimizer import BaselineOptimizer
33 from webkitpy.common.system.filesystem_mock import MockFileSystem
34 from webkitpy.common.host_mock import MockHost
35
36
37 class TestBaselineOptimizer(BaselineOptimizer):
38     def __init__(self, mock_results_by_directory):
39         host = MockHost()
40         BaselineOptimizer.__init__(self, host)
41         self._mock_results_by_directory = mock_results_by_directory
42
43     # We override this method for testing so we don't have to construct an
44     # elaborate mock file system.
45     def _read_results_by_directory(self, baseline_name):
46         return self._mock_results_by_directory
47
48     def _move_baselines(self, baseline_name, results_by_directory, new_results_by_directory):
49         self.new_results_by_directory = new_results_by_directory
50
51
52 class BaselineOptimizerTest(unittest.TestCase):
53     def _assertOptimization(self, results_by_directory, expected_new_results_by_directory):
54         baseline_optimizer = TestBaselineOptimizer(results_by_directory)
55         self.assertTrue(baseline_optimizer.optimize('mock-baseline.png'))
56         self.assertEqual(baseline_optimizer.new_results_by_directory, expected_new_results_by_directory)
57
58     def _assertOptimizationFailed(self, results_by_directory):
59         baseline_optimizer = TestBaselineOptimizer(results_by_directory)
60         self.assertFalse(baseline_optimizer.optimize('mock-baseline.png'))
61
62     def test_move_baselines(self):
63         host = MockHost()
64         host.filesystem.write_binary_file('/mock-checkout/LayoutTests/platform/chromium-win/another/test-expected.txt', 'result A')
65         host.filesystem.write_binary_file('/mock-checkout/LayoutTests/platform/chromium-mac/another/test-expected.txt', 'result A')
66         host.filesystem.write_binary_file('/mock-checkout/LayoutTests/platform/chromium/another/test-expected.txt', 'result B')
67         baseline_optimizer = BaselineOptimizer(host)
68         baseline_optimizer._move_baselines('another/test-expected.txt', {
69             'LayoutTests/platform/chromium-win': 'aaa',
70             'LayoutTests/platform/chromium-mac': 'aaa',
71             'LayoutTests/platform/chromium': 'bbb',
72         }, {
73             'LayoutTests/platform/chromium': 'aaa',
74         })
75         self.assertEqual(host.filesystem.read_binary_file('/mock-checkout/LayoutTests/platform/chromium/another/test-expected.txt'), 'result A')
76
77     def test_chromium_linux_redundant_with_win(self):
78         self._assertOptimization({
79             'LayoutTests/platform/chromium-win': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
80             'LayoutTests/platform/chromium-linux': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
81         }, {
82             'LayoutTests/platform/chromium-win': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
83         })
84
85     def test_no_add_mac_future(self):
86         self._assertOptimization({
87             'LayoutTests/platform/mac': '29a1715a6470d5dd9486a142f609708de84cdac8',
88             'LayoutTests/platform/win-xp': '453e67177a75b2e79905154ece0efba6e5bfb65d',
89             'LayoutTests/platform/mac-lion': 'c43eaeb358f49d5e835236ae23b7e49d7f2b089f',
90             'LayoutTests/platform/chromium-mac': 'a9ba153c700a94ae1b206d8e4a75a621a89b4554',
91         }, {
92             'LayoutTests/platform/mac': '29a1715a6470d5dd9486a142f609708de84cdac8',
93             'LayoutTests/platform/win-xp': '453e67177a75b2e79905154ece0efba6e5bfb65d',
94             'LayoutTests/platform/mac-lion': 'c43eaeb358f49d5e835236ae23b7e49d7f2b089f',
95             'LayoutTests/platform/chromium-mac': 'a9ba153c700a94ae1b206d8e4a75a621a89b4554',
96         })
97
98     def test_chromium_covers_mac_win_linux(self):
99         self._assertOptimization({
100             'LayoutTests/platform/chromium-mac': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
101             'LayoutTests/platform/chromium-win': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
102             'LayoutTests/platform/chromium-linux': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
103         }, {
104             'LayoutTests/platform/chromium': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
105         })
106
107     def test_chromium_mac_redundant_with_apple_mac(self):
108         self._assertOptimization({
109             'LayoutTests/platform/chromium-mac': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
110             'LayoutTests/platform/mac': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
111         }, {
112             'LayoutTests/platform/mac': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
113         })
114
115     def test_mac_future(self):
116         self._assertOptimization({
117             'LayoutTests/platform/mac-lion': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
118         }, {
119             'LayoutTests/platform/mac-lion': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
120         })
121
122     def test_qt_unknown(self):
123         self._assertOptimization({
124             'LayoutTests/platform/qt': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
125         }, {
126             'LayoutTests/platform/qt': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
127         })
128
129     def test_win_does_not_drop_to_win_7sp0(self):
130         self._assertOptimization({
131             'LayoutTests/platform/win': '1',
132             'LayoutTests/platform/mac': '2',
133             'LayoutTests/platform/gtk': '3',
134             'LayoutTests/platform/qt': '4',
135             'LayoutTests/platform/chromium': '5',
136         }, {
137             'LayoutTests/platform/win': '1',
138             'LayoutTests/platform/mac': '2',
139             'LayoutTests/platform/gtk': '3',
140             'LayoutTests/platform/qt': '4',
141             'LayoutTests/platform/chromium': '5',
142         })
143
144     def test_common_directory_includes_root(self):
145         # This test case checks that we don't throw an exception when we fail
146         # to optimize.
147         self._assertOptimizationFailed({
148             'LayoutTests/platform/gtk': 'e8608763f6241ddacdd5c1ef1973ba27177d0846',
149             'LayoutTests/platform/qt': 'bcbd457d545986b7abf1221655d722363079ac87',
150             'LayoutTests/platform/chromium-win': '3764ac11e1f9fbadd87a90a2e40278319190a0d3',
151             'LayoutTests/platform/mac': 'e8608763f6241ddacdd5c1ef1973ba27177d0846',
152         })
153
154         self._assertOptimization({
155             'LayoutTests/platform/chromium-win': '23a30302a6910f8a48b1007fa36f3e3158341834',
156             'LayoutTests': '9c876f8c3e4cc2aef9519a6c1174eb3432591127',
157             'LayoutTests/platform/chromium-mac': '23a30302a6910f8a48b1007fa36f3e3158341834',
158             'LayoutTests/platform/chromium': '1',
159         }, {
160             'LayoutTests/platform/chromium': '23a30302a6910f8a48b1007fa36f3e3158341834',
161             'LayoutTests': '9c876f8c3e4cc2aef9519a6c1174eb3432591127',
162         })
163
164     def test_complex_shadowing(self):
165         # This test relies on OS specific functionality, so it doesn't work on Windows.
166         # FIXME: What functionality does this rely on?  When can we remove this if?
167         if sys.platform == 'win32':
168             return
169         self._assertOptimization({
170             'LayoutTests/platform/chromium-win': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
171             'LayoutTests/platform/mac': '5daa78e55f05d9f0d1bb1f32b0cd1bc3a01e9364',
172             'LayoutTests/platform/chromium-win-xp': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
173             'LayoutTests/platform/mac-lion': '7ad045ece7c030e2283c5d21d9587be22bcba56e',
174             'LayoutTests/platform/chromium-win': 'f83af9732ce74f702b8c9c4a3d9a4c6636b8d3bd',
175             'LayoutTests/platform/win-xp': '5b1253ef4d5094530d5f1bc6cdb95c90b446bec7',
176             'LayoutTests/platform/chromium-linux': 'f52fcdde9e4be8bd5142171cd859230bd4471036',
177         }, {
178             'LayoutTests/platform/chromium-win': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
179             'LayoutTests/platform/mac': '5daa78e55f05d9f0d1bb1f32b0cd1bc3a01e9364',
180             'LayoutTests/platform/chromium-win-xp': '462d03b9c025db1b0392d7453310dbee5f9a9e74',
181             'LayoutTests/platform/mac-lion': '7ad045ece7c030e2283c5d21d9587be22bcba56e',
182             'LayoutTests/platform/chromium-win': 'f83af9732ce74f702b8c9c4a3d9a4c6636b8d3bd',
183             'LayoutTests/platform/win-xp': '5b1253ef4d5094530d5f1bc6cdb95c90b446bec7',
184             'LayoutTests/platform/chromium-linux': 'f52fcdde9e4be8bd5142171cd859230bd4471036'
185         })
186
187     def test_virtual_ports_filtered(self):
188         self._assertOptimization({
189             'LayoutTests/platform/chromium-mac': '1',
190             'LayoutTests/platform/chromium-mac-snowleopard': '1',
191             'LayoutTests/platform/chromium-win': '2',
192             'LayoutTests/platform/gtk': '3',
193             'LayoutTests/platform/efl': '3',
194             'LayoutTests/platform/qt': '4',
195             'LayoutTests/platform/mac': '5',
196         }, {
197             'LayoutTests/platform/chromium-mac': '1',
198             'LayoutTests/platform/chromium-win': '2',
199             'LayoutTests': '3',
200             'LayoutTests/platform/qt': '4',
201             'LayoutTests/platform/mac': '5',
202         })