webkitpy: clean up options for specifying multiple platforms at once
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / rebaseline_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #    * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #    * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #    * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import unittest
30
31 from webkitpy.common.system.outputcapture import OutputCapture
32 from webkitpy.common.checkout.baselineoptimizer import BaselineOptimizer
33 from webkitpy.common.net.buildbot.buildbot_mock import MockBuilder
34 from webkitpy.common.system.executive_mock import MockExecutive2
35 from webkitpy.thirdparty.mock import Mock
36 from webkitpy.tool.commands.rebaseline import *
37 from webkitpy.tool.mocktool import MockTool, MockOptions
38
39
40 class _BaseTestCase(unittest.TestCase):
41     MOCK_WEB_RESULT = 'MOCK Web result, convert 404 to None=True'
42     WEB_PREFIX = 'http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results'
43
44     command_constructor = None
45
46     def setUp(self):
47         self.tool = MockTool()
48         self.command = self.command_constructor()
49         self.command.bind_to_tool(self.tool)
50         self.lion_port = self.tool.port_factory.get_from_builder_name("WebKit Mac10.7")
51         self.lion_expectations_path = self.lion_port.path_to_test_expectations_file()
52
53         # FIXME: we should override builders._exact_matches here to point to a set
54         # of test ports and restore the value in tearDown(), and that way the
55         # individual tests wouldn't have to worry about it.
56
57     def _expand(self, path):
58         if self.tool.filesystem.isabs(path):
59             return path
60         return self.tool.filesystem.join(self.lion_port.layout_tests_dir(), path)
61
62     def _read(self, path):
63         return self.tool.filesystem.read_text_file(self._expand(path))
64
65     def _write(self, path, contents):
66         self.tool.filesystem.write_text_file(self._expand(path), contents)
67
68     def _zero_out_test_expectations(self):
69         for port_name in self.tool.port_factory.all_port_names():
70             port = self.tool.port_factory.get(port_name)
71             for path in port.expectations_files():
72                 self._write(path, '')
73         self.tool.filesystem.written_files = {}
74
75
76 class TestRebaselineTest(_BaseTestCase):
77     command_constructor = RebaselineTest  # AKA webkit-patch rebaseline-test-internal
78
79     def test_baseline_directory(self):
80         command = self.command
81         self.assertEqual(command._baseline_directory("Apple Win XP Debug (Tests)"), "/mock-checkout/LayoutTests/platform/win-xp")
82         self.assertEqual(command._baseline_directory("Apple Win 7 Release (Tests)"), "/mock-checkout/LayoutTests/platform/win")
83         self.assertEqual(command._baseline_directory("Apple Lion Release WK1 (Tests)"), "/mock-checkout/LayoutTests/platform/mac-lion")
84         self.assertEqual(command._baseline_directory("Apple Lion Release WK2 (Tests)"), "/mock-checkout/LayoutTests/platform/mac-wk2")
85         self.assertEqual(command._baseline_directory("GTK Linux 32-bit Release"), "/mock-checkout/LayoutTests/platform/gtk")
86         self.assertEqual(command._baseline_directory("EFL Linux 64-bit Debug"), "/mock-checkout/LayoutTests/platform/efl-wk1")
87         self.assertEqual(command._baseline_directory("Qt Linux Release"), "/mock-checkout/LayoutTests/platform/qt")
88         self.assertEqual(command._baseline_directory("WebKit Mac10.7"), "/mock-checkout/LayoutTests/platform/chromium-mac-lion")
89         self.assertEqual(command._baseline_directory("WebKit Mac10.6"), "/mock-checkout/LayoutTests/platform/chromium-mac-snowleopard")
90
91     def test_rebaseline_updates_expectations_file_noop(self):
92         self._zero_out_test_expectations()
93         self._write(self.lion_expectations_path, """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ]
94 Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ]
95 """)
96         self._write("fast/dom/Window/window-postmessage-clone-really-deep-array.html", "Dummy test contents")
97         self._write("fast/css/large-list-of-rules-crash.html", "Dummy test contents")
98         self._write("userscripts/another-test.html", "Dummy test contents")
99
100         self.command._rebaseline_test_and_update_expectations("WebKit Mac10.7", "userscripts/another-test.html", None, self.WEB_PREFIX)
101
102         self.assertEquals(self.tool.web.urls_fetched,
103             [self.WEB_PREFIX + '/userscripts/another-test-actual.png',
104              self.WEB_PREFIX + '/userscripts/another-test-actual.wav',
105              self.WEB_PREFIX + '/userscripts/another-test-actual.txt'])
106         new_expectations = self._read(self.lion_expectations_path)
107         self.assertEqual(new_expectations, """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ]
108 Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ]
109 """)
110
111     def test_rebaseline_updates_expectations_file(self):
112         self._write(self.lion_expectations_path, "Bug(x) [ Mac ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n")
113         self._write("userscripts/another-test.html", "Dummy test contents")
114
115         self.command._rebaseline_test_and_update_expectations("WebKit Mac10.7", "userscripts/another-test.html", None, self.WEB_PREFIX)
116
117         self.assertEquals(self.tool.web.urls_fetched,
118             [self.WEB_PREFIX + '/userscripts/another-test-actual.png',
119              self.WEB_PREFIX + '/userscripts/another-test-actual.wav',
120              self.WEB_PREFIX + '/userscripts/another-test-actual.txt'])
121         new_expectations = self._read(self.lion_expectations_path)
122         self.assertEqual(new_expectations, "Bug(x) [ MountainLion SnowLeopard ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n")
123
124     def test_rebaseline_does_not_include_overrides(self):
125         self._write(self.lion_expectations_path, "Bug(x) [ Mac ] userscripts/another-test.html [ ImageOnlyFailure ]\nBug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n")
126         self._write(self.lion_port.path_from_chromium_base('skia', 'skia_test_expectations.txt'), "Bug(y) [ Mac ] other-test.html [ Failure ]\n")
127         self._write("userscripts/another-test.html", "Dummy test contents")
128
129         self.command._rebaseline_test_and_update_expectations("WebKit Mac10.7", "userscripts/another-test.html", None, self.WEB_PREFIX)
130
131         self.assertEquals(self.tool.web.urls_fetched,
132             [self.WEB_PREFIX + '/userscripts/another-test-actual.png',
133              self.WEB_PREFIX + '/userscripts/another-test-actual.wav',
134              self.WEB_PREFIX + '/userscripts/another-test-actual.txt'])
135
136         new_expectations = self._read(self.lion_expectations_path)
137         self.assertEqual(new_expectations, "Bug(x) [ MountainLion SnowLeopard ] userscripts/another-test.html [ ImageOnlyFailure ]\nBug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n")
138
139     def test_rebaseline_test(self):
140         self.command._rebaseline_test("WebKit Linux", "userscripts/another-test.html", None, "txt", self.WEB_PREFIX)
141         self.assertEquals(self.tool.web.urls_fetched, [self.WEB_PREFIX + '/userscripts/another-test-actual.txt'])
142
143     def test_rebaseline_test_with_results_directory(self):
144         self.command._rebaseline_test("WebKit Linux", "userscripts/another-test.html", None, "txt", '/tmp')
145         self.assertEquals(self.tool.web.urls_fetched, ['/tmp/userscripts/another-test-actual.txt'])
146
147     def test_rebaseline_test_and_print_scm_changes(self):
148         self.command._print_scm_changes = True
149         self.command._scm_changes = {'add': [], 'delete': []}
150         self.tool._scm.exists = lambda x: False
151
152         self.command._rebaseline_test("WebKit Linux", "userscripts/another-test.html", None, "txt", None)
153
154         self.assertEquals(self.command._scm_changes, {'add': ['/mock-checkout/LayoutTests/platform/chromium-linux/userscripts/another-test-expected.txt'], 'delete': []})
155
156     def test_rebaseline_and_copy_test(self):
157         self._write("userscripts/another-test-expected.txt", "generic result")
158
159         self.command._rebaseline_test("WebKit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt", None)
160
161         self.assertEquals(self._read('platform/chromium-mac-lion/userscripts/another-test-expected.txt'), self.MOCK_WEB_RESULT)
162         self.assertEquals(self._read('platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt'), 'generic result')
163
164     def test_rebaseline_and_copy_test_no_existing_result(self):
165         self.command._rebaseline_test("WebKit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt", None)
166
167         self.assertEquals(self._read('platform/chromium-mac-lion/userscripts/another-test-expected.txt'), self.MOCK_WEB_RESULT)
168         self.assertFalse(self.tool.filesystem.exists(self._expand('platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt')))
169
170     def test_rebaseline_and_copy_test_with_lion_result(self):
171         self._write("platform/chromium-mac-lion/userscripts/another-test-expected.txt", "original lion result")
172
173         self.command._rebaseline_test("WebKit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt", self.WEB_PREFIX)
174
175         self.assertEquals(self.tool.web.urls_fetched, [self.WEB_PREFIX + '/userscripts/another-test-actual.txt'])
176         self.assertEquals(self._read("platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt"), "original lion result")
177         self.assertEquals(self._read("platform/chromium-mac-lion/userscripts/another-test-expected.txt"), self.MOCK_WEB_RESULT)
178
179     def test_rebaseline_and_copy_no_overwrite_test(self):
180         self._write("platform/chromium-mac-lion/userscripts/another-test-expected.txt", "original lion result")
181         self._write("platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt", "original snowleopard result")
182
183         self.command._rebaseline_test("WebKit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt", None)
184
185         self.assertEquals(self._read("platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt"), "original snowleopard result")
186         self.assertEquals(self._read("platform/chromium-mac-lion/userscripts/another-test-expected.txt"), self.MOCK_WEB_RESULT)
187
188     def test_rebaseline_test_internal_with_move_overwritten_baselines_to(self):
189         self.tool.executive = MockExecutive2()
190
191         # FIXME: it's confusing that this is the test- port, and not the regular lion port. Really all of the tests should be using the test ports.
192         port = self.tool.port_factory.get('test-mac-snowleopard')
193         self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-mac-snowleopard/failures/expected/image-expected.txt'), 'original snowleopard result')
194
195         old_exact_matches = builders._exact_matches
196         oc = OutputCapture()
197         try:
198             builders._exact_matches = {
199                 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
200                 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])},
201             }
202
203             options = MockOptions(optimize=True, builder="MOCK SnowLeopard", suffixes="txt",
204                 move_overwritten_baselines_to=["test-mac-leopard"], verbose=True, test="failures/expected/image.html",
205                 results_directory=None)
206
207             oc.capture_output()
208             self.command.execute(options, [], self.tool)
209         finally:
210             out, _, _ = oc.restore_output()
211             builders._exact_matches = old_exact_matches
212
213         self.assertEquals(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-mac-leopard/failures/expected/image-expected.txt')), 'original snowleopard result')
214         self.assertEquals(out, '{"add": []}\n')
215
216
217 class TestRebaselineJson(_BaseTestCase):
218     command_constructor = RebaselineJson
219
220     def setUp(self):
221         super(TestRebaselineJson, self).setUp()
222         self.tool.executive = MockExecutive2()
223         self.old_exact_matches = builders._exact_matches
224         builders._exact_matches = {
225             "MOCK builder": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"]),
226                              "move_overwritten_baselines_to": ["test-mac-leopard"]},
227             "MOCK builder (Debug)": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier", "debug"])},
228         }
229
230     def tearDown(self):
231         builders._exact_matches = self.old_exact_matches
232         super(TestRebaselineJson, self).tearDown()
233
234     def test_rebaseline_all(self):
235         options = MockOptions(optimize=True, verbose=True, move_overwritten_baselines=False, results_directory=None)
236         self.command._rebaseline(options,  {"user-scripts/another-test.html": {"MOCK builder": ["txt", "png"]}})
237
238         # Note that we have one run_in_parallel() call followed by a run_command()
239         self.assertEquals(self.tool.executive.calls,
240             [[['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'user-scripts/another-test.html', '--verbose']],
241              ['echo', '--verbose', 'optimize-baselines', '--suffixes', 'txt,png', 'user-scripts/another-test.html']])
242
243     def test_rebaseline_debug(self):
244         options = MockOptions(optimize=True, verbose=True, move_overwritten_baselines=False, results_directory=None)
245         self.command._rebaseline(options,  {"user-scripts/another-test.html": {"MOCK builder (Debug)": ["txt", "png"]}})
246
247         # Note that we have one run_in_parallel() call followed by a run_command()
248         self.assertEquals(self.tool.executive.calls,
249             [[['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'user-scripts/another-test.html', '--verbose']],
250              ['echo', '--verbose', 'optimize-baselines', '--suffixes', 'txt,png', 'user-scripts/another-test.html']])
251
252     def test_move_overwritten(self):
253         options = MockOptions(optimize=True, verbose=True, move_overwritten_baselines=True, results_directory=None)
254         self.command._rebaseline(options,  {"user-scripts/another-test.html": {"MOCK builder": ["txt", "png"]}})
255
256         # Note that we have one run_in_parallel() call followed by a run_command()
257         self.assertEquals(self.tool.executive.calls,
258             [[['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'user-scripts/another-test.html', '--move-overwritten-baselines-to', 'test-mac-leopard', '--verbose']],
259              ['echo', '--verbose', 'optimize-baselines', '--suffixes', 'txt,png', 'user-scripts/another-test.html']])
260
261     def test_no_optimize(self):
262         options = MockOptions(optimize=False, verbose=True, move_overwritten_baselines=False, results_directory=None)
263         self.command._rebaseline(options,  {"user-scripts/another-test.html": {"MOCK builder (Debug)": ["txt", "png"]}})
264
265         # Note that we have only one run_in_parallel() call
266         self.assertEquals(self.tool.executive.calls,
267             [[['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'user-scripts/another-test.html', '--verbose']]])
268
269     def test_results_directory(self):
270         options = MockOptions(optimize=False, verbose=True, move_overwritten_baselines=False, results_directory='/tmp')
271         self.command._rebaseline(options,  {"user-scripts/another-test.html": {"MOCK builder": ["txt", "png"]}})
272
273         # Note that we have only one run_in_parallel() call
274         self.assertEquals(self.tool.executive.calls,
275             [[['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'user-scripts/another-test.html', '--results_directory', '/tmp', '--verbose']]])
276
277
278 class TestRebaseline(_BaseTestCase):
279     # This command shares most of its logic with RebaselineJson, so these tests just test what is different.
280
281     command_constructor = Rebaseline  # AKA webkit-patch rebaseline
282
283     def test_tests_to_update(self):
284         build = Mock()
285         OutputCapture().assert_outputs(self, self.command._tests_to_update, [build])
286
287     def test_rebaseline(self):
288         self.command._builders_to_pull_from = lambda: [MockBuilder('MOCK builder')]
289         self.command._tests_to_update = lambda builder: ['mock/path/to/test.html']
290
291         self._zero_out_test_expectations()
292
293         old_exact_matches = builders._exact_matches
294         oc = OutputCapture()
295         try:
296             builders._exact_matches = {
297                 "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
298             }
299             oc.capture_output()
300             self.command.execute(MockOptions(optimize=False, builders=None, suffixes="txt,png", verbose=True, move_overwritten_baselines=False), [], self.tool)
301         finally:
302             oc.restore_output()
303             builders._exact_matches = old_exact_matches
304
305         calls = filter(lambda x: x != ['qmake', '-v'] and x[0] != 'perl', self.tool.executive.calls)
306         self.assertEquals(calls,
307             [[['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'mock/path/to/test.html', '--verbose']]])
308
309
310 class TestRebaselineExpectations(_BaseTestCase):
311     command_constructor = RebaselineExpectations
312
313     def setUp(self):
314         super(TestRebaselineExpectations, self).setUp()
315         self.options = MockOptions(optimize=False, builders=None, suffixes=['txt'], verbose=False, platform=None,
316                                    move_overwritten_baselines=False, results_directory=None)
317
318     def test_rebaseline_expectations(self):
319         self._zero_out_test_expectations()
320
321         self.tool.executive = MockExecutive2()
322
323         self.command._tests_to_rebaseline = lambda port: {'userscripts/another-test.html': set(['txt']), 'userscripts/images.svg': set(['png'])}
324         self.command.execute(self.options, [], self.tool)
325
326         # FIXME: change this to use the test- ports.
327         calls = filter(lambda x: x != ['qmake', '-v'], self.tool.executive.calls)
328         self.assertTrue(len(calls) == 1)
329         self.assertTrue(len(calls[0]) == 26)
330
331     def test_rebaseline_expectations_noop(self):
332         self._zero_out_test_expectations()
333
334         oc = OutputCapture()
335         try:
336             oc.capture_output()
337             self.command.execute(self.options, [], self.tool)
338         finally:
339             _, _, logs = oc.restore_output()
340             self.assertEquals(self.tool.filesystem.written_files, {})
341             self.assertEquals(logs, 'Did not find any tests marked Rebaseline.\n')
342
343     def disabled_test_overrides_are_included_correctly(self):
344         # This tests that the any tests marked as REBASELINE in the overrides are found, but
345         # that the overrides do not get written into the main file.
346         self._zero_out_test_expectations()
347
348         self._write(self.lion_expectations_path, '')
349         self.lion_port.expectations_dict = lambda: {
350             self.lion_expectations_path: '',
351             'overrides': ('Bug(x) userscripts/another-test.html [ Failure Rebaseline ]\n'
352                           'Bug(y) userscripts/test.html [ Crash ]\n')}
353         self._write('/userscripts/another-test.html', '')
354
355         self.assertEquals(self.command._tests_to_rebaseline(self.lion_port), {'userscripts/another-test.html': set(['png', 'txt', 'wav'])})
356         self.assertEquals(self._read(self.lion_expectations_path), '')
357
358
359 class _FakeOptimizer(BaselineOptimizer):
360     def read_results_by_directory(self, baseline_name):
361         if baseline_name.endswith('txt'):
362             return {'LayoutTests/passes/text.html': '123456',
363                     'LayoutTests/platform/test-mac-leopard/passes/text.html': 'abcdef'}
364         return {}
365
366
367 class TestAnalyzeBaselines(_BaseTestCase):
368     command_constructor = AnalyzeBaselines
369
370     def setUp(self):
371         super(TestAnalyzeBaselines, self).setUp()
372         self.port = self.tool.port_factory.get('test')
373         self.tool.port_factory.get = lambda port_name=None, options=None: self.port
374         self.lines = []
375         self.command._optimizer_class = _FakeOptimizer
376         self.command._write = lambda msg: self.lines.append(msg)
377
378     def test_default(self):
379         self.command.execute(MockOptions(suffixes='txt', missing=False), ['passes/text.html'], self.tool)
380         self.assertEquals(self.lines,
381             ['passes/text-expected.txt:',
382              '  (generic): 123456',
383              '  test-mac-leopard: abcdef'])
384
385     def test_missing_baselines(self):
386         self.command.execute(MockOptions(suffixes='png,txt', missing=True), ['passes/text.html'], self.tool)
387         self.assertEquals(self.lines,
388             ['passes/text-expected.png: (no baselines found)',
389              'passes/text-expected.txt:',
390              '  (generic): 123456',
391              '  test-mac-leopard: abcdef'])