Rename WebKitTools to Tools
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / rebaselineserver_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #    * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #    * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #    * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import unittest
30
31 from webkitpy.common.system import filesystem_mock
32 from webkitpy.layout_tests.port import base
33 from webkitpy.layout_tests.port.webkit import WebKitPort
34 from webkitpy.tool.commands import rebaselineserver
35 from webkitpy.tool.mocktool import MockSCM
36
37
38 class RebaselineTestTest(unittest.TestCase):
39     def test_text_rebaseline_update(self):
40         self._assertRebaseline(
41             test_files=(
42                 'fast/text-expected.txt',
43                 'platform/mac/fast/text-expected.txt',
44             ),
45             results_files=(
46                 'fast/text-actual.txt',
47             ),
48             test_name='fast/text.html',
49             baseline_target='mac',
50             baseline_move_to='none',
51             expected_success=True,
52             expected_log=[
53                 'Rebaselining fast/text...',
54                 '  Updating baselines for mac',
55                 '    Updated text-expected.txt',
56             ])
57
58     def test_text_rebaseline_new(self):
59         self._assertRebaseline(
60             test_files=(
61                 'fast/text-expected.txt',
62             ),
63             results_files=(
64                 'fast/text-actual.txt',
65             ),
66             test_name='fast/text.html',
67             baseline_target='mac',
68             baseline_move_to='none',
69             expected_success=True,
70             expected_log=[
71                 'Rebaselining fast/text...',
72                 '  Updating baselines for mac',
73                 '    Updated text-expected.txt',
74             ])
75
76     def test_text_rebaseline_move_no_op_1(self):
77         self._assertRebaseline(
78             test_files=(
79                 'fast/text-expected.txt',
80                 'platform/win/fast/text-expected.txt',
81             ),
82             results_files=(
83                 'fast/text-actual.txt',
84             ),
85             test_name='fast/text.html',
86             baseline_target='mac',
87             baseline_move_to='mac-leopard',
88             expected_success=True,
89             expected_log=[
90                 'Rebaselining fast/text...',
91                 '  Updating baselines for mac',
92                 '    Updated text-expected.txt',
93             ])
94
95     def test_text_rebaseline_move_no_op_2(self):
96         self._assertRebaseline(
97             test_files=(
98                 'fast/text-expected.txt',
99                 'platform/mac/fast/text-expected.checksum',
100             ),
101             results_files=(
102                 'fast/text-actual.txt',
103             ),
104             test_name='fast/text.html',
105             baseline_target='mac',
106             baseline_move_to='mac-leopard',
107             expected_success=True,
108             expected_log=[
109                 'Rebaselining fast/text...',
110                 '  Moving current mac baselines to mac-leopard',
111                 '    No current baselines to move',
112                 '  Updating baselines for mac',
113                 '    Updated text-expected.txt',
114             ])
115
116     def test_text_rebaseline_move(self):
117         self._assertRebaseline(
118             test_files=(
119                 'fast/text-expected.txt',
120                 'platform/mac/fast/text-expected.txt',
121             ),
122             results_files=(
123                 'fast/text-actual.txt',
124             ),
125             test_name='fast/text.html',
126             baseline_target='mac',
127             baseline_move_to='mac-leopard',
128             expected_success=True,
129             expected_log=[
130                 'Rebaselining fast/text...',
131                 '  Moving current mac baselines to mac-leopard',
132                 '    Moved text-expected.txt',
133                 '  Updating baselines for mac',
134                 '    Updated text-expected.txt',
135             ])
136
137     def test_text_rebaseline_move_only_images(self):
138         self._assertRebaseline(
139             test_files=(
140                 'fast/image-expected.txt',
141                 'platform/mac/fast/image-expected.txt',
142                 'platform/mac/fast/image-expected.png',
143                 'platform/mac/fast/image-expected.checksum',
144             ),
145             results_files=(
146                 'fast/image-actual.png',
147                 'fast/image-actual.checksum',
148             ),
149             test_name='fast/image.html',
150             baseline_target='mac',
151             baseline_move_to='mac-leopard',
152             expected_success=True,
153             expected_log=[
154                 'Rebaselining fast/image...',
155                 '  Moving current mac baselines to mac-leopard',
156                 '    Moved image-expected.checksum',
157                 '    Moved image-expected.png',
158                 '  Updating baselines for mac',
159                 '    Updated image-expected.checksum',
160                 '    Updated image-expected.png',
161             ])
162
163     def test_text_rebaseline_move_already_exist(self):
164         self._assertRebaseline(
165             test_files=(
166                 'fast/text-expected.txt',
167                 'platform/mac-leopard/fast/text-expected.txt',
168                 'platform/mac/fast/text-expected.txt',
169             ),
170             results_files=(
171                 'fast/text-actual.txt',
172             ),
173             test_name='fast/text.html',
174             baseline_target='mac',
175             baseline_move_to='mac-leopard',
176             expected_success=False,
177             expected_log=[
178                 'Rebaselining fast/text...',
179                 '  Moving current mac baselines to mac-leopard',
180                 '    Already had baselines in mac-leopard, could not move existing mac ones',
181             ])
182
183     def test_image_rebaseline(self):
184         self._assertRebaseline(
185             test_files=(
186                 'fast/image-expected.txt',
187                 'platform/mac/fast/image-expected.png',
188                 'platform/mac/fast/image-expected.checksum',
189             ),
190             results_files=(
191                 'fast/image-actual.png',
192                 'fast/image-actual.checksum',
193             ),
194             test_name='fast/image.html',
195             baseline_target='mac',
196             baseline_move_to='none',
197             expected_success=True,
198             expected_log=[
199                 'Rebaselining fast/image...',
200                 '  Updating baselines for mac',
201                 '    Updated image-expected.checksum',
202                 '    Updated image-expected.png',
203             ])
204
205     def _assertRebaseline(self, test_files, results_files, test_name, baseline_target, baseline_move_to, expected_success, expected_log):
206         log = []
207         test_config = get_test_config(test_files, results_files)
208         success = rebaselineserver._rebaseline_test(
209             test_name,
210             baseline_target,
211             baseline_move_to,
212             test_config,
213             log=lambda l: log.append(l))
214         self.assertEqual(expected_log, log)
215         self.assertEqual(expected_success, success)
216
217
218 class GetActualResultFilesTest(unittest.TestCase):
219     def test(self):
220         test_config = get_test_config(result_files=(
221             'fast/text-actual.txt',
222             'fast2/text-actual.txt',
223             'fast/text2-actual.txt',
224             'fast/text-notactual.txt',
225         ))
226         self.assertEqual(
227             ('text-actual.txt',),
228             rebaselineserver._get_actual_result_files(
229                 'fast/text.html', test_config))
230
231
232 class GetBaselinesTest(unittest.TestCase):
233     def test_no_baselines(self):
234         self._assertBaselines(
235             test_files=(),
236             test_name='fast/missing.html',
237             expected_baselines={})
238
239     def test_text_baselines(self):
240         self._assertBaselines(
241             test_files=(
242                 'fast/text-expected.txt',
243                 'platform/mac/fast/text-expected.txt',
244             ),
245             test_name='fast/text.html',
246             expected_baselines={
247                 'mac': {'.txt': True},
248                 'base': {'.txt': False},
249             })
250
251     def test_image_and_text_baselines(self):
252         self._assertBaselines(
253             test_files=(
254                 'fast/image-expected.txt',
255                 'platform/mac/fast/image-expected.png',
256                 'platform/mac/fast/image-expected.checksum',
257                 'platform/win/fast/image-expected.png',
258                 'platform/win/fast/image-expected.checksum',
259             ),
260             test_name='fast/image.html',
261             expected_baselines={
262                 'base': {'.txt': True},
263                 'mac': {'.checksum': True, '.png': True},
264                 'win': {'.checksum': False, '.png': False},
265             })
266
267     def test_extra_baselines(self):
268         self._assertBaselines(
269             test_files=(
270                 'fast/text-expected.txt',
271                 'platform/nosuchplatform/fast/text-expected.txt',
272             ),
273             test_name='fast/text.html',
274             expected_baselines={'base': {'.txt': True}})
275
276     def _assertBaselines(self, test_files, test_name, expected_baselines):
277         actual_baselines = rebaselineserver._get_test_baselines(
278             test_name, get_test_config(test_files))
279         self.assertEqual(expected_baselines, actual_baselines)
280
281
282 def get_test_config(test_files=[], result_files=[]):
283     layout_tests_directory = base.Port().layout_tests_dir()
284     results_directory = '/WebKitBuild/Debug/layout-test-results'
285     mock_filesystem = filesystem_mock.MockFileSystem()
286     for file in test_files:
287         file_path = mock_filesystem.join(layout_tests_directory, file)
288         mock_filesystem.files[file_path] = ''
289     for file in result_files:
290         file_path = mock_filesystem.join(results_directory, file)
291         mock_filesystem.files[file_path] = ''
292
293     class TestMacPort(WebKitPort):
294         def __init__(self):
295             WebKitPort.__init__(self, filesystem=mock_filesystem)
296             self._name = 'mac'
297
298     return rebaselineserver.TestConfig(
299         TestMacPort(),
300         layout_tests_directory,
301         results_directory,
302         ('mac', 'mac-leopard', 'win', 'linux'),
303         mock_filesystem,
304         MockSCM())