138c1ef9b00620a16f9b6ddf3847a93775c55496
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / queries_unittest.py
1 # Copyright (C) 2009 Google Inc. All rights reserved.
2 # Copyright (C) 2012 Intel Corporation. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #    * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #    * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #    * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import unittest
31
32 from webkitpy.common.system.outputcapture import OutputCapture
33 from webkitpy.common.net.bugzilla import Bugzilla
34 from webkitpy.common.system.outputcapture import OutputCapture
35 from webkitpy.thirdparty.mock import Mock
36 from webkitpy.port.test import TestPort
37 from webkitpy.tool.commands.commandtest import CommandsTest
38 from webkitpy.tool.commands.queries import *
39 from webkitpy.tool.mocktool import MockTool, MockOptions
40
41
42 class MockTestPort1(object):
43     def skips_layout_test(self, test_name):
44         return test_name in ["media/foo/bar.html", "foo"]
45
46
47 class MockTestPort2(object):
48     def skips_layout_test(self, test_name):
49         return test_name == "media/foo/bar.html"
50
51
52 class MockPortFactory(object):
53     def __init__(self):
54         self._all_ports = {
55             "test_port1": MockTestPort1(),
56             "test_port2": MockTestPort2(),
57         }
58
59     def all_port_names(self, options=None):
60         return self._all_ports.keys()
61
62     def get(self, port_name):
63         return self._all_ports.get(port_name)
64
65
66 class QueryCommandsTest(CommandsTest):
67     def test_bugs_to_commit(self):
68         expected_logs = "Warning, attachment 10001 on bug 50000 has invalid committer (non-committer@example.com)\n"
69         self.assert_execute_outputs(BugsToCommit(), None, "50000\n50003\n", expected_logs=expected_logs)
70
71     def test_patches_in_commit_queue(self):
72         expected_stdout = "http://example.com/10000\nhttp://example.com/10002\n"
73         expected_logs = "Warning, attachment 10001 on bug 50000 has invalid committer (non-committer@example.com)\nPatches in commit queue:\n"
74         self.assert_execute_outputs(PatchesInCommitQueue(), None, expected_stdout, expected_logs=expected_logs)
75
76     def test_patches_to_commit_queue(self):
77         expected_stdout = "http://example.com/10003&action=edit\n"
78         expected_logs = "10000 already has cq=+\n10001 already has cq=+\n10004 committer = \"Eric Seidel\" <eric@webkit.org>\n"
79         options = Mock()
80         options.bugs = False
81         self.assert_execute_outputs(PatchesToCommitQueue(), None, expected_stdout, expected_logs=expected_logs, options=options)
82
83         expected_stdout = "http://example.com/50003\n"
84         options.bugs = True
85         self.assert_execute_outputs(PatchesToCommitQueue(), None, expected_stdout, expected_logs=expected_logs, options=options)
86
87     def test_patches_to_review(self):
88         options = Mock()
89
90         # When no cc_email is provided, we use the Bugzilla username by default.
91         # The MockBugzilla will fake the authentication using username@webkit.org
92         # as login and it should match the username at the report header.
93         options.cc_email = None
94         options.include_cq_denied = False
95         options.all = False
96         expected_stdout = \
97             "Bugs with attachments pending review that has username@webkit.org in the CC list:\n" \
98             "http://webkit.org/b/bugid   Description (age in days)\n" \
99             "Total: 0\n"
100         expected_stderr = ""
101         self.assert_execute_outputs(PatchesToReview(), None, expected_stdout, expected_stderr, options=options)
102
103         options.cc_email = "abarth@webkit.org"
104         options.include_cq_denied = True
105         options.all = False
106         expected_stdout = \
107             "Bugs with attachments pending review that has abarth@webkit.org in the CC list:\n" \
108             "http://webkit.org/b/bugid   Description (age in days)\n" \
109             "http://webkit.org/b/50001   Bug with a patch needing review. (0)\n" \
110             "Total: 1\n"
111         expected_stderr = ""
112         self.assert_execute_outputs(PatchesToReview(), None, expected_stdout, expected_stderr, options=options)
113
114         options.cc_email = None
115         options.include_cq_denied = True
116         options.all = True
117         expected_stdout = \
118             "Bugs with attachments pending review:\n" \
119             "http://webkit.org/b/bugid   Description (age in days)\n" \
120             "http://webkit.org/b/50001   Bug with a patch needing review. (0)\n" \
121             "Total: 1\n"
122         self.assert_execute_outputs(PatchesToReview(), None, expected_stdout, expected_stderr, options=options)
123
124         options.cc_email = None
125         options.include_cq_denied = False
126         options.all = True
127         expected_stdout = \
128             "Bugs with attachments pending review:\n" \
129             "http://webkit.org/b/bugid   Description (age in days)\n" \
130             "Total: 0\n"
131         self.assert_execute_outputs(PatchesToReview(), None, expected_stdout, expected_stderr, options=options)
132
133         options.cc_email = "invalid_email@example.com"
134         options.all = False
135         options.include_cq_denied = True
136         expected_stdout = \
137             "Bugs with attachments pending review that has invalid_email@example.com in the CC list:\n" \
138             "http://webkit.org/b/bugid   Description (age in days)\n" \
139             "Total: 0\n"
140         self.assert_execute_outputs(PatchesToReview(), None, expected_stdout, expected_stderr, options=options)
141
142     def test_tree_status(self):
143         expected_stdout = "ok   : Builder1\nok   : Builder2\n"
144         self.assert_execute_outputs(TreeStatus(), None, expected_stdout)
145
146
147 class FailureReasonTest(unittest.TestCase):
148     def test_blame_line_for_revision(self):
149         tool = MockTool()
150         command = FailureReason()
151         command.bind_to_tool(tool)
152         # This is an artificial example, mostly to test the CommitInfo lookup failure case.
153         self.assertEqual(command._blame_line_for_revision(0), "FAILED to fetch CommitInfo for r0, likely missing ChangeLog")
154
155         def raising_mock(self):
156             raise Exception("MESSAGE")
157         tool.checkout().commit_info_for_revision = raising_mock
158         self.assertEqual(command._blame_line_for_revision(0), "FAILED to fetch CommitInfo for r0, exception: MESSAGE")
159
160
161 class PrintExpectationsTest(unittest.TestCase):
162     def run_test(self, tests, expected_stdout, platform='test-win-xp', **args):
163         options = MockOptions(all=False, csv=False, full=False, platform=platform,
164                               include_keyword=[], exclude_keyword=[], paths=False).update(**args)
165         tool = MockTool()
166         tool.port_factory.all_port_names = lambda: TestPort.ALL_BASELINE_VARIANTS
167         command = PrintExpectations()
168         command.bind_to_tool(tool)
169
170         oc = OutputCapture()
171         try:
172             oc.capture_output()
173             command.execute(options, tests, tool)
174         finally:
175             stdout, _, _ = oc.restore_output()
176         self.assertMultiLineEqual(stdout, expected_stdout)
177
178     def test_basic(self):
179         self.run_test(['failures/expected/text.html', 'failures/expected/image.html'],
180                       ('// For test-win-xp\n'
181                        'failures/expected/image.html [ ImageOnlyFailure ]\n'
182                        'failures/expected/text.html [ Failure ]\n'))
183
184     def test_multiple(self):
185         self.run_test(['failures/expected/text.html', 'failures/expected/image.html'],
186                       ('// For test-win-vista\n'
187                        'failures/expected/image.html [ ImageOnlyFailure ]\n'
188                        'failures/expected/text.html [ Failure ]\n'
189                        '\n'
190                        '// For test-win-win7\n'
191                        'failures/expected/image.html [ ImageOnlyFailure ]\n'
192                        'failures/expected/text.html [ Failure ]\n'
193                        '\n'
194                        '// For test-win-xp\n'
195                        'failures/expected/image.html [ ImageOnlyFailure ]\n'
196                        'failures/expected/text.html [ Failure ]\n'),
197                        platform='test-win-*')
198
199     def test_full(self):
200         self.run_test(['failures/expected/text.html', 'failures/expected/image.html'],
201                       ('// For test-win-xp\n'
202                        'Bug(test) failures/expected/image.html [ ImageOnlyFailure ]\n'
203                        'Bug(test) failures/expected/text.html [ Failure ]\n'),
204                       full=True)
205
206     def test_exclude(self):
207         self.run_test(['failures/expected/text.html', 'failures/expected/image.html'],
208                       ('// For test-win-xp\n'
209                        'failures/expected/text.html [ Failure ]\n'),
210                       exclude_keyword=['image'])
211
212     def test_include(self):
213         self.run_test(['failures/expected/text.html', 'failures/expected/image.html'],
214                       ('// For test-win-xp\n'
215                        'failures/expected/image.html\n'),
216                       include_keyword=['image'])
217
218     def test_csv(self):
219         self.run_test(['failures/expected/text.html', 'failures/expected/image.html'],
220                       ('test-win-xp,failures/expected/image.html,BUGTEST,IMAGE\n'
221                        'test-win-xp,failures/expected/text.html,BUGTEST,FAIL\n'),
222                       csv=True)
223
224     def test_paths(self):
225         self.run_test([],
226                       ('LayoutTests/TestExpectations\n'
227                        'LayoutTests/platform/test/TestExpectations\n'
228                        'LayoutTests/platform/test-win-xp/TestExpectations\n'),
229                       paths=True)
230
231     def test_platform(self):
232         self.run_test(['platform/test-mac-leopard/http/test.html'],
233                       ('// For test-mac-snowleopard\n'
234                        'platform/test-mac-leopard [ Pass Skip WontFix ]\n'  # Note that this is the expectation (from being skipped internally), not the test name
235                        '\n'
236                        '// For test-mac-leopard\n'
237                        'platform/test-mac-leopard/http/test.html [ Pass ]\n'),
238                       platform='test-mac-*')
239
240 class PrintBaselinesTest(unittest.TestCase):
241     def setUp(self):
242         self.oc = None
243         self.tool = MockTool()
244         self.test_port = self.tool.port_factory.get('test-win-xp')
245         self.tool.port_factory.get = lambda port_name=None: self.test_port
246         self.tool.port_factory.all_port_names = lambda: TestPort.ALL_BASELINE_VARIANTS
247
248     def tearDown(self):
249         if self.oc:
250             self.restore_output()
251
252     def capture_output(self):
253         self.oc = OutputCapture()
254         self.oc.capture_output()
255
256     def restore_output(self):
257         stdout, stderr, logs = self.oc.restore_output()
258         self.oc = None
259         return (stdout, stderr, logs)
260
261     def test_basic(self):
262         command = PrintBaselines()
263         command.bind_to_tool(self.tool)
264         self.capture_output()
265         command.execute(MockOptions(all=False, include_virtual_tests=False, csv=False, platform=None), ['passes/text.html'], self.tool)
266         stdout, _, _ = self.restore_output()
267         self.assertMultiLineEqual(stdout,
268                           ('// For test-win-xp\n'
269                            'passes/text-expected.png\n'
270                            'passes/text-expected.txt\n'))
271
272     def test_multiple(self):
273         command = PrintBaselines()
274         command.bind_to_tool(self.tool)
275         self.capture_output()
276         command.execute(MockOptions(all=False, include_virtual_tests=False, csv=False, platform='test-win-*'), ['passes/text.html'], self.tool)
277         stdout, _, _ = self.restore_output()
278         self.assertMultiLineEqual(stdout,
279                           ('// For test-win-vista\n'
280                            'passes/text-expected.png\n'
281                            'passes/text-expected.txt\n'
282                            '\n'
283                            '// For test-win-win7\n'
284                            'passes/text-expected.png\n'
285                            'passes/text-expected.txt\n'
286                            '\n'
287                            '// For test-win-xp\n'
288                            'passes/text-expected.png\n'
289                            'passes/text-expected.txt\n'))
290
291     def test_csv(self):
292         command = PrintBaselines()
293         command.bind_to_tool(self.tool)
294         self.capture_output()
295         command.execute(MockOptions(all=False, platform='*xp', csv=True, include_virtual_tests=False), ['passes/text.html'], self.tool)
296         stdout, _, _ = self.restore_output()
297         self.assertMultiLineEqual(stdout,
298                           ('test-win-xp,passes/text.html,None,png,passes/text-expected.png,None\n'
299                            'test-win-xp,passes/text.html,None,txt,passes/text-expected.txt,None\n'))