Disable running tests on the MAC WK2 EWS queue.
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / earlywarningsystem.py
1 # Copyright (c) 2009 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import logging
30 from optparse import make_option
31
32 from webkitpy.common.config.committers import CommitterList
33 from webkitpy.common.config.ports import DeprecatedPort
34 from webkitpy.common.system.executive import ScriptError
35 from webkitpy.tool.bot.earlywarningsystemtask import EarlyWarningSystemTask, EarlyWarningSystemTaskDelegate
36 from webkitpy.tool.bot.expectedfailures import ExpectedFailures
37 from webkitpy.tool.bot.layouttestresultsreader import LayoutTestResultsReader
38 from webkitpy.tool.bot.patchanalysistask import UnableToApplyPatch
39 from webkitpy.tool.bot.queueengine import QueueEngine
40 from webkitpy.tool.commands.queues import AbstractReviewQueue
41
42 _log = logging.getLogger(__name__)
43
44
45 class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDelegate):
46     _build_style = "release"
47     # FIXME: Switch _default_run_tests from opt-in to opt-out once more bots are ready to run tests.
48     _default_run_tests = False
49
50     def __init__(self):
51         options = [make_option("--run-tests", action="store_true", dest="run_tests", default=self._default_run_tests, help="Run the Layout tests for each patch")]
52         AbstractReviewQueue.__init__(self, options=options)
53
54     def begin_work_queue(self):
55         AbstractReviewQueue.begin_work_queue(self)
56         self._expected_failures = ExpectedFailures()
57         self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._port.results_directory(), self._log_directory())
58
59     def _failing_tests_message(self, task, patch):
60         results = task.results_from_patch_test_run(patch)
61         unexpected_failures = self._expected_failures.unexpected_failures_observed(results)
62         if not unexpected_failures:
63             return None
64         return "New failing tests:\n%s" % "\n".join(unexpected_failures)
65
66     def _post_reject_message_on_bug(self, tool, patch, status_id, extra_message_text=None):
67         results_link = tool.status_server.results_url_for_status(status_id)
68         message = "Attachment %s did not pass %s (%s):\nOutput: %s" % (patch.id(), self.name, self.port_name, results_link)
69         if extra_message_text:
70             message += "\n\n%s" % extra_message_text
71         # FIXME: We might want to add some text about rejecting from the commit-queue in
72         # the case where patch.commit_queue() isn't already set to '-'.
73         if self.watchers:
74             tool.bugs.add_cc_to_bug(patch.bug_id(), self.watchers)
75         tool.bugs.set_flag_on_attachment(patch.id(), "commit-queue", "-", message)
76
77     def review_patch(self, patch):
78         task = EarlyWarningSystemTask(self, patch, self._options.run_tests)
79         if not task.validate():
80             self._did_error(patch, "%s did not process patch." % self.name)
81             return False
82         try:
83             return task.run()
84         except UnableToApplyPatch, e:
85             self._did_error(patch, "%s unable to apply patch." % self.name)
86             return False
87         except ScriptError, e:
88             self._post_reject_message_on_bug(self._tool, patch, task.failure_status_id, self._failing_tests_message(task, patch))
89             results_archive = task.results_archive_from_patch_test_run(patch)
90             if results_archive:
91                 self._upload_results_archive_for_patch(patch, results_archive)
92             self._did_fail(patch)
93             # FIXME: We're supposed to be able to raise e again here and have
94             # one of our base classes mark the patch as fail, but there seems
95             # to be an issue with the exit_code.
96             return False
97
98     # EarlyWarningSystemDelegate methods
99
100     def parent_command(self):
101         return self.name
102
103     def run_command(self, command):
104         self.run_webkit_patch(command + [self._deprecated_port.flag()])
105
106     def command_passed(self, message, patch):
107         pass
108
109     def command_failed(self, message, script_error, patch):
110         failure_log = self._log_from_script_error_for_upload(script_error)
111         return self._update_status(message, patch=patch, results_file=failure_log)
112
113     def expected_failures(self):
114         return self._expected_failures
115
116     def test_results(self):
117         return self._layout_test_results_reader.results()
118
119     def archive_last_test_results(self, patch):
120         return self._layout_test_results_reader.archive(patch)
121
122     def build_style(self):
123         return self._build_style
124
125     def refetch_patch(self, patch):
126         return self._tool.bugs.fetch_attachment(patch.id())
127
128     def report_flaky_tests(self, patch, flaky_test_results, results_archive):
129         pass
130
131     # StepSequenceErrorHandler methods
132
133     @classmethod
134     def handle_script_error(cls, tool, state, script_error):
135         # FIXME: Why does this not exit(1) like the superclass does?
136         _log.error(script_error.message_with_output())
137
138
139 class GtkEWS(AbstractEarlyWarningSystem):
140     name = "gtk-ews"
141     port_name = "gtk"
142     watchers = AbstractEarlyWarningSystem.watchers + [
143         "xan.lopez@gmail.com",
144     ]
145
146
147 class EflEWS(AbstractEarlyWarningSystem):
148     name = "efl-ews"
149     port_name = "efl"
150     watchers = AbstractEarlyWarningSystem.watchers + [
151         "leandro@profusion.mobi",
152         "antognolli@profusion.mobi",
153         "lucas.demarchi@profusion.mobi",
154         "gyuyoung.kim@samsung.com",
155     ]
156
157
158 class QtEWS(AbstractEarlyWarningSystem):
159     name = "qt-ews"
160     port_name = "qt"
161     watchers = AbstractEarlyWarningSystem.watchers + [
162         "webkit-ews@sed.inf.u-szeged.hu",
163     ]
164
165
166 class QtWK2EWS(AbstractEarlyWarningSystem):
167     name = "qt-wk2-ews"
168     port_name = "qt"
169     watchers = AbstractEarlyWarningSystem.watchers + [
170         "webkit-ews@sed.inf.u-szeged.hu",
171     ]
172
173
174 class WinEWS(AbstractEarlyWarningSystem):
175     name = "win-ews"
176     port_name = "win"
177     _default_run_tests = True
178
179 class AbstractChromiumEWS(AbstractEarlyWarningSystem):
180     port_name = "chromium"
181     watchers = AbstractEarlyWarningSystem.watchers + [
182         "dglazkov@chromium.org",
183     ]
184
185
186 class ChromiumLinuxEWS(AbstractChromiumEWS):
187     # FIXME: We should rename this command to cr-linux-ews, but that requires
188     #        a database migration. :(
189     name = "chromium-ews"
190     port_name = "chromium-xvfb"
191     _default_run_tests = True
192
193
194 class ChromiumLinuxDebugEWS(AbstractChromiumEWS):
195     name = "cr-linux-debug-ews"
196     port_name = "chromium-xvfb"
197     _build_style = "debug"
198
199
200 class ChromiumWindowsEWS(AbstractChromiumEWS):
201     name = "cr-win-ews"
202
203
204 class ChromiumAndroidEWS(AbstractChromiumEWS):
205     name = "cr-android-ews"
206     port_name = "chromium-android"
207     watchers = AbstractChromiumEWS.watchers + [
208         "peter+ews@chromium.org",
209     ]
210
211
212 class MacEWS(AbstractEarlyWarningSystem):
213     name = "mac-ews"
214     port_name = "mac"
215     watchers = AbstractEarlyWarningSystem.watchers + [
216         "rniwa@webkit.org",
217     ]
218
219
220 class MacWK2EWS(AbstractEarlyWarningSystem):
221     name = "mac-wk2-ews"
222     port_name = "mac-wk2"
223     watchers = AbstractEarlyWarningSystem.watchers + [
224         "rniwa@webkit.org",
225     ]