Isolate Early Warning System definitions into a JSON
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / earlywarningsystem.py
1 # Copyright (c) 2009 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import json
30 import logging
31 from optparse import make_option
32
33 from webkitpy.common.config.committers import CommitterList
34 from webkitpy.common.config.ports import DeprecatedPort
35 from webkitpy.common.system.filesystem import FileSystem
36 from webkitpy.common.system.executive import ScriptError
37 from webkitpy.tool.bot.earlywarningsystemtask import EarlyWarningSystemTask, EarlyWarningSystemTaskDelegate
38 from webkitpy.tool.bot.expectedfailures import ExpectedFailures
39 from webkitpy.tool.bot.layouttestresultsreader import LayoutTestResultsReader
40 from webkitpy.tool.bot.patchanalysistask import UnableToApplyPatch
41 from webkitpy.tool.bot.queueengine import QueueEngine
42 from webkitpy.tool.commands.queues import AbstractReviewQueue
43
44 _log = logging.getLogger(__name__)
45
46
47 class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDelegate):
48     _build_style = "release"
49     # FIXME: Switch _default_run_tests from opt-in to opt-out once more bots are ready to run tests.
50     run_tests = False
51
52     def __init__(self):
53         options = [make_option("--run-tests", action="store_true", dest="run_tests", default=self.run_tests, help="Run the Layout tests for each patch")]
54         AbstractReviewQueue.__init__(self, options=options)
55
56     def begin_work_queue(self):
57         AbstractReviewQueue.begin_work_queue(self)
58         self._expected_failures = ExpectedFailures()
59         self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._port.results_directory(), self._log_directory())
60
61     def _failing_tests_message(self, task, patch):
62         results = task.results_from_patch_test_run(patch)
63         unexpected_failures = self._expected_failures.unexpected_failures_observed(results)
64         if not unexpected_failures:
65             return None
66         return "New failing tests:\n%s" % "\n".join(unexpected_failures)
67
68     def _post_reject_message_on_bug(self, tool, patch, status_id, extra_message_text=None):
69         results_link = tool.status_server.results_url_for_status(status_id)
70         message = "Attachment %s did not pass %s (%s):\nOutput: %s" % (patch.id(), self.name, self.port_name, results_link)
71         if extra_message_text:
72             message += "\n\n%s" % extra_message_text
73         # FIXME: We might want to add some text about rejecting from the commit-queue in
74         # the case where patch.commit_queue() isn't already set to '-'.
75         if self.watchers:
76             tool.bugs.add_cc_to_bug(patch.bug_id(), self.watchers)
77         tool.bugs.set_flag_on_attachment(patch.id(), "commit-queue", "-", message)
78
79     def review_patch(self, patch):
80         task = EarlyWarningSystemTask(self, patch, self._options.run_tests)
81         if not task.validate():
82             self._did_error(patch, "%s did not process patch." % self.name)
83             return False
84         try:
85             return task.run()
86         except UnableToApplyPatch, e:
87             self._did_error(patch, "%s unable to apply patch." % self.name)
88             return False
89         except ScriptError, e:
90             self._post_reject_message_on_bug(self._tool, patch, task.failure_status_id, self._failing_tests_message(task, patch))
91             results_archive = task.results_archive_from_patch_test_run(patch)
92             if results_archive:
93                 self._upload_results_archive_for_patch(patch, results_archive)
94             self._did_fail(patch)
95             # FIXME: We're supposed to be able to raise e again here and have
96             # one of our base classes mark the patch as fail, but there seems
97             # to be an issue with the exit_code.
98             return False
99
100     # EarlyWarningSystemDelegate methods
101
102     def parent_command(self):
103         return self.name
104
105     def run_command(self, command):
106         self.run_webkit_patch(command + [self._deprecated_port.flag()])
107
108     def command_passed(self, message, patch):
109         pass
110
111     def command_failed(self, message, script_error, patch):
112         failure_log = self._log_from_script_error_for_upload(script_error)
113         return self._update_status(message, patch=patch, results_file=failure_log)
114
115     def expected_failures(self):
116         return self._expected_failures
117
118     def test_results(self):
119         return self._layout_test_results_reader.results()
120
121     def archive_last_test_results(self, patch):
122         return self._layout_test_results_reader.archive(patch)
123
124     def build_style(self):
125         return self._build_style
126
127     def refetch_patch(self, patch):
128         return self._tool.bugs.fetch_attachment(patch.id())
129
130     def report_flaky_tests(self, patch, flaky_test_results, results_archive):
131         pass
132
133     # StepSequenceErrorHandler methods
134
135     @classmethod
136     def handle_script_error(cls, tool, state, script_error):
137         # FIXME: Why does this not exit(1) like the superclass does?
138         _log.error(script_error.message_with_output())
139
140     @classmethod
141     def load_ews_classes(cls):
142         filesystem = FileSystem()
143         json_path = filesystem.join(filesystem.dirname(filesystem.path_to_module('webkitpy.common.config')), 'ews.json')
144         try:
145             ewses = json.loads(filesystem.read_text_file(json_path))
146         except ValueError:
147             return None
148
149         classes = []
150         for name, config in ewses.iteritems():
151             classes.append(type(str(name.replace(' ', '')), (AbstractEarlyWarningSystem,), {
152                 'name': config['port'] + '-ews',
153                 'port_name': config['port'],
154                 'watchers': config.get('watchers', []),
155                 'run_tests': config.get('runTests', cls.run_tests),
156             }))
157         return classes