Fix webkitpy tests after r190922.
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / earlywarningsystem.py
1 # Copyright (c) 2009 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import json
30 import logging
31 from optparse import make_option
32
33 from webkitpy.common.config.committers import CommitterList
34 from webkitpy.common.config.ports import DeprecatedPort
35 from webkitpy.common.system.filesystem import FileSystem
36 from webkitpy.common.system.executive import ScriptError
37 from webkitpy.tool.bot.earlywarningsystemtask import EarlyWarningSystemTask, EarlyWarningSystemTaskDelegate
38 from webkitpy.tool.bot.layouttestresultsreader import LayoutTestResultsReader
39 from webkitpy.tool.bot.patchanalysistask import UnableToApplyPatch, PatchIsNotValid
40 from webkitpy.tool.bot.queueengine import QueueEngine
41 from webkitpy.tool.commands.queues import AbstractReviewQueue
42
43 _log = logging.getLogger(__name__)
44
45
46 class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDelegate):
47     # FIXME: Switch _default_run_tests from opt-in to opt-out once more bots are ready to run tests.
48     run_tests = False
49
50     def __init__(self):
51         options = [make_option("--run-tests", action="store_true", dest="run_tests", default=self.run_tests, help="Run the Layout tests for each patch")]
52         AbstractReviewQueue.__init__(self, options=options)
53
54     def begin_work_queue(self):
55         AbstractReviewQueue.begin_work_queue(self)
56         self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._port.results_directory(), self._log_directory())
57
58     def _failing_tests_message(self, task, patch):
59         results = task.results_from_patch_test_run(patch)
60
61         if not results:
62             return None
63
64         if results.did_exceed_test_failure_limit():
65             return "Number of test failures exceeded the failure limit."
66         return "New failing tests:\n%s" % "\n".join(results.failing_tests())
67
68     def _post_reject_message_on_bug(self, tool, patch, status_id, extra_message_text=None):
69         if not extra_message_text:
70             return  # Don't comment on Bugzilla if we don't have failing tests.
71
72         results_link = tool.status_server.results_url_for_status(status_id)
73         message = "Attachment %s did not pass %s (%s):\nOutput: %s" % (patch.id(), self.name, self.port_name, results_link)
74         if extra_message_text:
75             message += "\n\n%s" % extra_message_text
76         # FIXME: We might want to add some text about rejecting from the commit-queue in
77         # the case where patch.commit_queue() isn't already set to '-'.
78         if self.watchers:
79             tool.bugs.add_cc_to_bug(patch.bug_id(), self.watchers)
80         tool.bugs.set_flag_on_attachment(patch.id(), "commit-queue", "-", message)
81
82     def review_patch(self, patch):
83         task = EarlyWarningSystemTask(self, patch, self._options.run_tests)
84         try:
85             succeeded = task.run()
86             if not succeeded:
87                 # Caller unlocks when review_patch returns True, so we only need to unlock on transient failure.
88                 self._unlock_patch(patch)
89             return succeeded
90         except PatchIsNotValid:
91             self._did_error(patch, "%s did not process patch." % self.name)
92             return False
93         except UnableToApplyPatch, e:
94             self._did_error(patch, "%s unable to apply patch." % self.name)
95             return False
96         except ScriptError, e:
97             self._post_reject_message_on_bug(self._tool, patch, task.failure_status_id, self._failing_tests_message(task, patch))
98             results_archive = task.results_archive_from_patch_test_run(patch)
99             if results_archive:
100                 self._upload_results_archive_for_patch(patch, results_archive)
101             self._did_fail(patch)
102             raise e
103
104     # EarlyWarningSystemDelegate methods
105
106     def parent_command(self):
107         return self.name
108
109     def run_command(self, command):
110         self.run_webkit_patch(command + [self._deprecated_port.flag()] + (['--architecture=%s' % self._port.architecture()] if self._port.architecture() and self._port.did_override_architecture else []))
111
112     def command_passed(self, message, patch):
113         pass
114
115     def command_failed(self, message, script_error, patch):
116         failure_log = self._log_from_script_error_for_upload(script_error)
117         return self._update_status(message, patch=patch, results_file=failure_log)
118
119     def test_results(self):
120         return self._layout_test_results_reader.results()
121
122     def archive_last_test_results(self, patch):
123         return self._layout_test_results_reader.archive(patch)
124
125     def build_style(self):
126         return self._build_style
127
128     def refetch_patch(self, patch):
129         return self._tool.bugs.fetch_attachment(patch.id())
130
131     def report_flaky_tests(self, patch, flaky_test_results, results_archive):
132         pass
133
134     # StepSequenceErrorHandler methods
135
136     @classmethod
137     def handle_script_error(cls, tool, state, script_error):
138         # FIXME: Why does this not exit(1) like the superclass does?
139         _log.error(script_error.message_with_output())
140
141     @classmethod
142     def load_ews_classes(cls):
143         filesystem = FileSystem()
144         json_path = filesystem.join(filesystem.dirname(filesystem.path_to_module('webkitpy.common.config')), 'ews.json')
145         try:
146             ewses = json.loads(filesystem.read_text_file(json_path))
147         except ValueError:
148             return None
149
150         classes = []
151         for name, config in ewses.iteritems():
152             classes.append(type(name.encode('utf-8').translate(None, ' -'), (AbstractEarlyWarningSystem,), {
153                 'name': config.get('name', config['port'] + '-ews'),
154                 'port_name': config['port'],
155                 'architecture': config.get('architecture', None),
156                 '_build_style': config.get('style', "release"),
157                 'watchers': config.get('watchers', []),
158                 'run_tests': config.get('runTests', cls.run_tests),
159             }))
160         return classes