Add support for webkitpy tests EWS
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / earlywarningsystem.py
1 # Copyright (c) 2009 Google Inc. All rights reserved.
2 # Copyright (c) 2017 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import json
31 import logging
32 import os
33 from optparse import make_option
34
35 from webkitpy.common.config.committers import CommitterList
36 from webkitpy.common.config.ports import DeprecatedPort
37 from webkitpy.common.system.filesystem import FileSystem
38 from webkitpy.common.system.executive import ScriptError
39 from webkitpy.tool.bot.earlywarningsystemtask import EarlyWarningSystemTask, EarlyWarningSystemTaskDelegate
40 from webkitpy.tool.bot.bindingstestresultsreader import BindingsTestResultsReader
41 from webkitpy.tool.bot.layouttestresultsreader import LayoutTestResultsReader
42 from webkitpy.tool.bot.webkitpytestresultsreader import WebkitpyTestResultsReader
43 from webkitpy.tool.bot.jsctestresultsreader import JSCTestResultsReader
44 from webkitpy.tool.bot.patchanalysistask import UnableToApplyPatch, PatchIsNotValid, PatchIsNotApplicable
45 from webkitpy.tool.bot.queueengine import QueueEngine
46 from webkitpy.tool.commands.queues import AbstractReviewQueue
47
48 _log = logging.getLogger(__name__)
49
50
51 class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDelegate):
52     # FIXME: Switch _default_run_tests from opt-in to opt-out once more bots are ready to run tests.
53     run_tests = False
54
55     def __init__(self):
56         options = [make_option("--run-tests", action="store_true", dest="run_tests", default=self.run_tests, help="Run the Layout tests for each patch")]
57         AbstractReviewQueue.__init__(self, options=options)
58
59     def begin_work_queue(self):
60         AbstractReviewQueue.begin_work_queue(self)
61
62         if self.group() == "jsc":
63             self._test_results_reader = JSCTestResultsReader(self._tool, self._port.jsc_results_directory())
64         elif self.group() == "bindings":
65             self._test_results_reader = BindingsTestResultsReader(self._tool, self._port.jsc_results_directory())
66         elif self.group() == "webkitpy":
67             self._test_results_reader = WebkitpyTestResultsReader(self._tool, self._port.python_unittest_results_directory())
68         else:
69             self._test_results_reader = LayoutTestResultsReader(self._tool, self._port.results_directory(), self._log_directory())
70
71     def _failing_tests_message(self, task, patch):
72         results = task.results_from_patch_test_run(patch)
73
74         if not results:
75             return None
76
77         if results.did_exceed_test_failure_limit():
78             return "Number of test failures exceeded the failure limit."
79         return "New failing tests:\n%s" % "\n".join(results.failing_tests())
80
81     def _post_reject_message_on_bug(self, tool, patch, status_id, extra_message_text=None):
82         if not extra_message_text:
83             return  # Don't comment on Bugzilla if we don't have failing tests.
84
85         results_link = tool.status_server.results_url_for_status(status_id)
86         message = "Attachment %s did not pass %s (%s):\nOutput: %s" % (patch.id(), self.name, self.port_name, results_link)
87         if extra_message_text:
88             message += "\n\n%s" % extra_message_text
89         # FIXME: We might want to add some text about rejecting from the commit-queue in
90         # the case where patch.commit_queue() isn't already set to '-'.
91         if self.watchers:
92             tool.bugs.add_cc_to_bug(patch.bug_id(), self.watchers)
93         tool.bugs.set_flag_on_attachment(patch.id(), "commit-queue", "-", message)
94
95     # This exists for mocking
96     def _create_task(self, patch):
97         return EarlyWarningSystemTask(self, patch, should_run_tests=self._options.run_tests, should_build=self.should_build)
98
99     def review_patch(self, patch):
100         task = self._create_task(patch)
101         try:
102             succeeded = task.run()
103             if not succeeded:
104                 # Caller unlocks when review_patch returns True, so we only need to unlock on transient failure.
105                 self._unlock_patch(patch)
106             return succeeded
107         except PatchIsNotValid as error:
108             self._did_error(patch, "%s did not process patch. Reason: %s" % (self.name, error.failure_message))
109             return False
110         except UnableToApplyPatch, e:
111             self._did_error(patch, "%s unable to apply patch." % self.name)
112             return False
113         except PatchIsNotApplicable, e:
114             self._did_skip(patch)
115             return False
116         except ScriptError, e:
117             self._post_reject_message_on_bug(self._tool, patch, task.failure_status_id, self._failing_tests_message(task, patch))
118             results_archive = task.results_archive_from_patch_test_run(patch)
119             if results_archive:
120                 self._upload_results_archive_for_patch(patch, results_archive)
121             self._did_fail(patch)
122             raise e
123
124     # EarlyWarningSystemDelegate methods
125
126     def parent_command(self):
127         return self.name
128
129     def run_command(self, command):
130         self.run_webkit_patch(command + [self._deprecated_port.flag()] + (['--architecture=%s' % self._port.architecture()] if self._port.architecture() and self._port.did_override_architecture else []))
131
132     def command_passed(self, message, patch):
133         self._update_status(message, patch=patch)
134
135     def command_failed(self, message, script_error, patch):
136         failure_log = self._log_from_script_error_for_upload(script_error)
137         return self._update_status(message, patch=patch, results_file=failure_log)
138
139     def test_results(self):
140         return self._test_results_reader.results()
141
142     def archive_last_test_results(self, patch):
143         return self._test_results_reader.archive(patch)
144
145     def build_style(self):
146         return self._build_style
147
148     def group(self):
149         return self._group
150
151     def refetch_patch(self, patch):
152         return self._tool.bugs.fetch_attachment(patch.id())
153
154     def report_flaky_tests(self, patch, flaky_test_results, results_archive):
155         pass
156
157     # StepSequenceErrorHandler methods
158
159     @classmethod
160     def handle_script_error(cls, tool, state, script_error):
161         # FIXME: Why does this not exit(1) like the superclass does?
162         _log.error(script_error.message_with_output())
163
164     @classmethod
165     def load_ews_classes(cls):
166         filesystem = FileSystem()
167         json_path = filesystem.join(filesystem.dirname(filesystem.path_to_module('webkitpy.common.config')), 'ews.json')
168         try:
169             ewses = json.loads(filesystem.read_text_file(json_path))
170         except ValueError:
171             return None
172
173         classes = []
174         for name, config in ewses.iteritems():
175             classes.append(type(name.encode('utf-8').translate(None, ' -'), (cls,), {
176                 'name': config.get('name', config['port'] + '-ews'),
177                 'port_name': config['port'],
178                 'architecture': config.get('architecture', None),
179                 '_build_style': config.get('style', "release"),
180                 'watchers': config.get('watchers', []),
181                 'run_tests': config.get('runTests', cls.run_tests),
182                 '_group': config.get('group', None),
183                 'should_build': config.get('shouldBuild', True),
184             }))
185         return classes