[WinCairo] httpd service install needs to precede server start
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / perfalizer.py
1 # Copyright (c) 2012 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import logging
30
31 from webkitpy.tool.bot.irc_command import IRCCommand
32 from webkitpy.tool.bot.irc_command import Help
33 from webkitpy.tool.bot.irc_command import Hi
34 from webkitpy.tool.bot.irc_command import Restart
35 from webkitpy.tool.bot.ircbot import IRCBot
36 from webkitpy.tool.bot.patchanalysistask import PatchAnalysisTask, PatchAnalysisTaskDelegate, UnableToApplyPatch
37 from webkitpy.tool.bot.sheriff import Sheriff
38 from webkitpy.tool.commands.queues import AbstractQueue
39 from webkitpy.tool.commands.stepsequence import StepSequenceErrorHandler
40
41 _log = logging.getLogger(__name__)
42
43
44 class PerfalizerTask(PatchAnalysisTask):
45     def __init__(self, tool, patch, logger):
46         PatchAnalysisTask.__init__(self, self, patch)
47         self._port = tool.port_factory.get()
48         self._tool = tool
49         self._logger = logger
50
51     def _copy_build_product_without_patch(self):
52         filesystem = self._tool.filesystem
53         configuration = filesystem.basename(self._port._build_path())
54         self._build_directory = filesystem.dirname(self._port._build_path())
55         self._build_directory_without_patch = self._build_directory + 'WithoutPatch'
56
57         try:
58             filesystem.rmtree(self._build_directory_without_patch)
59             filesystem.copytree(filesystem.join(self._build_directory, configuration),
60                 filesystem.join(self._build_directory_without_patch, configuration))
61             return True
62         except:
63             return False
64
65     def validate(self):
66         return True
67
68     def run(self):
69         if not self._patch.committer() and not self._patch.attacher().can_commit:
70             self._logger('The patch %d is not authorized by a commmitter' % self._patch.id())
71             return False
72
73         self._logger('Preparing to run performance tests for the attachment %d...' % self._patch.id())
74         if not self._clean() or not self._update():
75             return False
76
77         head_revision = self._tool.scm().head_svn_revision()
78
79         self._logger('Building WebKit at r%s without the patch' % head_revision)
80         if not self._build_without_patch():
81             return False
82
83         if not self._port.check_build():
84             self._logger('Failed to build DumpRenderTree.')
85             return False
86
87         if not self._copy_build_product_without_patch():
88             self._logger('Failed to copy the build product from %s to %s' % (self._build_directory, self._build_directory_without_patch))
89             return False
90
91         self._logger('Building WebKit at r%s with the patch' % head_revision)
92         if not self._apply() or not self._build():
93             return False
94
95         if not self._port.check_build():
96             self._logger('Failed to build DumpRenderTree.')
97             return False
98
99         filesystem = self._tool.filesystem
100         if filesystem.exists(self._json_path()):
101             filesystem.remove(self._json_path())
102
103         self._logger("Running performance tests...")
104         if self._run_perf_test(self._build_directory_without_patch, 'without %d' % self._patch.id()) < 0:
105             self._logger('Failed to run performance tests without the patch.')
106             return False
107
108         if self._run_perf_test(self._build_directory, 'with %d' % self._patch.id()) < 0:
109             self._logger('Failed to run performance tests with the patch.')
110             return False
111
112         if not filesystem.exists(self._results_page_path()):
113             self._logger('Failed to generate the results page.')
114             return False
115
116         results_page = filesystem.read_text_file(self._results_page_path())
117         self._tool.bugs.add_attachment_to_bug(self._patch.bug_id(), results_page,
118             description="Performance tests results for %d" % self._patch.id(), mimetype='text/html')
119
120         self._logger("Uploaded the results on the bug %d" % self._patch.bug_id())
121         return True
122
123     def parent_command(self):
124         return "perfalizer"
125
126     def run_webkit_patch(self, args):
127         webkit_patch_args = [self._tool.path()]
128         webkit_patch_args.extend(args)
129         return self._tool.executive.run_and_throw_if_fail(webkit_patch_args, cwd=self._tool.scm().checkout_root)
130
131     def _json_path(self):
132         return self._tool.filesystem.join(self._build_directory, 'PerformanceTestResults.json')
133
134     def _results_page_path(self):
135         return self._tool.filesystem.join(self._build_directory, 'PerformanceTestResults.html')
136
137     def _run_perf_test(self, build_path, description):
138         filesystem = self._tool.filesystem
139         script_path = filesystem.join(filesystem.dirname(self._tool.path()), 'run-perf-tests')
140         perf_test_runner_args = [script_path, '--no-build', '--no-show-results', '--build-directory', build_path,
141             '--output-json-path', self._json_path(), '--description', description]
142         return self._tool.executive.run_and_throw_if_fail(perf_test_runner_args, cwd=self._tool.scm().checkout_root)
143
144     def run_command(self, command):
145         self.run_webkit_patch(command)
146
147     def command_passed(self, message, patch):
148         pass
149
150     def command_failed(self, message, script_error, patch):
151         self._logger(message)
152
153     def refetch_patch(self, patch):
154         return self._tool.bugs.fetch_attachment(patch.id())
155
156     def build_style(self):
157         return "release"
158
159
160 class PerfTest(IRCCommand):
161     def execute(self, nick, args, tool, sheriff):
162         if not args:
163             tool.irc().post(nick + ": Please specify an attachment/patch id")
164             return
165
166         patch_id = args[0]
167         patch = tool.bugs.fetch_attachment(patch_id)
168         if not patch:
169             tool.irc().post(nick + ": Could not fetch the patch")
170             return
171
172         task = PerfalizerTask(tool, patch, lambda message: tool.irc().post('%s: %s' % (nick, message)))
173         task.run()
174
175
176 class Perfalizer(AbstractQueue, StepSequenceErrorHandler):
177     name = "perfalizer"
178     watchers = AbstractQueue.watchers + ["rniwa@webkit.org"]
179
180     _commands = {
181         "help": Help,
182         "hi": Hi,
183         "restart": Restart,
184         "test": PerfTest,
185     }
186
187     # AbstractQueue methods
188
189     def begin_work_queue(self):
190         AbstractQueue.begin_work_queue(self)
191         self._sheriff = Sheriff(self._tool, self)
192         self._irc_bot = IRCBot("perfalizer", self._tool, self._sheriff, self._commands)
193         self._tool.ensure_irc_connected(self._irc_bot.irc_delegate())
194
195     def work_item_log_path(self, failure_map):
196         return None
197
198     def _is_old_failure(self, revision):
199         return self._tool.status_server.svn_revision(revision)
200
201     def next_work_item(self):
202         self._irc_bot.process_pending_messages()
203         return
204
205     def process_work_item(self, failure_map):
206         return True
207
208     def handle_unexpected_error(self, failure_map, message):
209         _log.error(message)
210
211     # StepSequenceErrorHandler methods
212
213     @classmethod
214     def handle_script_error(cls, tool, state, script_error):
215         # Ideally we would post some information to IRC about what went wrong
216         # here, but we don't have the IRC password in the child process.
217         pass