2010-11-30 Mihai Parparita <mihaip@chromium.org>
[WebKit-https.git] / WebKitTools / Scripts / webkitpy / tool / commands / rebaselineserver.py
1 # Copyright (c) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 """Starts a local HTTP server which displays layout test failures (given a test
30 results directory), provides comparisons of expected and actual results (both
31 images and text) and allows one-click rebaselining of tests."""
32 from __future__ import with_statement
33
34 import codecs
35 import datetime
36 import mimetypes
37 import os
38 import os.path
39 import shutil
40 import threading
41 import time
42 import urlparse
43 import BaseHTTPServer
44
45 from optparse import make_option
46 from wsgiref.handlers import format_date_time
47
48 from webkitpy.common import system
49 from webkitpy.layout_tests.port import factory
50 from webkitpy.layout_tests.port.webkit import WebKitPort
51 from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
52 from webkitpy.thirdparty import simplejson
53
54 STATE_NEEDS_REBASELINE = 'needs_rebaseline'
55 STATE_REBASELINE_FAILED = 'rebaseline_failed'
56 STATE_REBASELINE_SUCCEEDED = 'rebaseline_succeeded'
57
58 class RebaselineHTTPServer(BaseHTTPServer.HTTPServer):
59     def __init__(self, httpd_port, results_directory, results_json, platforms_json):
60         BaseHTTPServer.HTTPServer.__init__(self, ("", httpd_port), RebaselineHTTPRequestHandler)
61         self.results_directory = results_directory
62         self.results_json = results_json
63         self.platforms_json = platforms_json
64
65
66 class RebaselineHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
67     STATIC_FILE_NAMES = frozenset([
68         "index.html",
69         "loupe.js",
70         "main.js",
71         "main.css",
72         "queue.js",
73         "util.js",
74     ])
75
76     STATIC_FILE_DIRECTORY = os.path.join(
77         os.path.dirname(__file__), "data", "rebaselineserver")
78
79     def do_GET(self):
80         self._handle_request()
81
82     def do_POST(self):
83         self._handle_request()
84
85     def _handle_request(self):
86         # Parse input.
87         if "?" in self.path:
88             path, query_string = self.path.split("?", 1)
89             self.query = urlparse.parse_qs(query_string)
90         else:
91             path = self.path
92             self.query = {}
93         function_or_file_name = path[1:] or "index.html"
94
95         # See if a static file matches.
96         if function_or_file_name in RebaselineHTTPRequestHandler.STATIC_FILE_NAMES:
97             self._serve_static_file(function_or_file_name)
98             return
99
100         # See if a class method matches.
101         function_name = function_or_file_name.replace(".", "_")
102         if not hasattr(self, function_name):
103             self.send_error(404, "Unknown function %s" % function_name)
104             return
105         if function_name[0] == "_":
106             self.send_error(
107                 401, "Not allowed to invoke private or protected methods")
108             return
109         function = getattr(self, function_name)
110         function()
111
112     def _serve_static_file(self, static_path):
113         self._serve_file(os.path.join(
114             RebaselineHTTPRequestHandler.STATIC_FILE_DIRECTORY, static_path))
115
116     def quitquitquit(self):
117         self.send_response(200)
118         self.send_header("Content-type", "text/plain")
119         self.end_headers()
120         self.wfile.write("Quit.\n")
121
122         # Shutdown has to happen on another thread from the server's thread,
123         # otherwise there's a deadlock
124         threading.Thread(target=lambda: self.server.shutdown()).start()
125
126     def test_result(self):
127         test_name, _ = os.path.splitext(self.query['test'][0])
128         mode = self.query['mode'][0]
129         if mode == 'expected-image':
130             file_name = test_name + '-expected.png'
131         elif mode == 'actual-image':
132             file_name = test_name + '-actual.png'
133         if mode == 'expected-checksum':
134             file_name = test_name + '-expected.checksum'
135         elif mode == 'actual-checksum':
136             file_name = test_name + '-actual.checksum'
137         elif mode == 'diff-image':
138             file_name = test_name + '-diff.png'
139         if mode == 'expected-text':
140             file_name = test_name + '-expected.txt'
141         elif mode == 'actual-text':
142             file_name = test_name + '-actual.txt'
143         elif mode == 'diff-text':
144             file_name = test_name + '-diff.txt'
145
146         file_path = os.path.join(self.server.results_directory, file_name)
147
148         # Let results be cached for 60 seconds, so that they can be pre-fetched
149         # by the UI
150         self._serve_file(file_path, cacheable_seconds=60)
151
152     def results_json(self):
153         self._serve_json(self.server.results_json)
154
155     def platforms_json(self):
156         self._serve_json(self.server.platforms_json)
157
158     def _serve_json(self, json):
159         self.send_response(200)
160         self.send_header('Content-type', 'application/json')
161         self.end_headers()
162         simplejson.dump(json, self.wfile)
163
164     def _serve_file(self, file_path, cacheable_seconds=0):
165         if not os.path.exists(file_path):
166             self.send_error(404, "File not found")
167             return
168         with codecs.open(file_path, "rb") as static_file:
169             self.send_response(200)
170             self.send_header("Content-Length", os.path.getsize(file_path))
171             mime_type, encoding = mimetypes.guess_type(file_path)
172             if mime_type:
173                 self.send_header("Content-type", mime_type)
174
175             if cacheable_seconds:
176                 expires_time = (datetime.datetime.now() +
177                     datetime.timedelta(0, cacheable_seconds))
178                 expires_formatted = format_date_time(
179                     time.mktime(expires_time.timetuple()))
180                 self.send_header("Expires", expires_formatted)
181             self.end_headers()
182
183             shutil.copyfileobj(static_file, self.wfile)
184
185
186 def _get_test_baselines(test_file, test_port, layout_tests_directory, platforms, filesystem):
187     class AllPlatformsPort(WebKitPort):
188         def __init__(self):
189             WebKitPort.__init__(self, filesystem=filesystem)
190             self._platforms_by_directory = dict(
191                 [(self._webkit_baseline_path(p), p) for p in platforms])
192
193         def baseline_search_path(self):
194             return self._platforms_by_directory.keys()
195
196         def platform_from_directory(self, directory):
197             return self._platforms_by_directory[directory]
198
199     test_path = filesystem.join(layout_tests_directory, test_file)
200
201     all_platforms_port = AllPlatformsPort()
202
203     all_test_baselines = {}
204     for baseline_extension in ('.txt', '.checksum', '.png'):
205         test_baselines = test_port.expected_baselines(
206             test_path, baseline_extension)
207         baselines = all_platforms_port.expected_baselines(
208             test_path, baseline_extension, all_baselines=True)
209         for platform_directory, expected_filename in baselines:
210             if not platform_directory:
211                 continue
212             if platform_directory == layout_tests_directory:
213                 platform = 'base'
214             else:
215                 platform = all_platforms_port.platform_from_directory(
216                     platform_directory)
217             platform_baselines = all_test_baselines.setdefault(platform, {})
218             was_used_for_test = (
219                 platform_directory, expected_filename) in test_baselines
220             platform_baselines[baseline_extension] = was_used_for_test
221         
222     return all_test_baselines
223
224 class RebaselineServer(AbstractDeclarativeCommand):
225     name = "rebaseline-server"
226     help_text = __doc__
227     argument_names = "/path/to/results/directory"
228
229     def __init__(self):
230         options = [
231             make_option("--httpd-port", action="store", type="int", default=8127, help="Port to use for the the rebaseline HTTP server"),
232         ]
233         AbstractDeclarativeCommand.__init__(self, options=options)
234
235     def execute(self, options, args, tool):
236         results_directory = args[0]
237         filesystem = system.filesystem.FileSystem()
238
239         print 'Parsing unexpected_results.json...'
240         results_json_path = filesystem.join(
241             results_directory, 'unexpected_results.json')
242         with codecs.open(results_json_path, "r") as results_json_file:
243             results_json_file = file(results_json_path)
244             results_json = simplejson.load(results_json_file)
245
246         port = factory.get()
247         layout_tests_directory = port.layout_tests_dir()
248         platforms = filesystem.listdir(
249             filesystem.join(layout_tests_directory, 'platform'))
250
251         print 'Gathering current baselines...'
252         for test_file, test_json in results_json['tests'].items():
253             test_json['state'] = STATE_NEEDS_REBASELINE
254             test_path = filesystem.join(layout_tests_directory, test_file)
255             test_json['baselines'] = _get_test_baselines(
256                 test_file, port, layout_tests_directory, platforms, filesystem)
257
258         server_url = "http://localhost:%d/" % options.httpd_port
259         print "Starting server at %s" % server_url
260         print ("Use the 'Exit' link in the UI, %squitquitquit "
261             "or Ctrl-C to stop") % server_url
262
263         threading.Timer(
264             .1, lambda: self._tool.user.open_url(server_url)).start()
265
266         httpd = RebaselineHTTPServer(
267             httpd_port=options.httpd_port,
268             results_directory=results_directory,
269             results_json=results_json,
270             platforms_json={
271                 'platforms': platforms,
272                 'defaultPlatform': port.name(),
273             })
274         httpd.serve_forever()