Clean up ChunkedUpdateDrawingAreaProxy
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / rebaselineserver.py
1 # Copyright (c) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 """Starts a local HTTP server which displays layout test failures (given a test
30 results directory), provides comparisons of expected and actual results (both
31 images and text) and allows one-click rebaselining of tests."""
32 from __future__ import with_statement
33
34 import codecs
35 import datetime
36 import fnmatch
37 import mimetypes
38 import os
39 import os.path
40 import shutil
41 import threading
42 import time
43 import urlparse
44 import BaseHTTPServer
45
46 from optparse import make_option
47 from wsgiref.handlers import format_date_time
48
49 from webkitpy.common import system
50 from webkitpy.layout_tests.port import factory
51 from webkitpy.layout_tests.port.webkit import WebKitPort
52 from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
53 from webkitpy.thirdparty import simplejson
54
55 STATE_NEEDS_REBASELINE = 'needs_rebaseline'
56 STATE_REBASELINE_FAILED = 'rebaseline_failed'
57 STATE_REBASELINE_SUCCEEDED = 'rebaseline_succeeded'
58
59 class RebaselineHTTPServer(BaseHTTPServer.HTTPServer):
60     def __init__(self, httpd_port, test_config, results_json, platforms_json):
61         BaseHTTPServer.HTTPServer.__init__(self, ("", httpd_port), RebaselineHTTPRequestHandler)
62         self.test_config = test_config
63         self.results_json = results_json
64         self.platforms_json = platforms_json
65
66
67 class RebaselineHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
68     STATIC_FILE_NAMES = frozenset([
69         "index.html",
70         "loupe.js",
71         "main.js",
72         "main.css",
73         "queue.js",
74         "util.js",
75     ])
76
77     STATIC_FILE_DIRECTORY = os.path.join(
78         os.path.dirname(__file__), "data", "rebaselineserver")
79
80     def do_GET(self):
81         self._handle_request()
82
83     def do_POST(self):
84         self._handle_request()
85
86     def _handle_request(self):
87         # Parse input.
88         if "?" in self.path:
89             path, query_string = self.path.split("?", 1)
90             self.query = urlparse.parse_qs(query_string)
91         else:
92             path = self.path
93             self.query = {}
94         function_or_file_name = path[1:] or "index.html"
95
96         # See if a static file matches.
97         if function_or_file_name in RebaselineHTTPRequestHandler.STATIC_FILE_NAMES:
98             self._serve_static_file(function_or_file_name)
99             return
100
101         # See if a class method matches.
102         function_name = function_or_file_name.replace(".", "_")
103         if not hasattr(self, function_name):
104             self.send_error(404, "Unknown function %s" % function_name)
105             return
106         if function_name[0] == "_":
107             self.send_error(
108                 401, "Not allowed to invoke private or protected methods")
109             return
110         function = getattr(self, function_name)
111         function()
112
113     def _serve_static_file(self, static_path):
114         self._serve_file(os.path.join(
115             RebaselineHTTPRequestHandler.STATIC_FILE_DIRECTORY, static_path))
116
117     def rebaseline(self):
118         test = self.query['test'][0]
119         baseline_target = self.query['baseline-target'][0]
120         baseline_move_to = self.query['baseline-move-to'][0]
121         test_json = self.server.results_json['tests'][test]
122
123         if test_json['state'] != STATE_NEEDS_REBASELINE:
124             self.send_error(400, "Test %s is in unexpected state: %s" %
125                 (test, test_json["state"]))
126             return
127
128         log = []
129         success = _rebaseline_test(
130             test,
131             baseline_target,
132             baseline_move_to,
133             self.server.test_config,
134             log=lambda l: log.append(l))
135
136         if success:
137             test_json['state'] = STATE_REBASELINE_SUCCEEDED
138             self.send_response(200)
139         else:
140             test_json['state'] = STATE_REBASELINE_FAILED
141             self.send_response(500)
142
143         self.send_header('Content-type', 'text/plain')
144         self.end_headers()
145         self.wfile.write('\n'.join(log))
146
147     def quitquitquit(self):
148         self.send_response(200)
149         self.send_header("Content-type", "text/plain")
150         self.end_headers()
151         self.wfile.write("Quit.\n")
152
153         # Shutdown has to happen on another thread from the server's thread,
154         # otherwise there's a deadlock
155         threading.Thread(target=lambda: self.server.shutdown()).start()
156
157     def test_result(self):
158         test_name, _ = os.path.splitext(self.query['test'][0])
159         mode = self.query['mode'][0]
160         if mode == 'expected-image':
161             file_name = test_name + '-expected.png'
162         elif mode == 'actual-image':
163             file_name = test_name + '-actual.png'
164         if mode == 'expected-checksum':
165             file_name = test_name + '-expected.checksum'
166         elif mode == 'actual-checksum':
167             file_name = test_name + '-actual.checksum'
168         elif mode == 'diff-image':
169             file_name = test_name + '-diff.png'
170         if mode == 'expected-text':
171             file_name = test_name + '-expected.txt'
172         elif mode == 'actual-text':
173             file_name = test_name + '-actual.txt'
174         elif mode == 'diff-text':
175             file_name = test_name + '-diff.txt'
176         elif mode == 'diff-text-pretty':
177             file_name = test_name + '-pretty-diff.html'
178
179         file_path = os.path.join(self.server.test_config.results_directory, file_name)
180
181         # Let results be cached for 60 seconds, so that they can be pre-fetched
182         # by the UI
183         self._serve_file(file_path, cacheable_seconds=60)
184
185     def results_json(self):
186         self._serve_json(self.server.results_json)
187
188     def platforms_json(self):
189         self._serve_json(self.server.platforms_json)
190
191     def _serve_json(self, json):
192         self.send_response(200)
193         self.send_header('Content-type', 'application/json')
194         self.end_headers()
195         simplejson.dump(json, self.wfile)
196
197     def _serve_file(self, file_path, cacheable_seconds=0):
198         if not os.path.exists(file_path):
199             self.send_error(404, "File not found")
200             return
201         with codecs.open(file_path, "rb") as static_file:
202             self.send_response(200)
203             self.send_header("Content-Length", os.path.getsize(file_path))
204             mime_type, encoding = mimetypes.guess_type(file_path)
205             if mime_type:
206                 self.send_header("Content-type", mime_type)
207
208             if cacheable_seconds:
209                 expires_time = (datetime.datetime.now() +
210                     datetime.timedelta(0, cacheable_seconds))
211                 expires_formatted = format_date_time(
212                     time.mktime(expires_time.timetuple()))
213                 self.send_header("Expires", expires_formatted)
214             self.end_headers()
215
216             shutil.copyfileobj(static_file, self.wfile)
217
218
219 class TestConfig(object):
220     def __init__(self, test_port, layout_tests_directory, results_directory, platforms, filesystem, scm):
221         self.test_port = test_port
222         self.layout_tests_directory = layout_tests_directory
223         self.results_directory = results_directory
224         self.platforms = platforms
225         self.filesystem = filesystem
226         self.scm = scm
227
228
229 def _get_actual_result_files(test_file, test_config):
230     test_name, _ = os.path.splitext(test_file)
231     test_directory = os.path.dirname(test_file)
232
233     test_results_directory = test_config.filesystem.join(
234         test_config.results_directory, test_directory)
235     actual_pattern = os.path.basename(test_name) + '-actual.*'
236     actual_files = []
237     for filename in test_config.filesystem.listdir(test_results_directory):
238         if fnmatch.fnmatch(filename, actual_pattern):
239             actual_files.append(filename)
240     actual_files.sort()
241     return tuple(actual_files)
242
243
244 def _rebaseline_test(test_file, baseline_target, baseline_move_to, test_config, log):
245     test_name, _ = os.path.splitext(test_file)
246     test_directory = os.path.dirname(test_name)
247
248     log('Rebaselining %s...' % test_name)
249
250     actual_result_files = _get_actual_result_files(test_file, test_config)
251     filesystem = test_config.filesystem
252     scm = test_config.scm
253     layout_tests_directory = test_config.layout_tests_directory
254     results_directory = test_config.results_directory
255     target_expectations_directory = filesystem.join(
256         layout_tests_directory, 'platform', baseline_target, test_directory)
257     test_results_directory = test_config.filesystem.join(
258         test_config.results_directory, test_directory)
259
260     # If requested, move current baselines out
261     current_baselines = _get_test_baselines(test_file, test_config)
262     if baseline_target in current_baselines and baseline_move_to != 'none':
263         log('  Moving current %s baselines to %s' %
264             (baseline_target, baseline_move_to))
265
266         # See which ones we need to move (only those that are about to be
267         # updated), and make sure we're not clobbering any files in the
268         # destination.
269         current_extensions = set(current_baselines[baseline_target].keys())
270         actual_result_extensions = [
271             os.path.splitext(f)[1] for f in actual_result_files]
272         extensions_to_move = current_extensions.intersection(
273             actual_result_extensions)
274
275         if extensions_to_move.intersection(
276             current_baselines.get(baseline_move_to, {}).keys()):
277             log('    Already had baselines in %s, could not move existing '
278                 '%s ones' % (baseline_move_to, baseline_target))
279             return False
280
281         # Do the actual move.
282         if extensions_to_move:
283             if not _move_test_baselines(
284                 test_file,
285                 list(extensions_to_move),
286                 baseline_target,
287                 baseline_move_to,
288                 test_config,
289                 log):
290                 return False
291         else:
292             log('    No current baselines to move')
293
294     log('  Updating baselines for %s' % baseline_target)
295     filesystem.maybe_make_directory(target_expectations_directory)
296     for source_file in actual_result_files:
297         source_path = filesystem.join(test_results_directory, source_file)
298         destination_file = source_file.replace('-actual', '-expected')
299         destination_path = filesystem.join(
300             target_expectations_directory, destination_file)
301         filesystem.copyfile(source_path, destination_path)
302         exit_code = scm.add(destination_path, return_exit_code=True)
303         if exit_code:
304             log('    Could not update %s in SCM, exit code %d' %
305                 (destination_file, exit_code))
306             return False
307         else:
308             log('    Updated %s' % destination_file)
309
310     return True
311
312
313 def _move_test_baselines(test_file, extensions_to_move, source_platform, destination_platform, test_config, log):
314     test_file_name = os.path.splitext(os.path.basename(test_file))[0]
315     test_directory = os.path.dirname(test_file)
316     filesystem = test_config.filesystem
317
318     # Want predictable output order for unit tests.
319     extensions_to_move.sort()
320
321     source_directory = os.path.join(
322         test_config.layout_tests_directory,
323         'platform',
324         source_platform,
325         test_directory)
326     destination_directory = os.path.join(
327         test_config.layout_tests_directory,
328         'platform',
329         destination_platform,
330         test_directory)
331     filesystem.maybe_make_directory(destination_directory)
332
333     for extension in extensions_to_move:
334         file_name = test_file_name + '-expected' + extension
335         source_path = filesystem.join(source_directory, file_name)
336         destination_path = filesystem.join(destination_directory, file_name)
337         filesystem.copyfile(source_path, destination_path)
338         exit_code = test_config.scm.add(destination_path, return_exit_code=True)
339         if exit_code:
340             log('    Could not update %s in SCM, exit code %d' %
341                 (file_name, exit_code))
342             return False
343         else:
344             log('    Moved %s' % file_name)
345
346     return True
347
348 def _get_test_baselines(test_file, test_config):
349     class AllPlatformsPort(WebKitPort):
350         def __init__(self):
351             WebKitPort.__init__(self, filesystem=test_config.filesystem)
352             self._platforms_by_directory = dict(
353                 [(self._webkit_baseline_path(p), p) for p in test_config.platforms])
354
355         def baseline_search_path(self):
356             return self._platforms_by_directory.keys()
357
358         def platform_from_directory(self, directory):
359             return self._platforms_by_directory[directory]
360
361     test_path = test_config.filesystem.join(
362         test_config.layout_tests_directory, test_file)
363
364     all_platforms_port = AllPlatformsPort()
365
366     all_test_baselines = {}
367     for baseline_extension in ('.txt', '.checksum', '.png'):
368         test_baselines = test_config.test_port.expected_baselines(
369             test_path, baseline_extension)
370         baselines = all_platforms_port.expected_baselines(
371             test_path, baseline_extension, all_baselines=True)
372         for platform_directory, expected_filename in baselines:
373             if not platform_directory:
374                 continue
375             if platform_directory == test_config.layout_tests_directory:
376                 platform = 'base'
377             else:
378                 platform = all_platforms_port.platform_from_directory(
379                     platform_directory)
380             platform_baselines = all_test_baselines.setdefault(platform, {})
381             was_used_for_test = (
382                 platform_directory, expected_filename) in test_baselines
383             platform_baselines[baseline_extension] = was_used_for_test
384
385     return all_test_baselines
386
387
388 class RebaselineServer(AbstractDeclarativeCommand):
389     name = "rebaseline-server"
390     help_text = __doc__
391     argument_names = "/path/to/results/directory"
392
393     def __init__(self):
394         options = [
395             make_option("--httpd-port", action="store", type="int", default=8127, help="Port to use for the the rebaseline HTTP server"),
396         ]
397         AbstractDeclarativeCommand.__init__(self, options=options)
398
399     def execute(self, options, args, tool):
400         results_directory = args[0]
401         filesystem = system.filesystem.FileSystem()
402         scm = self._tool.scm()
403
404         if options.dry_run:
405
406             def no_op_copyfile(src, dest):
407                 pass
408
409             def no_op_add(path, return_exit_code=False):
410                 if return_exit_code:
411                     return 0
412
413             filesystem.copyfile = no_op_copyfile
414             scm.add = no_op_add
415
416         print 'Parsing unexpected_results.json...'
417         results_json_path = filesystem.join(
418             results_directory, 'unexpected_results.json')
419         with codecs.open(results_json_path, "r") as results_json_file:
420             results_json_file = file(results_json_path)
421             results_json = simplejson.load(results_json_file)
422
423         port = factory.get()
424         layout_tests_directory = port.layout_tests_dir()
425         platforms = filesystem.listdir(
426             filesystem.join(layout_tests_directory, 'platform'))
427         test_config = TestConfig(
428             port,
429             layout_tests_directory,
430             results_directory,
431             platforms,
432             filesystem,
433             scm)
434
435         print 'Gathering current baselines...'
436         for test_file, test_json in results_json['tests'].items():
437             test_json['state'] = STATE_NEEDS_REBASELINE
438             test_path = filesystem.join(layout_tests_directory, test_file)
439             test_json['baselines'] = _get_test_baselines(test_file, test_config)
440
441         server_url = "http://localhost:%d/" % options.httpd_port
442         print "Starting server at %s" % server_url
443         print ("Use the 'Exit' link in the UI, %squitquitquit "
444             "or Ctrl-C to stop") % server_url
445
446         threading.Timer(
447             .1, lambda: self._tool.user.open_url(server_url)).start()
448
449         httpd = RebaselineHTTPServer(
450             httpd_port=options.httpd_port,
451             test_config=test_config,
452             results_json=results_json,
453             platforms_json={
454                 'platforms': platforms,
455                 'defaultPlatform': port.name(),
456             })
457         httpd.serve_forever()