2010-09-22 Dirk Pranke <dpranke@chromium.org>
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / rebaseline_chromium_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Rebaselining tool that automatically produces baselines for all platforms.
31
32 The script does the following for each platform specified:
33   1. Compile a list of tests that need rebaselining.
34   2. Download test result archive from buildbot for the platform.
35   3. Extract baselines from the archive file for all identified files.
36   4. Add new baselines to SVN repository.
37   5. For each test that has been rebaselined, remove this platform option from
38      the test in test_expectation.txt. If no other platforms remain after
39      removal, delete the rebaselined test from the file.
40
41 At the end, the script generates a html that compares old and new baselines.
42 """
43
44 from __future__ import with_statement
45
46 import codecs
47 import copy
48 import logging
49 import optparse
50 import os
51 import re
52 import shutil
53 import subprocess
54 import sys
55 import tempfile
56 import time
57 import urllib
58 import zipfile
59
60 from webkitpy.common.system import user
61 from webkitpy.common.system.executive import run_command, ScriptError
62 import webkitpy.common.checkout.scm as scm
63
64 import port
65 from layout_package import test_expectations
66
67 _log = logging.getLogger("webkitpy.layout_tests."
68                          "rebaseline_chromium_webkit_tests")
69
70 BASELINE_SUFFIXES = ['.txt', '.png', '.checksum']
71 REBASELINE_PLATFORM_ORDER = ['mac', 'win', 'win-xp', 'win-vista', 'linux']
72 ARCHIVE_DIR_NAME_DICT = {'win': 'webkit-rel',
73                          'win-vista': 'webkit-dbg-vista',
74                          'win-xp': 'webkit-rel',
75                          'mac': 'webkit-rel-mac5',
76                          'linux': 'webkit-rel-linux',
77                          'win-canary': 'webkit-rel-webkit-org',
78                          'win-vista-canary': 'webkit-dbg-vista',
79                          'win-xp-canary': 'webkit-rel-webkit-org',
80                          'mac-canary': 'webkit-rel-mac-webkit-org',
81                          'linux-canary': 'webkit-rel-linux-webkit-org'}
82
83
84 # FIXME: Should be rolled into webkitpy.Executive
85 def run_shell_with_return_code(command, print_output=False):
86     """Executes a command and returns the output and process return code.
87
88     Args:
89       command: program and arguments.
90       print_output: if true, print the command results to standard output.
91
92     Returns:
93       command output, return code
94     """
95
96     # Use a shell for subcommands on Windows to get a PATH search.
97     # FIXME: shell=True is a trail of tears, and should be removed.
98     use_shell = sys.platform.startswith('win')
99     # Note: Not thread safe: http://bugs.python.org/issue2320
100     p = subprocess.Popen(command, stdout=subprocess.PIPE,
101                          stderr=subprocess.STDOUT, shell=use_shell)
102     if print_output:
103         output_array = []
104         while True:
105             line = p.stdout.readline()
106             if not line:
107                 break
108             if print_output:
109                 print line.strip('\n')
110             output_array.append(line)
111         output = ''.join(output_array)
112     else:
113         output = p.stdout.read()
114     p.wait()
115     p.stdout.close()
116
117     return output, p.returncode
118
119
120 # FIXME: Should be rolled into webkitpy.Executive
121 def run_shell(command, print_output=False):
122     """Executes a command and returns the output.
123
124     Args:
125       command: program and arguments.
126       print_output: if true, print the command results to standard output.
127
128     Returns:
129       command output
130     """
131
132     output, return_code = run_shell_with_return_code(command, print_output)
133     return output
134
135
136 def log_dashed_string(text, platform, logging_level=logging.INFO):
137     """Log text message with dashes on both sides."""
138
139     msg = text
140     if platform:
141         msg += ': ' + platform
142     if len(msg) < 78:
143         dashes = '-' * ((78 - len(msg)) / 2)
144         msg = '%s %s %s' % (dashes, msg, dashes)
145
146     if logging_level == logging.ERROR:
147         _log.error(msg)
148     elif logging_level == logging.WARNING:
149         _log.warn(msg)
150     else:
151         _log.info(msg)
152
153
154 def setup_html_directory(html_directory):
155     """Setup the directory to store html results.
156
157        All html related files are stored in the "rebaseline_html" subdirectory.
158
159     Args:
160       html_directory: parent directory that stores the rebaselining results.
161                       If None, a temp directory is created.
162
163     Returns:
164       the directory that stores the html related rebaselining results.
165     """
166
167     if not html_directory:
168         html_directory = tempfile.mkdtemp()
169     elif not os.path.exists(html_directory):
170         os.mkdir(html_directory)
171
172     html_directory = os.path.join(html_directory, 'rebaseline_html')
173     _log.info('Html directory: "%s"', html_directory)
174
175     if os.path.exists(html_directory):
176         shutil.rmtree(html_directory, True)
177         _log.info('Deleted file at html directory: "%s"', html_directory)
178
179     if not os.path.exists(html_directory):
180         os.mkdir(html_directory)
181     return html_directory
182
183
184 def get_result_file_fullpath(html_directory, baseline_filename, platform,
185                              result_type):
186     """Get full path of the baseline result file.
187
188     Args:
189       html_directory: directory that stores the html related files.
190       baseline_filename: name of the baseline file.
191       platform: win, linux or mac
192       result_type: type of the baseline result: '.txt', '.png'.
193
194     Returns:
195       Full path of the baseline file for rebaselining result comparison.
196     """
197
198     base, ext = os.path.splitext(baseline_filename)
199     result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext)
200     fullpath = os.path.join(html_directory, result_filename)
201     _log.debug('  Result file full path: "%s".', fullpath)
202     return fullpath
203
204
205 class Rebaseliner(object):
206     """Class to produce new baselines for a given platform."""
207
208     REVISION_REGEX = r'<a href=\"(\d+)/\">'
209
210     def __init__(self, running_port, target_port, platform, options):
211         """
212         Args:
213             running_port: the Port the script is running on.
214             target_port: the Port the script uses to find port-specific
215                 configuration information like the test_expectations.txt
216                 file location and the list of test platforms.
217             platform: the test platform to rebaseline
218             options: the command-line options object."""
219         self._platform = platform
220         self._options = options
221         self._port = running_port
222         self._target_port = target_port
223         self._rebaseline_port = port.get(
224             self._target_port.test_platform_name_to_name(platform), options)
225         self._rebaselining_tests = []
226         self._rebaselined_tests = []
227
228         # Create tests and expectations helper which is used to:
229         #   -. compile list of tests that need rebaselining.
230         #   -. update the tests in test_expectations file after rebaseline
231         #      is done.
232         expectations_str = self._rebaseline_port.test_expectations()
233         self._test_expectations = \
234             test_expectations.TestExpectations(self._rebaseline_port,
235                                                None,
236                                                expectations_str,
237                                                self._platform,
238                                                False,
239                                                False)
240         self._scm = scm.default_scm()
241
242     def run(self, backup):
243         """Run rebaseline process."""
244
245         log_dashed_string('Compiling rebaselining tests', self._platform)
246         if not self._compile_rebaselining_tests():
247             return True
248
249         log_dashed_string('Downloading archive', self._platform)
250         archive_file = self._download_buildbot_archive()
251         _log.info('')
252         if not archive_file:
253             _log.error('No archive found.')
254             return False
255
256         log_dashed_string('Extracting and adding new baselines',
257                           self._platform)
258         if not self._extract_and_add_new_baselines(archive_file):
259             return False
260
261         log_dashed_string('Updating rebaselined tests in file',
262                           self._platform)
263         self._update_rebaselined_tests_in_file(backup)
264         _log.info('')
265
266         if len(self._rebaselining_tests) != len(self._rebaselined_tests):
267             _log.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN '
268                          'REBASELINED.')
269             _log.warning('  Total tests needing rebaselining: %d',
270                          len(self._rebaselining_tests))
271             _log.warning('  Total tests rebaselined: %d',
272                          len(self._rebaselined_tests))
273             return False
274
275         _log.warning('All tests needing rebaselining were successfully '
276                      'rebaselined.')
277
278         return True
279
280     def get_rebaselining_tests(self):
281         return self._rebaselining_tests
282
283     def _compile_rebaselining_tests(self):
284         """Compile list of tests that need rebaselining for the platform.
285
286         Returns:
287           List of tests that need rebaselining or
288           None if there is no such test.
289         """
290
291         self._rebaselining_tests = \
292             self._test_expectations.get_rebaselining_failures()
293         if not self._rebaselining_tests:
294             _log.warn('No tests found that need rebaselining.')
295             return None
296
297         _log.info('Total number of tests needing rebaselining '
298                   'for "%s": "%d"', self._platform,
299                   len(self._rebaselining_tests))
300
301         test_no = 1
302         for test in self._rebaselining_tests:
303             _log.info('  %d: %s', test_no, test)
304             test_no += 1
305
306         return self._rebaselining_tests
307
308     def _get_latest_revision(self, url):
309         """Get the latest layout test revision number from buildbot.
310
311         Args:
312           url: Url to retrieve layout test revision numbers.
313
314         Returns:
315           latest revision or
316           None on failure.
317         """
318
319         _log.debug('Url to retrieve revision: "%s"', url)
320
321         f = urllib.urlopen(url)
322         content = f.read()
323         f.close()
324
325         revisions = re.findall(self.REVISION_REGEX, content)
326         if not revisions:
327             _log.error('Failed to find revision, content: "%s"', content)
328             return None
329
330         revisions.sort(key=int)
331         _log.info('Latest revision: "%s"', revisions[len(revisions) - 1])
332         return revisions[len(revisions) - 1]
333
334     def _get_archive_dir_name(self, platform, webkit_canary):
335         """Get name of the layout test archive directory.
336
337         Returns:
338           Directory name or
339           None on failure
340         """
341
342         if webkit_canary:
343             platform += '-canary'
344
345         if platform in ARCHIVE_DIR_NAME_DICT:
346             return ARCHIVE_DIR_NAME_DICT[platform]
347         else:
348             _log.error('Cannot find platform key %s in archive '
349                        'directory name dictionary', platform)
350             return None
351
352     def _get_archive_url(self):
353         """Generate the url to download latest layout test archive.
354
355         Returns:
356           Url to download archive or
357           None on failure
358         """
359
360         if self._options.force_archive_url:
361             return self._options.force_archive_url
362
363         dir_name = self._get_archive_dir_name(self._platform,
364                                               self._options.webkit_canary)
365         if not dir_name:
366             return None
367
368         _log.debug('Buildbot platform dir name: "%s"', dir_name)
369
370         url_base = '%s/%s/' % (self._options.archive_url, dir_name)
371         latest_revision = self._get_latest_revision(url_base)
372         if latest_revision is None or latest_revision <= 0:
373             return None
374         archive_url = ('%s%s/layout-test-results.zip' % (url_base,
375                                                          latest_revision))
376         _log.info('Archive url: "%s"', archive_url)
377         return archive_url
378
379     def _download_buildbot_archive(self):
380         """Download layout test archive file from buildbot.
381
382         Returns:
383           True if download succeeded or
384           False otherwise.
385         """
386
387         url = self._get_archive_url()
388         if url is None:
389             return None
390
391         fn = urllib.urlretrieve(url)[0]
392         _log.info('Archive downloaded and saved to file: "%s"', fn)
393         return fn
394
395     def _extract_and_add_new_baselines(self, archive_file):
396         """Extract new baselines from archive and add them to SVN repository.
397
398         Args:
399           archive_file: full path to the archive file.
400
401         Returns:
402           List of tests that have been rebaselined or
403           None on failure.
404         """
405
406         zip_file = zipfile.ZipFile(archive_file, 'r')
407         zip_namelist = zip_file.namelist()
408
409         _log.debug('zip file namelist:')
410         for name in zip_namelist:
411             _log.debug('  ' + name)
412
413         platform = self._rebaseline_port.test_platform_name_to_name(
414             self._platform)
415         _log.debug('Platform dir: "%s"', platform)
416
417         test_no = 1
418         self._rebaselined_tests = []
419         for test in self._rebaselining_tests:
420             _log.info('Test %d: %s', test_no, test)
421
422             found = False
423             scm_error = False
424             test_basename = os.path.splitext(test)[0]
425             for suffix in BASELINE_SUFFIXES:
426                 archive_test_name = ('layout-test-results/%s-actual%s' %
427                                       (test_basename, suffix))
428                 _log.debug('  Archive test file name: "%s"',
429                            archive_test_name)
430                 if not archive_test_name in zip_namelist:
431                     _log.info('  %s file not in archive.', suffix)
432                     continue
433
434                 found = True
435                 _log.info('  %s file found in archive.', suffix)
436
437                 # Extract new baseline from archive and save it to a temp file.
438                 data = zip_file.read(archive_test_name)
439                 temp_fd, temp_name = tempfile.mkstemp(suffix)
440                 f = os.fdopen(temp_fd, 'wb')
441                 f.write(data)
442                 f.close()
443
444                 expected_filename = '%s-expected%s' % (test_basename, suffix)
445                 expected_fullpath = os.path.join(
446                     self._rebaseline_port.baseline_path(), expected_filename)
447                 expected_fullpath = os.path.normpath(expected_fullpath)
448                 _log.debug('  Expected file full path: "%s"',
449                            expected_fullpath)
450
451                 # TODO(victorw): for now, the rebaselining tool checks whether
452                 # or not THIS baseline is duplicate and should be skipped.
453                 # We could improve the tool to check all baselines in upper
454                 # and lower
455                 # levels and remove all duplicated baselines.
456                 if self._is_dup_baseline(temp_name,
457                                         expected_fullpath,
458                                         test,
459                                         suffix,
460                                         self._platform):
461                     os.remove(temp_name)
462                     self._delete_baseline(expected_fullpath)
463                     continue
464
465                 # Create the new baseline directory if it doesn't already
466                 # exist.
467                 self._port.maybe_make_directory(
468                     os.path.dirname(expected_fullpath))
469
470                 shutil.move(temp_name, expected_fullpath)
471
472                 if 0 != self._scm.add(expected_fullpath, return_exit_code=True):
473                     # FIXME: print detailed diagnose messages
474                     scm_error = True
475                 elif suffix != '.checksum':
476                     self._create_html_baseline_files(expected_fullpath)
477
478             if not found:
479                 _log.warn('  No new baselines found in archive.')
480             else:
481                 if scm_error:
482                     _log.warn('  Failed to add baselines to your repository.')
483                 else:
484                     _log.info('  Rebaseline succeeded.')
485                     self._rebaselined_tests.append(test)
486
487             test_no += 1
488
489         zip_file.close()
490         os.remove(archive_file)
491
492         return self._rebaselined_tests
493
494     def _is_dup_baseline(self, new_baseline, baseline_path, test, suffix,
495                          platform):
496         """Check whether a baseline is duplicate and can fallback to same
497            baseline for another platform. For example, if a test has same
498            baseline on linux and windows, then we only store windows
499            baseline and linux baseline will fallback to the windows version.
500
501         Args:
502           expected_filename: baseline expectation file name.
503           test: test name.
504           suffix: file suffix of the expected results, including dot;
505                   e.g. '.txt' or '.png'.
506           platform: baseline platform 'mac', 'win' or 'linux'.
507
508         Returns:
509           True if the baseline is unnecessary.
510           False otherwise.
511         """
512         test_filepath = os.path.join(self._target_port.layout_tests_dir(),
513                                      test)
514         all_baselines = self._rebaseline_port.expected_baselines(
515             test_filepath, suffix, True)
516         for (fallback_dir, fallback_file) in all_baselines:
517             if fallback_dir and fallback_file:
518                 fallback_fullpath = os.path.normpath(
519                     os.path.join(fallback_dir, fallback_file))
520                 if fallback_fullpath.lower() != baseline_path.lower():
521                     with codecs.open(new_baseline, "r",
522                                      None) as file_handle1:
523                         new_output = file_handle1.read()
524                     with codecs.open(fallback_fullpath, "r",
525                                      None) as file_handle2:
526                         fallback_output = file_handle2.read()
527                     is_image = baseline_path.lower().endswith('.png')
528                     if not self._diff_baselines(new_output, fallback_output,
529                                                 is_image):
530                         _log.info('  Found same baseline at %s',
531                                   fallback_fullpath)
532                         return True
533                     else:
534                         return False
535
536         return False
537
538     def _diff_baselines(self, output1, output2, is_image):
539         """Check whether two baselines are different.
540
541         Args:
542           output1, output2: contents of the baselines to compare.
543
544         Returns:
545           True if two files are different or have different extensions.
546           False otherwise.
547         """
548
549         if is_image:
550             return self._port.diff_image(output1, output2)
551         else:
552             return self._port.compare_text(output1, output2)
553
554     def _delete_baseline(self, filename):
555         """Remove the file from repository and delete it from disk.
556
557         Args:
558           filename: full path of the file to delete.
559         """
560
561         if not filename or not os.path.isfile(filename):
562             return
563         self._scm.delete(filename)
564
565     def _update_rebaselined_tests_in_file(self, backup):
566         """Update the rebaselined tests in test expectations file.
567
568         Args:
569           backup: if True, backup the original test expectations file.
570
571         Returns:
572           no
573         """
574
575         if self._rebaselined_tests:
576             new_expectations = (
577                 self._test_expectations.remove_platform_from_expectations(
578                 self._rebaselined_tests, self._platform))
579             path = self._target_port.path_to_test_expectations_file()
580             if backup:
581                 date_suffix = time.strftime('%Y%m%d%H%M%S',
582                                             time.localtime(time.time()))
583                 backup_file = ('%s.orig.%s' % (path, date_suffix))
584                 if os.path.exists(backup_file):
585                     os.remove(backup_file)
586                 _log.info('Saving original file to "%s"', backup_file)
587                 os.rename(path, backup_file)
588             # FIXME: What encoding are these files?
589             # Or is new_expectations always a byte array?
590             with open(path, "w") as file:
591                 file.write(new_expectations)
592             # self._scm.add(path)
593         else:
594             _log.info('No test was rebaselined so nothing to remove.')
595
596     def _create_html_baseline_files(self, baseline_fullpath):
597         """Create baseline files (old, new and diff) in html directory.
598
599            The files are used to compare the rebaselining results.
600
601         Args:
602           baseline_fullpath: full path of the expected baseline file.
603         """
604
605         if not baseline_fullpath or not os.path.exists(baseline_fullpath):
606             return
607
608         # Copy the new baseline to html directory for result comparison.
609         baseline_filename = os.path.basename(baseline_fullpath)
610         new_file = get_result_file_fullpath(self._options.html_directory,
611                                             baseline_filename, self._platform,
612                                             'new')
613         shutil.copyfile(baseline_fullpath, new_file)
614         _log.info('  Html: copied new baseline file from "%s" to "%s".',
615                   baseline_fullpath, new_file)
616
617         # Get the old baseline from the repository and save to the html directory.
618         try:
619             output = self._scm.show_head(baseline_fullpath)
620         except ScriptError, e:
621             _log.info(e)
622             output = ""
623
624         if (not output) or (output.upper().rstrip().endswith(
625             'NO SUCH FILE OR DIRECTORY')):
626             _log.info('  No base file: "%s"', baseline_fullpath)
627             return
628         base_file = get_result_file_fullpath(self._options.html_directory,
629                                              baseline_filename, self._platform,
630                                              'old')
631         # FIXME: This assumes run_shell returns a byte array.
632         # We should be using an explicit encoding here.
633         with open(base_file, "wb") as file:
634             file.write(output)
635         _log.info('  Html: created old baseline file: "%s".',
636                   base_file)
637
638         # Get the diff between old and new baselines and save to the html dir.
639         if baseline_filename.upper().endswith('.TXT'):
640             output = self._scm.diff_for_file(baseline_fullpath, log=_log)
641             if output:
642                 diff_file = get_result_file_fullpath(
643                     self._options.html_directory, baseline_filename,
644                     self._platform, 'diff')
645                 # FIXME: This assumes run_shell returns a byte array, not unicode()
646                 with open(diff_file, 'wb') as file:
647                     file.write(output)
648                 _log.info('  Html: created baseline diff file: "%s".',
649                           diff_file)
650
651 class HtmlGenerator(object):
652     """Class to generate rebaselining result comparison html."""
653
654     HTML_REBASELINE = ('<html>'
655                        '<head>'
656                        '<style>'
657                        'body {font-family: sans-serif;}'
658                        '.mainTable {background: #666666;}'
659                        '.mainTable td , .mainTable th {background: white;}'
660                        '.detail {margin-left: 10px; margin-top: 3px;}'
661                        '</style>'
662                        '<title>Rebaselining Result Comparison (%(time)s)'
663                        '</title>'
664                        '</head>'
665                        '<body>'
666                        '<h2>Rebaselining Result Comparison (%(time)s)</h2>'
667                        '%(body)s'
668                        '</body>'
669                        '</html>')
670     HTML_NO_REBASELINING_TESTS = (
671         '<p>No tests found that need rebaselining.</p>')
672     HTML_TABLE_TEST = ('<table class="mainTable" cellspacing=1 cellpadding=5>'
673                        '%s</table><br>')
674     HTML_TR_TEST = ('<tr>'
675                     '<th style="background-color: #CDECDE; border-bottom: '
676                     '1px solid black; font-size: 18pt; font-weight: bold" '
677                     'colspan="5">'
678                     '<a href="%s">%s</a>'
679                     '</th>'
680                     '</tr>')
681     HTML_TEST_DETAIL = ('<div class="detail">'
682                         '<tr>'
683                         '<th width="100">Baseline</th>'
684                         '<th width="100">Platform</th>'
685                         '<th width="200">Old</th>'
686                         '<th width="200">New</th>'
687                         '<th width="150">Difference</th>'
688                         '</tr>'
689                         '%s'
690                         '</div>')
691     HTML_TD_NOLINK = '<td align=center><a>%s</a></td>'
692     HTML_TD_LINK = '<td align=center><a href="%(uri)s">%(name)s</a></td>'
693     HTML_TD_LINK_IMG = ('<td><a href="%(uri)s">'
694                         '<img style="width: 200" src="%(uri)s" /></a></td>')
695     HTML_TR = '<tr>%s</tr>'
696
697     def __init__(self, target_port, options, platforms, rebaselining_tests):
698         self._html_directory = options.html_directory
699         self._target_port = target_port
700         self._platforms = platforms
701         self._rebaselining_tests = rebaselining_tests
702         self._html_file = os.path.join(options.html_directory,
703                                        'rebaseline.html')
704
705     def generate_html(self):
706         """Generate html file for rebaselining result comparison."""
707
708         _log.info('Generating html file')
709
710         html_body = ''
711         if not self._rebaselining_tests:
712             html_body += self.HTML_NO_REBASELINING_TESTS
713         else:
714             tests = list(self._rebaselining_tests)
715             tests.sort()
716
717             test_no = 1
718             for test in tests:
719                 _log.info('Test %d: %s', test_no, test)
720                 html_body += self._generate_html_for_one_test(test)
721
722         html = self.HTML_REBASELINE % ({'time': time.asctime(),
723                                         'body': html_body})
724         _log.debug(html)
725
726         with codecs.open(self._html_file, "w", "utf-8") as file:
727             file.write(html)
728
729         _log.info('Baseline comparison html generated at "%s"',
730                   self._html_file)
731
732     def show_html(self):
733         """Launch the rebaselining html in brwoser."""
734
735         _log.info('Launching html: "%s"', self._html_file)
736         user.User().open_url(self._html_file)
737         _log.info('Html launched.')
738
739     def _generate_baseline_links(self, test_basename, suffix, platform):
740         """Generate links for baseline results (old, new and diff).
741
742         Args:
743           test_basename: base filename of the test
744           suffix: baseline file suffixes: '.txt', '.png'
745           platform: win, linux or mac
746
747         Returns:
748           html links for showing baseline results (old, new and diff)
749         """
750
751         baseline_filename = '%s-expected%s' % (test_basename, suffix)
752         _log.debug('    baseline filename: "%s"', baseline_filename)
753
754         new_file = get_result_file_fullpath(self._html_directory,
755                                             baseline_filename, platform, 'new')
756         _log.info('    New baseline file: "%s"', new_file)
757         if not os.path.exists(new_file):
758             _log.info('    No new baseline file: "%s"', new_file)
759             return ''
760
761         old_file = get_result_file_fullpath(self._html_directory,
762                                             baseline_filename, platform, 'old')
763         _log.info('    Old baseline file: "%s"', old_file)
764         if suffix == '.png':
765             html_td_link = self.HTML_TD_LINK_IMG
766         else:
767             html_td_link = self.HTML_TD_LINK
768
769         links = ''
770         if os.path.exists(old_file):
771             links += html_td_link % {
772                 'uri': self._target_port.filename_to_uri(old_file),
773                 'name': baseline_filename}
774         else:
775             _log.info('    No old baseline file: "%s"', old_file)
776             links += self.HTML_TD_NOLINK % ''
777
778         links += html_td_link % {'uri': self._target_port.filename_to_uri(
779                                      new_file),
780                                  'name': baseline_filename}
781
782         diff_file = get_result_file_fullpath(self._html_directory,
783                                              baseline_filename, platform,
784                                              'diff')
785         _log.info('    Baseline diff file: "%s"', diff_file)
786         if os.path.exists(diff_file):
787             links += html_td_link % {'uri': self._target_port.filename_to_uri(
788                 diff_file), 'name': 'Diff'}
789         else:
790             _log.info('    No baseline diff file: "%s"', diff_file)
791             links += self.HTML_TD_NOLINK % ''
792
793         return links
794
795     def _generate_html_for_one_test(self, test):
796         """Generate html for one rebaselining test.
797
798         Args:
799           test: layout test name
800
801         Returns:
802           html that compares baseline results for the test.
803         """
804
805         test_basename = os.path.basename(os.path.splitext(test)[0])
806         _log.info('  basename: "%s"', test_basename)
807         rows = []
808         for suffix in BASELINE_SUFFIXES:
809             if suffix == '.checksum':
810                 continue
811
812             _log.info('  Checking %s files', suffix)
813             for platform in self._platforms:
814                 links = self._generate_baseline_links(test_basename, suffix,
815                     platform)
816                 if links:
817                     row = self.HTML_TD_NOLINK % self._get_baseline_result_type(
818                         suffix)
819                     row += self.HTML_TD_NOLINK % platform
820                     row += links
821                     _log.debug('    html row: %s', row)
822
823                     rows.append(self.HTML_TR % row)
824
825         if rows:
826             test_path = os.path.join(self._target_port.layout_tests_dir(),
827                                      test)
828             html = self.HTML_TR_TEST % (
829                 self._target_port.filename_to_uri(test_path), test)
830             html += self.HTML_TEST_DETAIL % ' '.join(rows)
831
832             _log.debug('    html for test: %s', html)
833             return self.HTML_TABLE_TEST % html
834
835         return ''
836
837     def _get_baseline_result_type(self, suffix):
838         """Name of the baseline result type."""
839
840         if suffix == '.png':
841             return 'Pixel'
842         elif suffix == '.txt':
843             return 'Render Tree'
844         else:
845             return 'Other'
846
847
848 def get_host_port_object(options):
849     """Return a port object for the platform we're running on."""
850     # The only thing we really need on the host is a way to diff
851     # text files and image files, which means we need to check that some
852     # version of ImageDiff has been built. We will look for either Debug
853     # or Release versions of the default port on the platform.
854     options.configuration = "Release"
855     port_obj = port.get(None, options)
856     if not port_obj.check_image_diff(override_step=None, logging=False):
857         _log.debug('No release version of the image diff binary was found.')
858         options.configuration = "Debug"
859         port_obj = port.get(None, options)
860         if not port_obj.check_image_diff(override_step=None, logging=False):
861             _log.error('No version of image diff was found. Check your build.')
862             return None
863         else:
864             _log.debug('Found the debug version of the image diff binary.')
865     else:
866         _log.debug('Found the release version of the image diff binary.')
867     return port_obj
868
869
870 def main():
871     """Main function to produce new baselines."""
872
873     option_parser = optparse.OptionParser()
874     option_parser.add_option('-v', '--verbose',
875                              action='store_true',
876                              default=False,
877                              help='include debug-level logging.')
878
879     option_parser.add_option('-q', '--quiet',
880                              action='store_true',
881                              help='Suppress result HTML viewing')
882
883     option_parser.add_option('-p', '--platforms',
884                              default='mac,win,win-xp,win-vista,linux',
885                              help=('Comma delimited list of platforms '
886                                    'that need rebaselining.'))
887
888     option_parser.add_option('-u', '--archive_url',
889                              default=('http://build.chromium.org/buildbot/'
890                                       'layout_test_results'),
891                              help=('Url to find the layout test result archive'
892                                    ' file.'))
893     option_parser.add_option('-U', '--force_archive_url',
894                              help=('Url of result zip file. This option is for debugging '
895                                    'purposes'))
896
897     option_parser.add_option('-w', '--webkit_canary',
898                              action='store_true',
899                              default=False,
900                              help=('If True, pull baselines from webkit.org '
901                                    'canary bot.'))
902
903     option_parser.add_option('-b', '--backup',
904                              action='store_true',
905                              default=False,
906                              help=('Whether or not to backup the original test'
907                                    ' expectations file after rebaseline.'))
908
909     option_parser.add_option('-d', '--html_directory',
910                              default='',
911                              help=('The directory that stores the results for '
912                                    'rebaselining comparison.'))
913
914     option_parser.add_option('', '--use_drt',
915                              action='store_true',
916                              default=False,
917                              help=('Use ImageDiff from DumpRenderTree instead '
918                                    'of image_diff for pixel tests.'))
919
920     option_parser.add_option('', '--target-platform',
921                              default='chromium',
922                              help=('The target platform to rebaseline '
923                                    '("mac", "chromium", "qt", etc.). Defaults '
924                                    'to "chromium".'))
925     options = option_parser.parse_args()[0]
926
927     # We need to create three different Port objects over the life of this
928     # script. |target_port_obj| is used to determine configuration information:
929     # location of the expectations file, names of ports to rebaseline, etc.
930     # |port_obj| is used for runtime functionality like actually diffing
931     # Then we create a rebaselining port to actual find and manage the
932     # baselines.
933     target_options = copy.copy(options)
934     if options.target_platform == 'chromium':
935         target_options.chromium = True
936     target_port_obj = port.get(None, target_options)
937
938     # Set up our logging format.
939     log_level = logging.INFO
940     if options.verbose:
941         log_level = logging.DEBUG
942     logging.basicConfig(level=log_level,
943                         format=('%(asctime)s %(filename)s:%(lineno)-3d '
944                                 '%(levelname)s %(message)s'),
945                         datefmt='%y%m%d %H:%M:%S')
946
947     host_port_obj = get_host_port_object(options)
948     if not host_port_obj:
949         sys.exit(1)
950
951     # Verify 'platforms' option is valid.
952     if not options.platforms:
953         _log.error('Invalid "platforms" option. --platforms must be '
954                    'specified in order to rebaseline.')
955         sys.exit(1)
956     platforms = [p.strip().lower() for p in options.platforms.split(',')]
957     for platform in platforms:
958         if not platform in REBASELINE_PLATFORM_ORDER:
959             _log.error('Invalid platform: "%s"' % (platform))
960             sys.exit(1)
961
962     # Adjust the platform order so rebaseline tool is running at the order of
963     # 'mac', 'win' and 'linux'. This is in same order with layout test baseline
964     # search paths. It simplifies how the rebaseline tool detects duplicate
965     # baselines. Check _IsDupBaseline method for details.
966     rebaseline_platforms = []
967     for platform in REBASELINE_PLATFORM_ORDER:
968         if platform in platforms:
969             rebaseline_platforms.append(platform)
970
971     options.html_directory = setup_html_directory(options.html_directory)
972
973     rebaselining_tests = set()
974     backup = options.backup
975     for platform in rebaseline_platforms:
976         rebaseliner = Rebaseliner(host_port_obj, target_port_obj,
977                                   platform, options)
978
979         _log.info('')
980         log_dashed_string('Rebaseline started', platform)
981         if rebaseliner.run(backup):
982             # Only need to backup one original copy of test expectation file.
983             backup = False
984             log_dashed_string('Rebaseline done', platform)
985         else:
986             log_dashed_string('Rebaseline failed', platform, logging.ERROR)
987
988         rebaselining_tests |= set(rebaseliner.get_rebaselining_tests())
989
990     _log.info('')
991     log_dashed_string('Rebaselining result comparison started', None)
992     html_generator = HtmlGenerator(target_port_obj,
993                                    options,
994                                    rebaseline_platforms,
995                                    rebaselining_tests)
996     html_generator.generate_html()
997     if not options.quiet:
998         html_generator.show_html()
999     log_dashed_string('Rebaselining result comparison done', None)
1000
1001     sys.exit(0)
1002
1003 if '__main__' == __name__:
1004     main()