92f103258246268e54cfa0251777e6db4f04039e
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / rebaseline_chromium_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Rebaselining tool that automatically produces baselines for all platforms.
31
32 The script does the following for each platform specified:
33   1. Compile a list of tests that need rebaselining.
34   2. Download test result archive from buildbot for the platform.
35   3. Extract baselines from the archive file for all identified files.
36   4. Add new baselines to SVN repository.
37   5. For each test that has been rebaselined, remove this platform option from
38      the test in test_expectation.txt. If no other platforms remain after
39      removal, delete the rebaselined test from the file.
40
41 At the end, the script generates a html that compares old and new baselines.
42 """
43
44 from __future__ import with_statement
45
46 import codecs
47 import copy
48 import logging
49 import optparse
50 import os
51 import re
52 import shutil
53 import subprocess
54 import sys
55 import tempfile
56 import time
57 import urllib
58 import webbrowser
59 import zipfile
60
61 from webkitpy.common.system.executive import run_command, ScriptError
62 import webkitpy.common.checkout.scm as scm
63
64 import port
65 from layout_package import test_expectations
66 from test_types import image_diff
67 from test_types import text_diff
68
69 _log = logging.getLogger("webkitpy.layout_tests."
70                          "rebaseline_chromium_webkit_tests")
71
72 BASELINE_SUFFIXES = ['.txt', '.png', '.checksum']
73 REBASELINE_PLATFORM_ORDER = ['mac', 'win', 'win-xp', 'win-vista', 'linux']
74 ARCHIVE_DIR_NAME_DICT = {'win': 'webkit-rel',
75                          'win-vista': 'webkit-dbg-vista',
76                          'win-xp': 'webkit-rel',
77                          'mac': 'webkit-rel-mac5',
78                          'linux': 'webkit-rel-linux',
79                          'win-canary': 'webkit-rel-webkit-org',
80                          'win-vista-canary': 'webkit-dbg-vista',
81                          'win-xp-canary': 'webkit-rel-webkit-org',
82                          'mac-canary': 'webkit-rel-mac-webkit-org',
83                          'linux-canary': 'webkit-rel-linux-webkit-org'}
84
85
86 # FIXME: Should be rolled into webkitpy.Executive
87 def run_shell_with_return_code(command, print_output=False):
88     """Executes a command and returns the output and process return code.
89
90     Args:
91       command: program and arguments.
92       print_output: if true, print the command results to standard output.
93
94     Returns:
95       command output, return code
96     """
97
98     # Use a shell for subcommands on Windows to get a PATH search.
99     # FIXME: shell=True is a trail of tears, and should be removed.
100     use_shell = sys.platform.startswith('win')
101     # Note: Not thread safe: http://bugs.python.org/issue2320
102     p = subprocess.Popen(command, stdout=subprocess.PIPE,
103                          stderr=subprocess.STDOUT, shell=use_shell)
104     if print_output:
105         output_array = []
106         while True:
107             line = p.stdout.readline()
108             if not line:
109                 break
110             if print_output:
111                 print line.strip('\n')
112             output_array.append(line)
113         output = ''.join(output_array)
114     else:
115         output = p.stdout.read()
116     p.wait()
117     p.stdout.close()
118
119     return output, p.returncode
120
121
122 # FIXME: Should be rolled into webkitpy.Executive
123 def run_shell(command, print_output=False):
124     """Executes a command and returns the output.
125
126     Args:
127       command: program and arguments.
128       print_output: if true, print the command results to standard output.
129
130     Returns:
131       command output
132     """
133
134     output, return_code = run_shell_with_return_code(command, print_output)
135     return output
136
137
138 def log_dashed_string(text, platform, logging_level=logging.INFO):
139     """Log text message with dashes on both sides."""
140
141     msg = text
142     if platform:
143         msg += ': ' + platform
144     if len(msg) < 78:
145         dashes = '-' * ((78 - len(msg)) / 2)
146         msg = '%s %s %s' % (dashes, msg, dashes)
147
148     if logging_level == logging.ERROR:
149         _log.error(msg)
150     elif logging_level == logging.WARNING:
151         _log.warn(msg)
152     else:
153         _log.info(msg)
154
155
156 def setup_html_directory(html_directory):
157     """Setup the directory to store html results.
158
159        All html related files are stored in the "rebaseline_html" subdirectory.
160
161     Args:
162       html_directory: parent directory that stores the rebaselining results.
163                       If None, a temp directory is created.
164
165     Returns:
166       the directory that stores the html related rebaselining results.
167     """
168
169     if not html_directory:
170         html_directory = tempfile.mkdtemp()
171     elif not os.path.exists(html_directory):
172         os.mkdir(html_directory)
173
174     html_directory = os.path.join(html_directory, 'rebaseline_html')
175     _log.info('Html directory: "%s"', html_directory)
176
177     if os.path.exists(html_directory):
178         shutil.rmtree(html_directory, True)
179         _log.info('Deleted file at html directory: "%s"', html_directory)
180
181     if not os.path.exists(html_directory):
182         os.mkdir(html_directory)
183     return html_directory
184
185
186 def get_result_file_fullpath(html_directory, baseline_filename, platform,
187                              result_type):
188     """Get full path of the baseline result file.
189
190     Args:
191       html_directory: directory that stores the html related files.
192       baseline_filename: name of the baseline file.
193       platform: win, linux or mac
194       result_type: type of the baseline result: '.txt', '.png'.
195
196     Returns:
197       Full path of the baseline file for rebaselining result comparison.
198     """
199
200     base, ext = os.path.splitext(baseline_filename)
201     result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext)
202     fullpath = os.path.join(html_directory, result_filename)
203     _log.debug('  Result file full path: "%s".', fullpath)
204     return fullpath
205
206
207 class Rebaseliner(object):
208     """Class to produce new baselines for a given platform."""
209
210     REVISION_REGEX = r'<a href=\"(\d+)/\">'
211
212     def __init__(self, running_port, target_port, platform, options):
213         """
214         Args:
215             running_port: the Port the script is running on.
216             target_port: the Port the script uses to find port-specific
217                 configuration information like the test_expectations.txt
218                 file location and the list of test platforms.
219             platform: the test platform to rebaseline
220             options: the command-line options object."""
221         self._platform = platform
222         self._options = options
223         self._port = running_port
224         self._target_port = target_port
225         self._rebaseline_port = port.get(
226             self._target_port.test_platform_name_to_name(platform), options)
227         self._rebaselining_tests = []
228         self._rebaselined_tests = []
229
230         # Create tests and expectations helper which is used to:
231         #   -. compile list of tests that need rebaselining.
232         #   -. update the tests in test_expectations file after rebaseline
233         #      is done.
234         expectations_str = self._rebaseline_port.test_expectations()
235         self._test_expectations = \
236             test_expectations.TestExpectations(self._rebaseline_port,
237                                                None,
238                                                expectations_str,
239                                                self._platform,
240                                                False,
241                                                False)
242         self._scm = scm.default_scm()
243
244     def run(self, backup):
245         """Run rebaseline process."""
246
247         log_dashed_string('Compiling rebaselining tests', self._platform)
248         if not self._compile_rebaselining_tests():
249             return True
250
251         log_dashed_string('Downloading archive', self._platform)
252         archive_file = self._download_buildbot_archive()
253         _log.info('')
254         if not archive_file:
255             _log.error('No archive found.')
256             return False
257
258         log_dashed_string('Extracting and adding new baselines',
259                           self._platform)
260         if not self._extract_and_add_new_baselines(archive_file):
261             return False
262
263         log_dashed_string('Updating rebaselined tests in file',
264                           self._platform)
265         self._update_rebaselined_tests_in_file(backup)
266         _log.info('')
267
268         if len(self._rebaselining_tests) != len(self._rebaselined_tests):
269             _log.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN '
270                          'REBASELINED.')
271             _log.warning('  Total tests needing rebaselining: %d',
272                          len(self._rebaselining_tests))
273             _log.warning('  Total tests rebaselined: %d',
274                          len(self._rebaselined_tests))
275             return False
276
277         _log.warning('All tests needing rebaselining were successfully '
278                      'rebaselined.')
279
280         return True
281
282     def get_rebaselining_tests(self):
283         return self._rebaselining_tests
284
285     def _compile_rebaselining_tests(self):
286         """Compile list of tests that need rebaselining for the platform.
287
288         Returns:
289           List of tests that need rebaselining or
290           None if there is no such test.
291         """
292
293         self._rebaselining_tests = \
294             self._test_expectations.get_rebaselining_failures()
295         if not self._rebaselining_tests:
296             _log.warn('No tests found that need rebaselining.')
297             return None
298
299         _log.info('Total number of tests needing rebaselining '
300                   'for "%s": "%d"', self._platform,
301                   len(self._rebaselining_tests))
302
303         test_no = 1
304         for test in self._rebaselining_tests:
305             _log.info('  %d: %s', test_no, test)
306             test_no += 1
307
308         return self._rebaselining_tests
309
310     def _get_latest_revision(self, url):
311         """Get the latest layout test revision number from buildbot.
312
313         Args:
314           url: Url to retrieve layout test revision numbers.
315
316         Returns:
317           latest revision or
318           None on failure.
319         """
320
321         _log.debug('Url to retrieve revision: "%s"', url)
322
323         f = urllib.urlopen(url)
324         content = f.read()
325         f.close()
326
327         revisions = re.findall(self.REVISION_REGEX, content)
328         if not revisions:
329             _log.error('Failed to find revision, content: "%s"', content)
330             return None
331
332         revisions.sort(key=int)
333         _log.info('Latest revision: "%s"', revisions[len(revisions) - 1])
334         return revisions[len(revisions) - 1]
335
336     def _get_archive_dir_name(self, platform, webkit_canary):
337         """Get name of the layout test archive directory.
338
339         Returns:
340           Directory name or
341           None on failure
342         """
343
344         if webkit_canary:
345             platform += '-canary'
346
347         if platform in ARCHIVE_DIR_NAME_DICT:
348             return ARCHIVE_DIR_NAME_DICT[platform]
349         else:
350             _log.error('Cannot find platform key %s in archive '
351                        'directory name dictionary', platform)
352             return None
353
354     def _get_archive_url(self):
355         """Generate the url to download latest layout test archive.
356
357         Returns:
358           Url to download archive or
359           None on failure
360         """
361
362         if self._options.force_archive_url:
363             return self._options.force_archive_url
364
365         dir_name = self._get_archive_dir_name(self._platform,
366                                               self._options.webkit_canary)
367         if not dir_name:
368             return None
369
370         _log.debug('Buildbot platform dir name: "%s"', dir_name)
371
372         url_base = '%s/%s/' % (self._options.archive_url, dir_name)
373         latest_revision = self._get_latest_revision(url_base)
374         if latest_revision is None or latest_revision <= 0:
375             return None
376         archive_url = ('%s%s/layout-test-results.zip' % (url_base,
377                                                          latest_revision))
378         _log.info('Archive url: "%s"', archive_url)
379         return archive_url
380
381     def _download_buildbot_archive(self):
382         """Download layout test archive file from buildbot.
383
384         Returns:
385           True if download succeeded or
386           False otherwise.
387         """
388
389         url = self._get_archive_url()
390         if url is None:
391             return None
392
393         fn = urllib.urlretrieve(url)[0]
394         _log.info('Archive downloaded and saved to file: "%s"', fn)
395         return fn
396
397     def _extract_and_add_new_baselines(self, archive_file):
398         """Extract new baselines from archive and add them to SVN repository.
399
400         Args:
401           archive_file: full path to the archive file.
402
403         Returns:
404           List of tests that have been rebaselined or
405           None on failure.
406         """
407
408         zip_file = zipfile.ZipFile(archive_file, 'r')
409         zip_namelist = zip_file.namelist()
410
411         _log.debug('zip file namelist:')
412         for name in zip_namelist:
413             _log.debug('  ' + name)
414
415         platform = self._rebaseline_port.test_platform_name_to_name(
416             self._platform)
417         _log.debug('Platform dir: "%s"', platform)
418
419         test_no = 1
420         self._rebaselined_tests = []
421         for test in self._rebaselining_tests:
422             _log.info('Test %d: %s', test_no, test)
423
424             found = False
425             scm_error = False
426             test_basename = os.path.splitext(test)[0]
427             for suffix in BASELINE_SUFFIXES:
428                 archive_test_name = ('layout-test-results/%s-actual%s' %
429                                       (test_basename, suffix))
430                 _log.debug('  Archive test file name: "%s"',
431                            archive_test_name)
432                 if not archive_test_name in zip_namelist:
433                     _log.info('  %s file not in archive.', suffix)
434                     continue
435
436                 found = True
437                 _log.info('  %s file found in archive.', suffix)
438
439                 # Extract new baseline from archive and save it to a temp file.
440                 data = zip_file.read(archive_test_name)
441                 temp_fd, temp_name = tempfile.mkstemp(suffix)
442                 f = os.fdopen(temp_fd, 'wb')
443                 f.write(data)
444                 f.close()
445
446                 expected_filename = '%s-expected%s' % (test_basename, suffix)
447                 expected_fullpath = os.path.join(
448                     self._rebaseline_port.baseline_path(), expected_filename)
449                 expected_fullpath = os.path.normpath(expected_fullpath)
450                 _log.debug('  Expected file full path: "%s"',
451                            expected_fullpath)
452
453                 # TODO(victorw): for now, the rebaselining tool checks whether
454                 # or not THIS baseline is duplicate and should be skipped.
455                 # We could improve the tool to check all baselines in upper
456                 # and lower
457                 # levels and remove all duplicated baselines.
458                 if self._is_dup_baseline(temp_name,
459                                         expected_fullpath,
460                                         test,
461                                         suffix,
462                                         self._platform):
463                     os.remove(temp_name)
464                     self._delete_baseline(expected_fullpath)
465                     continue
466
467                 # Create the new baseline directory if it doesn't already
468                 # exist.
469                 self._port.maybe_make_directory(
470                     os.path.dirname(expected_fullpath))
471
472                 shutil.move(temp_name, expected_fullpath)
473
474                 if 0 != self._scm.add(expected_fullpath, return_exit_code=True):
475                     # FIXME: print detailed diagnose messages
476                     scm_error = True
477                 elif suffix != '.checksum':
478                     self._create_html_baseline_files(expected_fullpath)
479
480             if not found:
481                 _log.warn('  No new baselines found in archive.')
482             else:
483                 if scm_error:
484                     _log.warn('  Failed to add baselines to your repository.')
485                 else:
486                     _log.info('  Rebaseline succeeded.')
487                     self._rebaselined_tests.append(test)
488
489             test_no += 1
490
491         zip_file.close()
492         os.remove(archive_file)
493
494         return self._rebaselined_tests
495
496     def _is_dup_baseline(self, new_baseline, baseline_path, test, suffix,
497                          platform):
498         """Check whether a baseline is duplicate and can fallback to same
499            baseline for another platform. For example, if a test has same
500            baseline on linux and windows, then we only store windows
501            baseline and linux baseline will fallback to the windows version.
502
503         Args:
504           expected_filename: baseline expectation file name.
505           test: test name.
506           suffix: file suffix of the expected results, including dot;
507                   e.g. '.txt' or '.png'.
508           platform: baseline platform 'mac', 'win' or 'linux'.
509
510         Returns:
511           True if the baseline is unnecessary.
512           False otherwise.
513         """
514         test_filepath = os.path.join(self._target_port.layout_tests_dir(),
515                                      test)
516         all_baselines = self._rebaseline_port.expected_baselines(
517             test_filepath, suffix, True)
518         for (fallback_dir, fallback_file) in all_baselines:
519             if fallback_dir and fallback_file:
520                 fallback_fullpath = os.path.normpath(
521                     os.path.join(fallback_dir, fallback_file))
522                 if fallback_fullpath.lower() != baseline_path.lower():
523                     if not self._diff_baselines(new_baseline,
524                                                 fallback_fullpath):
525                         _log.info('  Found same baseline at %s',
526                                   fallback_fullpath)
527                         return True
528                     else:
529                         return False
530
531         return False
532
533     def _diff_baselines(self, file1, file2):
534         """Check whether two baselines are different.
535
536         Args:
537           file1, file2: full paths of the baselines to compare.
538
539         Returns:
540           True if two files are different or have different extensions.
541           False otherwise.
542         """
543
544         ext1 = os.path.splitext(file1)[1].upper()
545         ext2 = os.path.splitext(file2)[1].upper()
546         if ext1 != ext2:
547             _log.warn('Files to compare have different ext. '
548                       'File1: %s; File2: %s', file1, file2)
549             return True
550
551         if ext1 == '.PNG':
552             return image_diff.ImageDiff(self._port,
553                '').diff_files(self._port, file1, file2)
554         else:
555             return text_diff.TestTextDiff(self._port,
556                 '').diff_files(self._port, file1, file2)
557
558     def _delete_baseline(self, filename):
559         """Remove the file from repository and delete it from disk.
560
561         Args:
562           filename: full path of the file to delete.
563         """
564
565         if not filename or not os.path.isfile(filename):
566             return
567         self._scm.delete(filename)
568
569     def _update_rebaselined_tests_in_file(self, backup):
570         """Update the rebaselined tests in test expectations file.
571
572         Args:
573           backup: if True, backup the original test expectations file.
574
575         Returns:
576           no
577         """
578
579         if self._rebaselined_tests:
580             new_expectations = (
581                 self._test_expectations.remove_platform_from_expectations(
582                 self._rebaselined_tests, self._platform))
583             path = self._target_port.path_to_test_expectations_file()
584             if backup:
585                 date_suffix = time.strftime('%Y%m%d%H%M%S',
586                                             time.localtime(time.time()))
587                 backup_file = ('%s.orig.%s' % (path, date_suffix))
588                 if os.path.exists(backup_file):
589                     os.remove(backup_file)
590                 _log.info('Saving original file to "%s"', backup_file)
591                 os.rename(path, backup_file)
592             # FIXME: What encoding are these files?
593             # Or is new_expectations always a byte array?
594             with open(path, "w") as file:
595                 file.write(new_expectations)
596             self._scm.add(path)
597         else:
598             _log.info('No test was rebaselined so nothing to remove.')
599
600     def _create_html_baseline_files(self, baseline_fullpath):
601         """Create baseline files (old, new and diff) in html directory.
602
603            The files are used to compare the rebaselining results.
604
605         Args:
606           baseline_fullpath: full path of the expected baseline file.
607         """
608
609         if not baseline_fullpath or not os.path.exists(baseline_fullpath):
610             return
611
612         # Copy the new baseline to html directory for result comparison.
613         baseline_filename = os.path.basename(baseline_fullpath)
614         new_file = get_result_file_fullpath(self._options.html_directory,
615                                             baseline_filename, self._platform,
616                                             'new')
617         shutil.copyfile(baseline_fullpath, new_file)
618         _log.info('  Html: copied new baseline file from "%s" to "%s".',
619                   baseline_fullpath, new_file)
620
621         # Get the old baseline from the repository and save to the html directory.
622         try:
623             output = self._scm.show_head(baseline_fullpath)
624         except ScriptError, e:
625             _log.info(e)
626             output = ""
627
628         if (not output) or (output.upper().rstrip().endswith(
629             'NO SUCH FILE OR DIRECTORY')):
630             _log.info('  No base file: "%s"', baseline_fullpath)
631             return
632         base_file = get_result_file_fullpath(self._options.html_directory,
633                                              baseline_filename, self._platform,
634                                              'old')
635         # FIXME: This assumes run_shell returns a byte array.
636         # We should be using an explicit encoding here.
637         with open(base_file, "wb") as file:
638             file.write(output)
639         _log.info('  Html: created old baseline file: "%s".',
640                   base_file)
641
642         # Get the diff between old and new baselines and save to the html dir.
643         if baseline_filename.upper().endswith('.TXT'):
644             output = self._scm.diff_for_file(baseline_fullpath, log=_log)
645             if output:
646                 diff_file = get_result_file_fullpath(
647                     self._options.html_directory, baseline_filename,
648                     self._platform, 'diff')
649                 # FIXME: This assumes run_shell returns a byte array, not unicode()
650                 with open(diff_file, 'wb') as file:
651                     file.write(output)
652                 _log.info('  Html: created baseline diff file: "%s".',
653                           diff_file)
654
655 class HtmlGenerator(object):
656     """Class to generate rebaselining result comparison html."""
657
658     HTML_REBASELINE = ('<html>'
659                        '<head>'
660                        '<style>'
661                        'body {font-family: sans-serif;}'
662                        '.mainTable {background: #666666;}'
663                        '.mainTable td , .mainTable th {background: white;}'
664                        '.detail {margin-left: 10px; margin-top: 3px;}'
665                        '</style>'
666                        '<title>Rebaselining Result Comparison (%(time)s)'
667                        '</title>'
668                        '</head>'
669                        '<body>'
670                        '<h2>Rebaselining Result Comparison (%(time)s)</h2>'
671                        '%(body)s'
672                        '</body>'
673                        '</html>')
674     HTML_NO_REBASELINING_TESTS = (
675         '<p>No tests found that need rebaselining.</p>')
676     HTML_TABLE_TEST = ('<table class="mainTable" cellspacing=1 cellpadding=5>'
677                        '%s</table><br>')
678     HTML_TR_TEST = ('<tr>'
679                     '<th style="background-color: #CDECDE; border-bottom: '
680                     '1px solid black; font-size: 18pt; font-weight: bold" '
681                     'colspan="5">'
682                     '<a href="%s">%s</a>'
683                     '</th>'
684                     '</tr>')
685     HTML_TEST_DETAIL = ('<div class="detail">'
686                         '<tr>'
687                         '<th width="100">Baseline</th>'
688                         '<th width="100">Platform</th>'
689                         '<th width="200">Old</th>'
690                         '<th width="200">New</th>'
691                         '<th width="150">Difference</th>'
692                         '</tr>'
693                         '%s'
694                         '</div>')
695     HTML_TD_NOLINK = '<td align=center><a>%s</a></td>'
696     HTML_TD_LINK = '<td align=center><a href="%(uri)s">%(name)s</a></td>'
697     HTML_TD_LINK_IMG = ('<td><a href="%(uri)s">'
698                         '<img style="width: 200" src="%(uri)s" /></a></td>')
699     HTML_TR = '<tr>%s</tr>'
700
701     def __init__(self, target_port, options, platforms, rebaselining_tests):
702         self._html_directory = options.html_directory
703         self._target_port = target_port
704         self._platforms = platforms
705         self._rebaselining_tests = rebaselining_tests
706         self._html_file = os.path.join(options.html_directory,
707                                        'rebaseline.html')
708
709     def generate_html(self):
710         """Generate html file for rebaselining result comparison."""
711
712         _log.info('Generating html file')
713
714         html_body = ''
715         if not self._rebaselining_tests:
716             html_body += self.HTML_NO_REBASELINING_TESTS
717         else:
718             tests = list(self._rebaselining_tests)
719             tests.sort()
720
721             test_no = 1
722             for test in tests:
723                 _log.info('Test %d: %s', test_no, test)
724                 html_body += self._generate_html_for_one_test(test)
725
726         html = self.HTML_REBASELINE % ({'time': time.asctime(),
727                                         'body': html_body})
728         _log.debug(html)
729
730         with codecs.open(self._html_file, "w", "utf-8") as file:
731             file.write(html)
732
733         _log.info('Baseline comparison html generated at "%s"',
734                   self._html_file)
735
736     def show_html(self):
737         """Launch the rebaselining html in brwoser."""
738
739         _log.info('Launching html: "%s"', self._html_file)
740
741         html_uri = self._target_port.filename_to_uri(self._html_file)
742         webbrowser.open(html_uri, 1)
743
744         _log.info('Html launched.')
745
746     def _generate_baseline_links(self, test_basename, suffix, platform):
747         """Generate links for baseline results (old, new and diff).
748
749         Args:
750           test_basename: base filename of the test
751           suffix: baseline file suffixes: '.txt', '.png'
752           platform: win, linux or mac
753
754         Returns:
755           html links for showing baseline results (old, new and diff)
756         """
757
758         baseline_filename = '%s-expected%s' % (test_basename, suffix)
759         _log.debug('    baseline filename: "%s"', baseline_filename)
760
761         new_file = get_result_file_fullpath(self._html_directory,
762                                             baseline_filename, platform, 'new')
763         _log.info('    New baseline file: "%s"', new_file)
764         if not os.path.exists(new_file):
765             _log.info('    No new baseline file: "%s"', new_file)
766             return ''
767
768         old_file = get_result_file_fullpath(self._html_directory,
769                                             baseline_filename, platform, 'old')
770         _log.info('    Old baseline file: "%s"', old_file)
771         if suffix == '.png':
772             html_td_link = self.HTML_TD_LINK_IMG
773         else:
774             html_td_link = self.HTML_TD_LINK
775
776         links = ''
777         if os.path.exists(old_file):
778             links += html_td_link % {
779                 'uri': self._target_port.filename_to_uri(old_file),
780                 'name': baseline_filename}
781         else:
782             _log.info('    No old baseline file: "%s"', old_file)
783             links += self.HTML_TD_NOLINK % ''
784
785         links += html_td_link % {'uri': self._target_port.filename_to_uri(
786                                      new_file),
787                                  'name': baseline_filename}
788
789         diff_file = get_result_file_fullpath(self._html_directory,
790                                              baseline_filename, platform,
791                                              'diff')
792         _log.info('    Baseline diff file: "%s"', diff_file)
793         if os.path.exists(diff_file):
794             links += html_td_link % {'uri': self._target_port.filename_to_uri(
795                 diff_file), 'name': 'Diff'}
796         else:
797             _log.info('    No baseline diff file: "%s"', diff_file)
798             links += self.HTML_TD_NOLINK % ''
799
800         return links
801
802     def _generate_html_for_one_test(self, test):
803         """Generate html for one rebaselining test.
804
805         Args:
806           test: layout test name
807
808         Returns:
809           html that compares baseline results for the test.
810         """
811
812         test_basename = os.path.basename(os.path.splitext(test)[0])
813         _log.info('  basename: "%s"', test_basename)
814         rows = []
815         for suffix in BASELINE_SUFFIXES:
816             if suffix == '.checksum':
817                 continue
818
819             _log.info('  Checking %s files', suffix)
820             for platform in self._platforms:
821                 links = self._generate_baseline_links(test_basename, suffix,
822                     platform)
823                 if links:
824                     row = self.HTML_TD_NOLINK % self._get_baseline_result_type(
825                         suffix)
826                     row += self.HTML_TD_NOLINK % platform
827                     row += links
828                     _log.debug('    html row: %s', row)
829
830                     rows.append(self.HTML_TR % row)
831
832         if rows:
833             test_path = os.path.join(self._target_port.layout_tests_dir(),
834                                      test)
835             html = self.HTML_TR_TEST % (
836                 self._target_port.filename_to_uri(test_path), test)
837             html += self.HTML_TEST_DETAIL % ' '.join(rows)
838
839             _log.debug('    html for test: %s', html)
840             return self.HTML_TABLE_TEST % html
841
842         return ''
843
844     def _get_baseline_result_type(self, suffix):
845         """Name of the baseline result type."""
846
847         if suffix == '.png':
848             return 'Pixel'
849         elif suffix == '.txt':
850             return 'Render Tree'
851         else:
852             return 'Other'
853
854
855 def get_host_port_object(options):
856     """Return a port object for the platform we're running on."""
857     # The only thing we really need on the host is a way to diff
858     # text files and image files, which means we need to check that some
859     # version of ImageDiff has been built. We will look for either Debug
860     # or Release versions of the default port on the platform.
861     options.configuration = "Release"
862     port_obj = port.get(None, options)
863     if not port_obj.check_image_diff(override_step=None, logging=False):
864         _log.debug('No release version of the image diff binary was found.')
865         options.configuration = "Debug"
866         port_obj = port.get(None, options)
867         if not port_obj.check_image_diff(override_step=None, logging=False):
868             _log.error('No version of image diff was found. Check your build.')
869             return None
870         else:
871             _log.debug('Found the debug version of the image diff binary.')
872     else:
873         _log.debug('Found the release version of the image diff binary.')
874     return port_obj
875
876
877 def main():
878     """Main function to produce new baselines."""
879
880     option_parser = optparse.OptionParser()
881     option_parser.add_option('-v', '--verbose',
882                              action='store_true',
883                              default=False,
884                              help='include debug-level logging.')
885
886     option_parser.add_option('-q', '--quiet',
887                              action='store_true',
888                              help='Suppress result HTML viewing')
889
890     option_parser.add_option('-p', '--platforms',
891                              default='mac,win,win-xp,win-vista,linux',
892                              help=('Comma delimited list of platforms '
893                                    'that need rebaselining.'))
894
895     option_parser.add_option('-u', '--archive_url',
896                              default=('http://build.chromium.org/buildbot/'
897                                       'layout_test_results'),
898                              help=('Url to find the layout test result archive'
899                                    ' file.'))
900     option_parser.add_option('-U', '--force_archive_url',
901                              help=('Url of result zip file. This option is for debugging '
902                                    'purposes'))
903
904     option_parser.add_option('-w', '--webkit_canary',
905                              action='store_true',
906                              default=False,
907                              help=('If True, pull baselines from webkit.org '
908                                    'canary bot.'))
909
910     option_parser.add_option('-b', '--backup',
911                              action='store_true',
912                              default=False,
913                              help=('Whether or not to backup the original test'
914                                    ' expectations file after rebaseline.'))
915
916     option_parser.add_option('-d', '--html_directory',
917                              default='',
918                              help=('The directory that stores the results for '
919                                    'rebaselining comparison.'))
920
921     option_parser.add_option('', '--use_drt',
922                              action='store_true',
923                              default=False,
924                              help=('Use ImageDiff from DumpRenderTree instead '
925                                    'of image_diff for pixel tests.'))
926
927     option_parser.add_option('', '--target-platform',
928                              default='chromium',
929                              help=('The target platform to rebaseline '
930                                    '("mac", "chromium", "qt", etc.). Defaults '
931                                    'to "chromium".'))
932     options = option_parser.parse_args()[0]
933
934     # We need to create three different Port objects over the life of this
935     # script. |target_port_obj| is used to determine configuration information:
936     # location of the expectations file, names of ports to rebaseline, etc.
937     # |port_obj| is used for runtime functionality like actually diffing
938     # Then we create a rebaselining port to actual find and manage the
939     # baselines.
940     target_options = copy.copy(options)
941     if options.target_platform == 'chromium':
942         target_options.chromium = True
943     target_port_obj = port.get(None, target_options)
944
945     # Set up our logging format.
946     log_level = logging.INFO
947     if options.verbose:
948         log_level = logging.DEBUG
949     logging.basicConfig(level=log_level,
950                         format=('%(asctime)s %(filename)s:%(lineno)-3d '
951                                 '%(levelname)s %(message)s'),
952                         datefmt='%y%m%d %H:%M:%S')
953
954     host_port_obj = get_host_port_object(options)
955     if not host_port_obj:
956         sys.exit(1)
957
958     # Verify 'platforms' option is valid.
959     if not options.platforms:
960         _log.error('Invalid "platforms" option. --platforms must be '
961                    'specified in order to rebaseline.')
962         sys.exit(1)
963     platforms = [p.strip().lower() for p in options.platforms.split(',')]
964     for platform in platforms:
965         if not platform in REBASELINE_PLATFORM_ORDER:
966             _log.error('Invalid platform: "%s"' % (platform))
967             sys.exit(1)
968
969     # Adjust the platform order so rebaseline tool is running at the order of
970     # 'mac', 'win' and 'linux'. This is in same order with layout test baseline
971     # search paths. It simplifies how the rebaseline tool detects duplicate
972     # baselines. Check _IsDupBaseline method for details.
973     rebaseline_platforms = []
974     for platform in REBASELINE_PLATFORM_ORDER:
975         if platform in platforms:
976             rebaseline_platforms.append(platform)
977
978     options.html_directory = setup_html_directory(options.html_directory)
979
980     rebaselining_tests = set()
981     backup = options.backup
982     for platform in rebaseline_platforms:
983         rebaseliner = Rebaseliner(host_port_obj, target_port_obj,
984                                   platform, options)
985
986         _log.info('')
987         log_dashed_string('Rebaseline started', platform)
988         if rebaseliner.run(backup):
989             # Only need to backup one original copy of test expectation file.
990             backup = False
991             log_dashed_string('Rebaseline done', platform)
992         else:
993             log_dashed_string('Rebaseline failed', platform, logging.ERROR)
994
995         rebaselining_tests |= set(rebaseliner.get_rebaselining_tests())
996
997     _log.info('')
998     log_dashed_string('Rebaselining result comparison started', None)
999     html_generator = HtmlGenerator(target_port_obj,
1000                                    options,
1001                                    rebaseline_platforms,
1002                                    rebaselining_tests)
1003     html_generator.generate_html()
1004     if not options.quiet:
1005         html_generator.show_html()
1006     log_dashed_string('Rebaselining result comparison done', None)
1007
1008     sys.exit(0)
1009
1010 if '__main__' == __name__:
1011     main()