cdf3bca90d9871c94f502f0b3939fe0d38a69590
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / rebaseline_chromium_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Rebaselining tool that automatically produces baselines for all platforms.
31
32 The script does the following for each platform specified:
33   1. Compile a list of tests that need rebaselining.
34   2. Download test result archive from buildbot for the platform.
35   3. Extract baselines from the archive file for all identified files.
36   4. Add new baselines to SVN repository.
37   5. For each test that has been rebaselined, remove this platform option from
38      the test in test_expectation.txt. If no other platforms remain after
39      removal, delete the rebaselined test from the file.
40
41 At the end, the script generates a html that compares old and new baselines.
42 """
43
44 from __future__ import with_statement
45
46 import codecs
47 import copy
48 import logging
49 import optparse
50 import os
51 import re
52 import shutil
53 import subprocess
54 import sys
55 import tempfile
56 import time
57 import urllib
58 import zipfile
59
60 from webkitpy.common.system import user
61 from webkitpy.common.system.executive import run_command, ScriptError
62 import webkitpy.common.checkout.scm as scm
63
64 import port
65 from layout_package import test_expectations
66
67 _log = logging.getLogger("webkitpy.layout_tests."
68                          "rebaseline_chromium_webkit_tests")
69
70 BASELINE_SUFFIXES = ['.txt', '.png', '.checksum']
71 REBASELINE_PLATFORM_ORDER = ['mac', 'win', 'win-xp', 'win-vista', 'linux']
72 ARCHIVE_DIR_NAME_DICT = {'win': 'webkit-rel',
73                          'win-vista': 'webkit-dbg-vista',
74                          'win-xp': 'webkit-rel',
75                          'mac': 'webkit-rel-mac5',
76                          'linux': 'webkit-rel-linux',
77                          'win-canary': 'webkit-rel-webkit-org',
78                          'win-vista-canary': 'webkit-dbg-vista',
79                          'win-xp-canary': 'webkit-rel-webkit-org',
80                          'mac-canary': 'webkit-rel-mac-webkit-org',
81                          'linux-canary': 'webkit-rel-linux-webkit-org'}
82
83
84 # FIXME: Should be rolled into webkitpy.Executive
85 def run_shell_with_return_code(command, print_output=False):
86     """Executes a command and returns the output and process return code.
87
88     Args:
89       command: program and arguments.
90       print_output: if true, print the command results to standard output.
91
92     Returns:
93       command output, return code
94     """
95
96     # Use a shell for subcommands on Windows to get a PATH search.
97     # FIXME: shell=True is a trail of tears, and should be removed.
98     use_shell = sys.platform.startswith('win')
99     # Note: Not thread safe: http://bugs.python.org/issue2320
100     p = subprocess.Popen(command, stdout=subprocess.PIPE,
101                          stderr=subprocess.STDOUT, shell=use_shell)
102     if print_output:
103         output_array = []
104         while True:
105             line = p.stdout.readline()
106             if not line:
107                 break
108             if print_output:
109                 print line.strip('\n')
110             output_array.append(line)
111         output = ''.join(output_array)
112     else:
113         output = p.stdout.read()
114     p.wait()
115     p.stdout.close()
116
117     return output, p.returncode
118
119
120 # FIXME: Should be rolled into webkitpy.Executive
121 def run_shell(command, print_output=False):
122     """Executes a command and returns the output.
123
124     Args:
125       command: program and arguments.
126       print_output: if true, print the command results to standard output.
127
128     Returns:
129       command output
130     """
131
132     output, return_code = run_shell_with_return_code(command, print_output)
133     return output
134
135
136 def log_dashed_string(text, platform, logging_level=logging.INFO):
137     """Log text message with dashes on both sides."""
138
139     msg = text
140     if platform:
141         msg += ': ' + platform
142     if len(msg) < 78:
143         dashes = '-' * ((78 - len(msg)) / 2)
144         msg = '%s %s %s' % (dashes, msg, dashes)
145
146     if logging_level == logging.ERROR:
147         _log.error(msg)
148     elif logging_level == logging.WARNING:
149         _log.warn(msg)
150     else:
151         _log.info(msg)
152
153
154 def setup_html_directory(html_directory):
155     """Setup the directory to store html results.
156
157        All html related files are stored in the "rebaseline_html" subdirectory.
158
159     Args:
160       html_directory: parent directory that stores the rebaselining results.
161                       If None, a temp directory is created.
162
163     Returns:
164       the directory that stores the html related rebaselining results.
165     """
166
167     if not html_directory:
168         html_directory = tempfile.mkdtemp()
169     elif not os.path.exists(html_directory):
170         os.mkdir(html_directory)
171
172     html_directory = os.path.join(html_directory, 'rebaseline_html')
173     _log.info('Html directory: "%s"', html_directory)
174
175     if os.path.exists(html_directory):
176         shutil.rmtree(html_directory, True)
177         _log.info('Deleted file at html directory: "%s"', html_directory)
178
179     if not os.path.exists(html_directory):
180         os.mkdir(html_directory)
181     return html_directory
182
183
184 def get_result_file_fullpath(html_directory, baseline_filename, platform,
185                              result_type):
186     """Get full path of the baseline result file.
187
188     Args:
189       html_directory: directory that stores the html related files.
190       baseline_filename: name of the baseline file.
191       platform: win, linux or mac
192       result_type: type of the baseline result: '.txt', '.png'.
193
194     Returns:
195       Full path of the baseline file for rebaselining result comparison.
196     """
197
198     base, ext = os.path.splitext(baseline_filename)
199     result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext)
200     fullpath = os.path.join(html_directory, result_filename)
201     _log.debug('  Result file full path: "%s".', fullpath)
202     return fullpath
203
204
205 class Rebaseliner(object):
206     """Class to produce new baselines for a given platform."""
207
208     REVISION_REGEX = r'<a href=\"(\d+)/\">'
209
210     def __init__(self, running_port, target_port, platform, options):
211         """
212         Args:
213             running_port: the Port the script is running on.
214             target_port: the Port the script uses to find port-specific
215                 configuration information like the test_expectations.txt
216                 file location and the list of test platforms.
217             platform: the test platform to rebaseline
218             options: the command-line options object."""
219         self._platform = platform
220         self._options = options
221         self._port = running_port
222         self._target_port = target_port
223         self._rebaseline_port = port.get(
224             self._target_port.test_platform_name_to_name(platform), options)
225         self._rebaselining_tests = []
226         self._rebaselined_tests = []
227
228         # Create tests and expectations helper which is used to:
229         #   -. compile list of tests that need rebaselining.
230         #   -. update the tests in test_expectations file after rebaseline
231         #      is done.
232         expectations_str = self._rebaseline_port.test_expectations()
233         self._test_expectations = \
234             test_expectations.TestExpectations(self._rebaseline_port,
235                                                None,
236                                                expectations_str,
237                                                self._platform,
238                                                False,
239                                                False)
240         self._scm = scm.default_scm()
241
242     def run(self, backup):
243         """Run rebaseline process."""
244
245         log_dashed_string('Compiling rebaselining tests', self._platform)
246         if not self._compile_rebaselining_tests():
247             return True
248
249         log_dashed_string('Downloading archive', self._platform)
250         archive_file = self._download_buildbot_archive()
251         _log.info('')
252         if not archive_file:
253             _log.error('No archive found.')
254             return False
255
256         log_dashed_string('Extracting and adding new baselines',
257                           self._platform)
258         if not self._extract_and_add_new_baselines(archive_file):
259             return False
260
261         log_dashed_string('Updating rebaselined tests in file',
262                           self._platform)
263         self._update_rebaselined_tests_in_file(backup)
264         _log.info('')
265
266         if len(self._rebaselining_tests) != len(self._rebaselined_tests):
267             _log.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN '
268                          'REBASELINED.')
269             _log.warning('  Total tests needing rebaselining: %d',
270                          len(self._rebaselining_tests))
271             _log.warning('  Total tests rebaselined: %d',
272                          len(self._rebaselined_tests))
273             return False
274
275         _log.warning('All tests needing rebaselining were successfully '
276                      'rebaselined.')
277
278         return True
279
280     def get_rebaselining_tests(self):
281         return self._rebaselining_tests
282
283     def _compile_rebaselining_tests(self):
284         """Compile list of tests that need rebaselining for the platform.
285
286         Returns:
287           List of tests that need rebaselining or
288           None if there is no such test.
289         """
290
291         self._rebaselining_tests = \
292             self._test_expectations.get_rebaselining_failures()
293         if not self._rebaselining_tests:
294             _log.warn('No tests found that need rebaselining.')
295             return None
296
297         _log.info('Total number of tests needing rebaselining '
298                   'for "%s": "%d"', self._platform,
299                   len(self._rebaselining_tests))
300
301         test_no = 1
302         for test in self._rebaselining_tests:
303             _log.info('  %d: %s', test_no, test)
304             test_no += 1
305
306         return self._rebaselining_tests
307
308     def _get_latest_revision(self, url):
309         """Get the latest layout test revision number from buildbot.
310
311         Args:
312           url: Url to retrieve layout test revision numbers.
313
314         Returns:
315           latest revision or
316           None on failure.
317         """
318
319         _log.debug('Url to retrieve revision: "%s"', url)
320
321         f = urllib.urlopen(url)
322         content = f.read()
323         f.close()
324
325         revisions = re.findall(self.REVISION_REGEX, content)
326         if not revisions:
327             _log.error('Failed to find revision, content: "%s"', content)
328             return None
329
330         revisions.sort(key=int)
331         _log.info('Latest revision: "%s"', revisions[len(revisions) - 1])
332         return revisions[len(revisions) - 1]
333
334     def _get_archive_dir_name(self, platform, webkit_canary):
335         """Get name of the layout test archive directory.
336
337         Returns:
338           Directory name or
339           None on failure
340         """
341
342         if webkit_canary:
343             platform += '-canary'
344
345         if platform in ARCHIVE_DIR_NAME_DICT:
346             return ARCHIVE_DIR_NAME_DICT[platform]
347         else:
348             _log.error('Cannot find platform key %s in archive '
349                        'directory name dictionary', platform)
350             return None
351
352     def _get_archive_url(self):
353         """Generate the url to download latest layout test archive.
354
355         Returns:
356           Url to download archive or
357           None on failure
358         """
359
360         if self._options.force_archive_url:
361             return self._options.force_archive_url
362
363         dir_name = self._get_archive_dir_name(self._platform,
364                                               self._options.webkit_canary)
365         if not dir_name:
366             return None
367
368         _log.debug('Buildbot platform dir name: "%s"', dir_name)
369
370         url_base = '%s/%s/' % (self._options.archive_url, dir_name)
371         latest_revision = self._get_latest_revision(url_base)
372         if latest_revision is None or latest_revision <= 0:
373             return None
374         archive_url = ('%s%s/layout-test-results.zip' % (url_base,
375                                                          latest_revision))
376         _log.info('Archive url: "%s"', archive_url)
377         return archive_url
378
379     def _download_buildbot_archive(self):
380         """Download layout test archive file from buildbot.
381
382         Returns:
383           True if download succeeded or
384           False otherwise.
385         """
386
387         url = self._get_archive_url()
388         if url is None:
389             return None
390
391         fn = urllib.urlretrieve(url)[0]
392         _log.info('Archive downloaded and saved to file: "%s"', fn)
393         return fn
394
395     def _extract_and_add_new_baselines(self, archive_file):
396         """Extract new baselines from archive and add them to SVN repository.
397
398         Args:
399           archive_file: full path to the archive file.
400
401         Returns:
402           List of tests that have been rebaselined or
403           None on failure.
404         """
405
406         zip_file = zipfile.ZipFile(archive_file, 'r')
407         zip_namelist = zip_file.namelist()
408
409         _log.debug('zip file namelist:')
410         for name in zip_namelist:
411             _log.debug('  ' + name)
412
413         platform = self._rebaseline_port.test_platform_name_to_name(
414             self._platform)
415         _log.debug('Platform dir: "%s"', platform)
416
417         test_no = 1
418         self._rebaselined_tests = []
419         for test in self._rebaselining_tests:
420             _log.info('Test %d: %s', test_no, test)
421
422             found = False
423             scm_error = False
424             test_basename = os.path.splitext(test)[0]
425             for suffix in BASELINE_SUFFIXES:
426                 archive_test_name = ('layout-test-results/%s-actual%s' %
427                                       (test_basename, suffix))
428                 _log.debug('  Archive test file name: "%s"',
429                            archive_test_name)
430                 if not archive_test_name in zip_namelist:
431                     _log.info('  %s file not in archive.', suffix)
432                     continue
433
434                 found = True
435                 _log.info('  %s file found in archive.', suffix)
436
437                 # Extract new baseline from archive and save it to a temp file.
438                 data = zip_file.read(archive_test_name)
439                 temp_fd, temp_name = tempfile.mkstemp(suffix)
440                 f = os.fdopen(temp_fd, 'wb')
441                 f.write(data)
442                 f.close()
443
444                 expected_filename = '%s-expected%s' % (test_basename, suffix)
445                 expected_fullpath = os.path.join(
446                     self._rebaseline_port.baseline_path(), expected_filename)
447                 expected_fullpath = os.path.normpath(expected_fullpath)
448                 _log.debug('  Expected file full path: "%s"',
449                            expected_fullpath)
450
451                 # TODO(victorw): for now, the rebaselining tool checks whether
452                 # or not THIS baseline is duplicate and should be skipped.
453                 # We could improve the tool to check all baselines in upper
454                 # and lower
455                 # levels and remove all duplicated baselines.
456                 if self._is_dup_baseline(temp_name,
457                                         expected_fullpath,
458                                         test,
459                                         suffix,
460                                         self._platform):
461                     os.remove(temp_name)
462                     self._delete_baseline(expected_fullpath)
463                     continue
464
465                 # Create the new baseline directory if it doesn't already
466                 # exist.
467                 self._port.maybe_make_directory(
468                     os.path.dirname(expected_fullpath))
469
470                 shutil.move(temp_name, expected_fullpath)
471
472                 if 0 != self._scm.add(expected_fullpath, return_exit_code=True):
473                     # FIXME: print detailed diagnose messages
474                     scm_error = True
475                 elif suffix != '.checksum':
476                     self._create_html_baseline_files(expected_fullpath)
477
478             if not found:
479                 _log.warn('  No new baselines found in archive.')
480             else:
481                 if scm_error:
482                     _log.warn('  Failed to add baselines to your repository.')
483                 else:
484                     _log.info('  Rebaseline succeeded.')
485                     self._rebaselined_tests.append(test)
486
487             test_no += 1
488
489         zip_file.close()
490         os.remove(archive_file)
491
492         return self._rebaselined_tests
493
494     def _is_dup_baseline(self, new_baseline, baseline_path, test, suffix,
495                          platform):
496         """Check whether a baseline is duplicate and can fallback to same
497            baseline for another platform. For example, if a test has same
498            baseline on linux and windows, then we only store windows
499            baseline and linux baseline will fallback to the windows version.
500
501         Args:
502           expected_filename: baseline expectation file name.
503           test: test name.
504           suffix: file suffix of the expected results, including dot;
505                   e.g. '.txt' or '.png'.
506           platform: baseline platform 'mac', 'win' or 'linux'.
507
508         Returns:
509           True if the baseline is unnecessary.
510           False otherwise.
511         """
512         test_filepath = os.path.join(self._target_port.layout_tests_dir(),
513                                      test)
514         all_baselines = self._rebaseline_port.expected_baselines(
515             test_filepath, suffix, True)
516         for (fallback_dir, fallback_file) in all_baselines:
517             if fallback_dir and fallback_file:
518                 fallback_fullpath = os.path.normpath(
519                     os.path.join(fallback_dir, fallback_file))
520                 if fallback_fullpath.lower() != baseline_path.lower():
521                     with codecs.open(file1, "r", "utf8") as file_handle1:
522                         output1 = file_handle1.read()
523                     with codecs.open(file2, "r", "utf8") as file_handle2:
524                         output2 = file_handle2.read()
525                     if not self._diff_baselines(new_baseline,
526                                                 fallback_fullpath):
527                         _log.info('  Found same baseline at %s',
528                                   fallback_fullpath)
529                         return True
530                     else:
531                         return False
532
533         return False
534
535     def _diff_baselines(self, output1, output2, is_image):
536         """Check whether two baselines are different.
537
538         Args:
539           output1, output2: contents of the baselines to compare.
540
541         Returns:
542           True if two files are different or have different extensions.
543           False otherwise.
544         """
545
546         if is_image:
547             return self._port.diff_image(output1, output2)
548         else:
549             return self._port.compare_text(output1, output2)
550
551     def _delete_baseline(self, filename):
552         """Remove the file from repository and delete it from disk.
553
554         Args:
555           filename: full path of the file to delete.
556         """
557
558         if not filename or not os.path.isfile(filename):
559             return
560         self._scm.delete(filename)
561
562     def _update_rebaselined_tests_in_file(self, backup):
563         """Update the rebaselined tests in test expectations file.
564
565         Args:
566           backup: if True, backup the original test expectations file.
567
568         Returns:
569           no
570         """
571
572         if self._rebaselined_tests:
573             new_expectations = (
574                 self._test_expectations.remove_platform_from_expectations(
575                 self._rebaselined_tests, self._platform))
576             path = self._target_port.path_to_test_expectations_file()
577             if backup:
578                 date_suffix = time.strftime('%Y%m%d%H%M%S',
579                                             time.localtime(time.time()))
580                 backup_file = ('%s.orig.%s' % (path, date_suffix))
581                 if os.path.exists(backup_file):
582                     os.remove(backup_file)
583                 _log.info('Saving original file to "%s"', backup_file)
584                 os.rename(path, backup_file)
585             # FIXME: What encoding are these files?
586             # Or is new_expectations always a byte array?
587             with open(path, "w") as file:
588                 file.write(new_expectations)
589             self._scm.add(path)
590         else:
591             _log.info('No test was rebaselined so nothing to remove.')
592
593     def _create_html_baseline_files(self, baseline_fullpath):
594         """Create baseline files (old, new and diff) in html directory.
595
596            The files are used to compare the rebaselining results.
597
598         Args:
599           baseline_fullpath: full path of the expected baseline file.
600         """
601
602         if not baseline_fullpath or not os.path.exists(baseline_fullpath):
603             return
604
605         # Copy the new baseline to html directory for result comparison.
606         baseline_filename = os.path.basename(baseline_fullpath)
607         new_file = get_result_file_fullpath(self._options.html_directory,
608                                             baseline_filename, self._platform,
609                                             'new')
610         shutil.copyfile(baseline_fullpath, new_file)
611         _log.info('  Html: copied new baseline file from "%s" to "%s".',
612                   baseline_fullpath, new_file)
613
614         # Get the old baseline from the repository and save to the html directory.
615         try:
616             output = self._scm.show_head(baseline_fullpath)
617         except ScriptError, e:
618             _log.info(e)
619             output = ""
620
621         if (not output) or (output.upper().rstrip().endswith(
622             'NO SUCH FILE OR DIRECTORY')):
623             _log.info('  No base file: "%s"', baseline_fullpath)
624             return
625         base_file = get_result_file_fullpath(self._options.html_directory,
626                                              baseline_filename, self._platform,
627                                              'old')
628         # FIXME: This assumes run_shell returns a byte array.
629         # We should be using an explicit encoding here.
630         with open(base_file, "wb") as file:
631             file.write(output)
632         _log.info('  Html: created old baseline file: "%s".',
633                   base_file)
634
635         # Get the diff between old and new baselines and save to the html dir.
636         if baseline_filename.upper().endswith('.TXT'):
637             output = self._scm.diff_for_file(baseline_fullpath, log=_log)
638             if output:
639                 diff_file = get_result_file_fullpath(
640                     self._options.html_directory, baseline_filename,
641                     self._platform, 'diff')
642                 # FIXME: This assumes run_shell returns a byte array, not unicode()
643                 with open(diff_file, 'wb') as file:
644                     file.write(output)
645                 _log.info('  Html: created baseline diff file: "%s".',
646                           diff_file)
647
648 class HtmlGenerator(object):
649     """Class to generate rebaselining result comparison html."""
650
651     HTML_REBASELINE = ('<html>'
652                        '<head>'
653                        '<style>'
654                        'body {font-family: sans-serif;}'
655                        '.mainTable {background: #666666;}'
656                        '.mainTable td , .mainTable th {background: white;}'
657                        '.detail {margin-left: 10px; margin-top: 3px;}'
658                        '</style>'
659                        '<title>Rebaselining Result Comparison (%(time)s)'
660                        '</title>'
661                        '</head>'
662                        '<body>'
663                        '<h2>Rebaselining Result Comparison (%(time)s)</h2>'
664                        '%(body)s'
665                        '</body>'
666                        '</html>')
667     HTML_NO_REBASELINING_TESTS = (
668         '<p>No tests found that need rebaselining.</p>')
669     HTML_TABLE_TEST = ('<table class="mainTable" cellspacing=1 cellpadding=5>'
670                        '%s</table><br>')
671     HTML_TR_TEST = ('<tr>'
672                     '<th style="background-color: #CDECDE; border-bottom: '
673                     '1px solid black; font-size: 18pt; font-weight: bold" '
674                     'colspan="5">'
675                     '<a href="%s">%s</a>'
676                     '</th>'
677                     '</tr>')
678     HTML_TEST_DETAIL = ('<div class="detail">'
679                         '<tr>'
680                         '<th width="100">Baseline</th>'
681                         '<th width="100">Platform</th>'
682                         '<th width="200">Old</th>'
683                         '<th width="200">New</th>'
684                         '<th width="150">Difference</th>'
685                         '</tr>'
686                         '%s'
687                         '</div>')
688     HTML_TD_NOLINK = '<td align=center><a>%s</a></td>'
689     HTML_TD_LINK = '<td align=center><a href="%(uri)s">%(name)s</a></td>'
690     HTML_TD_LINK_IMG = ('<td><a href="%(uri)s">'
691                         '<img style="width: 200" src="%(uri)s" /></a></td>')
692     HTML_TR = '<tr>%s</tr>'
693
694     def __init__(self, target_port, options, platforms, rebaselining_tests):
695         self._html_directory = options.html_directory
696         self._target_port = target_port
697         self._platforms = platforms
698         self._rebaselining_tests = rebaselining_tests
699         self._html_file = os.path.join(options.html_directory,
700                                        'rebaseline.html')
701
702     def generate_html(self):
703         """Generate html file for rebaselining result comparison."""
704
705         _log.info('Generating html file')
706
707         html_body = ''
708         if not self._rebaselining_tests:
709             html_body += self.HTML_NO_REBASELINING_TESTS
710         else:
711             tests = list(self._rebaselining_tests)
712             tests.sort()
713
714             test_no = 1
715             for test in tests:
716                 _log.info('Test %d: %s', test_no, test)
717                 html_body += self._generate_html_for_one_test(test)
718
719         html = self.HTML_REBASELINE % ({'time': time.asctime(),
720                                         'body': html_body})
721         _log.debug(html)
722
723         with codecs.open(self._html_file, "w", "utf-8") as file:
724             file.write(html)
725
726         _log.info('Baseline comparison html generated at "%s"',
727                   self._html_file)
728
729     def show_html(self):
730         """Launch the rebaselining html in brwoser."""
731
732         _log.info('Launching html: "%s"', self._html_file)
733         user.open_url(self._html_file)
734         _log.info('Html launched.')
735
736     def _generate_baseline_links(self, test_basename, suffix, platform):
737         """Generate links for baseline results (old, new and diff).
738
739         Args:
740           test_basename: base filename of the test
741           suffix: baseline file suffixes: '.txt', '.png'
742           platform: win, linux or mac
743
744         Returns:
745           html links for showing baseline results (old, new and diff)
746         """
747
748         baseline_filename = '%s-expected%s' % (test_basename, suffix)
749         _log.debug('    baseline filename: "%s"', baseline_filename)
750
751         new_file = get_result_file_fullpath(self._html_directory,
752                                             baseline_filename, platform, 'new')
753         _log.info('    New baseline file: "%s"', new_file)
754         if not os.path.exists(new_file):
755             _log.info('    No new baseline file: "%s"', new_file)
756             return ''
757
758         old_file = get_result_file_fullpath(self._html_directory,
759                                             baseline_filename, platform, 'old')
760         _log.info('    Old baseline file: "%s"', old_file)
761         if suffix == '.png':
762             html_td_link = self.HTML_TD_LINK_IMG
763         else:
764             html_td_link = self.HTML_TD_LINK
765
766         links = ''
767         if os.path.exists(old_file):
768             links += html_td_link % {
769                 'uri': self._target_port.filename_to_uri(old_file),
770                 'name': baseline_filename}
771         else:
772             _log.info('    No old baseline file: "%s"', old_file)
773             links += self.HTML_TD_NOLINK % ''
774
775         links += html_td_link % {'uri': self._target_port.filename_to_uri(
776                                      new_file),
777                                  'name': baseline_filename}
778
779         diff_file = get_result_file_fullpath(self._html_directory,
780                                              baseline_filename, platform,
781                                              'diff')
782         _log.info('    Baseline diff file: "%s"', diff_file)
783         if os.path.exists(diff_file):
784             links += html_td_link % {'uri': self._target_port.filename_to_uri(
785                 diff_file), 'name': 'Diff'}
786         else:
787             _log.info('    No baseline diff file: "%s"', diff_file)
788             links += self.HTML_TD_NOLINK % ''
789
790         return links
791
792     def _generate_html_for_one_test(self, test):
793         """Generate html for one rebaselining test.
794
795         Args:
796           test: layout test name
797
798         Returns:
799           html that compares baseline results for the test.
800         """
801
802         test_basename = os.path.basename(os.path.splitext(test)[0])
803         _log.info('  basename: "%s"', test_basename)
804         rows = []
805         for suffix in BASELINE_SUFFIXES:
806             if suffix == '.checksum':
807                 continue
808
809             _log.info('  Checking %s files', suffix)
810             for platform in self._platforms:
811                 links = self._generate_baseline_links(test_basename, suffix,
812                     platform)
813                 if links:
814                     row = self.HTML_TD_NOLINK % self._get_baseline_result_type(
815                         suffix)
816                     row += self.HTML_TD_NOLINK % platform
817                     row += links
818                     _log.debug('    html row: %s', row)
819
820                     rows.append(self.HTML_TR % row)
821
822         if rows:
823             test_path = os.path.join(self._target_port.layout_tests_dir(),
824                                      test)
825             html = self.HTML_TR_TEST % (
826                 self._target_port.filename_to_uri(test_path), test)
827             html += self.HTML_TEST_DETAIL % ' '.join(rows)
828
829             _log.debug('    html for test: %s', html)
830             return self.HTML_TABLE_TEST % html
831
832         return ''
833
834     def _get_baseline_result_type(self, suffix):
835         """Name of the baseline result type."""
836
837         if suffix == '.png':
838             return 'Pixel'
839         elif suffix == '.txt':
840             return 'Render Tree'
841         else:
842             return 'Other'
843
844
845 def get_host_port_object(options):
846     """Return a port object for the platform we're running on."""
847     # The only thing we really need on the host is a way to diff
848     # text files and image files, which means we need to check that some
849     # version of ImageDiff has been built. We will look for either Debug
850     # or Release versions of the default port on the platform.
851     options.configuration = "Release"
852     port_obj = port.get(None, options)
853     if not port_obj.check_image_diff(override_step=None, logging=False):
854         _log.debug('No release version of the image diff binary was found.')
855         options.configuration = "Debug"
856         port_obj = port.get(None, options)
857         if not port_obj.check_image_diff(override_step=None, logging=False):
858             _log.error('No version of image diff was found. Check your build.')
859             return None
860         else:
861             _log.debug('Found the debug version of the image diff binary.')
862     else:
863         _log.debug('Found the release version of the image diff binary.')
864     return port_obj
865
866
867 def main():
868     """Main function to produce new baselines."""
869
870     option_parser = optparse.OptionParser()
871     option_parser.add_option('-v', '--verbose',
872                              action='store_true',
873                              default=False,
874                              help='include debug-level logging.')
875
876     option_parser.add_option('-q', '--quiet',
877                              action='store_true',
878                              help='Suppress result HTML viewing')
879
880     option_parser.add_option('-p', '--platforms',
881                              default='mac,win,win-xp,win-vista,linux',
882                              help=('Comma delimited list of platforms '
883                                    'that need rebaselining.'))
884
885     option_parser.add_option('-u', '--archive_url',
886                              default=('http://build.chromium.org/buildbot/'
887                                       'layout_test_results'),
888                              help=('Url to find the layout test result archive'
889                                    ' file.'))
890     option_parser.add_option('-U', '--force_archive_url',
891                              help=('Url of result zip file. This option is for debugging '
892                                    'purposes'))
893
894     option_parser.add_option('-w', '--webkit_canary',
895                              action='store_true',
896                              default=False,
897                              help=('If True, pull baselines from webkit.org '
898                                    'canary bot.'))
899
900     option_parser.add_option('-b', '--backup',
901                              action='store_true',
902                              default=False,
903                              help=('Whether or not to backup the original test'
904                                    ' expectations file after rebaseline.'))
905
906     option_parser.add_option('-d', '--html_directory',
907                              default='',
908                              help=('The directory that stores the results for '
909                                    'rebaselining comparison.'))
910
911     option_parser.add_option('', '--use_drt',
912                              action='store_true',
913                              default=False,
914                              help=('Use ImageDiff from DumpRenderTree instead '
915                                    'of image_diff for pixel tests.'))
916
917     option_parser.add_option('', '--target-platform',
918                              default='chromium',
919                              help=('The target platform to rebaseline '
920                                    '("mac", "chromium", "qt", etc.). Defaults '
921                                    'to "chromium".'))
922     options = option_parser.parse_args()[0]
923
924     # We need to create three different Port objects over the life of this
925     # script. |target_port_obj| is used to determine configuration information:
926     # location of the expectations file, names of ports to rebaseline, etc.
927     # |port_obj| is used for runtime functionality like actually diffing
928     # Then we create a rebaselining port to actual find and manage the
929     # baselines.
930     target_options = copy.copy(options)
931     if options.target_platform == 'chromium':
932         target_options.chromium = True
933     target_port_obj = port.get(None, target_options)
934
935     # Set up our logging format.
936     log_level = logging.INFO
937     if options.verbose:
938         log_level = logging.DEBUG
939     logging.basicConfig(level=log_level,
940                         format=('%(asctime)s %(filename)s:%(lineno)-3d '
941                                 '%(levelname)s %(message)s'),
942                         datefmt='%y%m%d %H:%M:%S')
943
944     host_port_obj = get_host_port_object(options)
945     if not host_port_obj:
946         sys.exit(1)
947
948     # Verify 'platforms' option is valid.
949     if not options.platforms:
950         _log.error('Invalid "platforms" option. --platforms must be '
951                    'specified in order to rebaseline.')
952         sys.exit(1)
953     platforms = [p.strip().lower() for p in options.platforms.split(',')]
954     for platform in platforms:
955         if not platform in REBASELINE_PLATFORM_ORDER:
956             _log.error('Invalid platform: "%s"' % (platform))
957             sys.exit(1)
958
959     # Adjust the platform order so rebaseline tool is running at the order of
960     # 'mac', 'win' and 'linux'. This is in same order with layout test baseline
961     # search paths. It simplifies how the rebaseline tool detects duplicate
962     # baselines. Check _IsDupBaseline method for details.
963     rebaseline_platforms = []
964     for platform in REBASELINE_PLATFORM_ORDER:
965         if platform in platforms:
966             rebaseline_platforms.append(platform)
967
968     options.html_directory = setup_html_directory(options.html_directory)
969
970     rebaselining_tests = set()
971     backup = options.backup
972     for platform in rebaseline_platforms:
973         rebaseliner = Rebaseliner(host_port_obj, target_port_obj,
974                                   platform, options)
975
976         _log.info('')
977         log_dashed_string('Rebaseline started', platform)
978         if rebaseliner.run(backup):
979             # Only need to backup one original copy of test expectation file.
980             backup = False
981             log_dashed_string('Rebaseline done', platform)
982         else:
983             log_dashed_string('Rebaseline failed', platform, logging.ERROR)
984
985         rebaselining_tests |= set(rebaseliner.get_rebaselining_tests())
986
987     _log.info('')
988     log_dashed_string('Rebaselining result comparison started', None)
989     html_generator = HtmlGenerator(target_port_obj,
990                                    options,
991                                    rebaseline_platforms,
992                                    rebaselining_tests)
993     html_generator.generate_html()
994     if not options.quiet:
995         html_generator.show_html()
996     log_dashed_string('Rebaselining result comparison done', None)
997
998     sys.exit(0)
999
1000 if '__main__' == __name__:
1001     main()