Rename WebKitTools to Tools
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / rebaseline_chromium_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Rebaselining tool that automatically produces baselines for all platforms.
31
32 The script does the following for each platform specified:
33   1. Compile a list of tests that need rebaselining.
34   2. Download test result archive from buildbot for the platform.
35   3. Extract baselines from the archive file for all identified files.
36   4. Add new baselines to SVN repository.
37   5. For each test that has been rebaselined, remove this platform option from
38      the test in test_expectation.txt. If no other platforms remain after
39      removal, delete the rebaselined test from the file.
40
41 At the end, the script generates a html that compares old and new baselines.
42 """
43
44 from __future__ import with_statement
45
46 import codecs
47 import copy
48 import logging
49 import optparse
50 import os
51 import re
52 import shutil
53 import subprocess
54 import sys
55 import tempfile
56 import time
57 import urllib
58 import zipfile
59
60 from webkitpy.common.system import path
61 from webkitpy.common.system import user
62 from webkitpy.common.system.executive import Executive, ScriptError
63 import webkitpy.common.checkout.scm as scm
64
65 import port
66 from layout_package import test_expectations
67
68 _log = logging.getLogger("webkitpy.layout_tests."
69                          "rebaseline_chromium_webkit_tests")
70
71 BASELINE_SUFFIXES = ['.txt', '.png', '.checksum']
72 REBASELINE_PLATFORM_ORDER = ['mac', 'win', 'win-xp', 'win-vista', 'linux']
73 ARCHIVE_DIR_NAME_DICT = {'win': 'Webkit_Win',
74                          'win-vista': 'webkit-dbg-vista',
75                          'win-xp': 'Webkit_Win',
76                          'mac': 'Webkit_Mac10_5',
77                          'linux': 'webkit-rel-linux64',
78                          'win-canary': 'webkit-rel-webkit-org',
79                          'win-vista-canary': 'webkit-dbg-vista',
80                          'win-xp-canary': 'webkit-rel-webkit-org',
81                          'mac-canary': 'webkit-rel-mac-webkit-org',
82                          'linux-canary': 'webkit-rel-linux-webkit-org'}
83
84
85 def log_dashed_string(text, platform, logging_level=logging.INFO):
86     """Log text message with dashes on both sides."""
87
88     msg = text
89     if platform:
90         msg += ': ' + platform
91     if len(msg) < 78:
92         dashes = '-' * ((78 - len(msg)) / 2)
93         msg = '%s %s %s' % (dashes, msg, dashes)
94
95     if logging_level == logging.ERROR:
96         _log.error(msg)
97     elif logging_level == logging.WARNING:
98         _log.warn(msg)
99     else:
100         _log.info(msg)
101
102
103 def setup_html_directory(html_directory):
104     """Setup the directory to store html results.
105
106        All html related files are stored in the "rebaseline_html" subdirectory.
107
108     Args:
109       html_directory: parent directory that stores the rebaselining results.
110                       If None, a temp directory is created.
111
112     Returns:
113       the directory that stores the html related rebaselining results.
114     """
115
116     if not html_directory:
117         html_directory = tempfile.mkdtemp()
118     elif not os.path.exists(html_directory):
119         os.mkdir(html_directory)
120
121     html_directory = os.path.join(html_directory, 'rebaseline_html')
122     _log.info('Html directory: "%s"', html_directory)
123
124     if os.path.exists(html_directory):
125         shutil.rmtree(html_directory, True)
126         _log.info('Deleted file at html directory: "%s"', html_directory)
127
128     if not os.path.exists(html_directory):
129         os.mkdir(html_directory)
130     return html_directory
131
132
133 def get_result_file_fullpath(html_directory, baseline_filename, platform,
134                              result_type):
135     """Get full path of the baseline result file.
136
137     Args:
138       html_directory: directory that stores the html related files.
139       baseline_filename: name of the baseline file.
140       platform: win, linux or mac
141       result_type: type of the baseline result: '.txt', '.png'.
142
143     Returns:
144       Full path of the baseline file for rebaselining result comparison.
145     """
146
147     base, ext = os.path.splitext(baseline_filename)
148     result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext)
149     fullpath = os.path.join(html_directory, result_filename)
150     _log.debug('  Result file full path: "%s".', fullpath)
151     return fullpath
152
153
154 class Rebaseliner(object):
155     """Class to produce new baselines for a given platform."""
156
157     REVISION_REGEX = r'<a href=\"(\d+)/\">'
158
159     def __init__(self, running_port, target_port, platform, options):
160         """
161         Args:
162             running_port: the Port the script is running on.
163             target_port: the Port the script uses to find port-specific
164                 configuration information like the test_expectations.txt
165                 file location and the list of test platforms.
166             platform: the test platform to rebaseline
167             options: the command-line options object."""
168         self._platform = platform
169         self._options = options
170         self._port = running_port
171         self._target_port = target_port
172         self._rebaseline_port = port.get(
173             self._target_port.test_platform_name_to_name(platform), options)
174         self._rebaselining_tests = []
175         self._rebaselined_tests = []
176
177         # Create tests and expectations helper which is used to:
178         #   -. compile list of tests that need rebaselining.
179         #   -. update the tests in test_expectations file after rebaseline
180         #      is done.
181         expectations_str = self._rebaseline_port.test_expectations()
182         self._test_expectations = \
183             test_expectations.TestExpectations(self._rebaseline_port,
184                                                None,
185                                                expectations_str,
186                                                self._platform,
187                                                False,
188                                                False)
189         self._scm = scm.default_scm()
190
191     def run(self, backup):
192         """Run rebaseline process."""
193
194         log_dashed_string('Compiling rebaselining tests', self._platform)
195         if not self._compile_rebaselining_tests():
196             return True
197
198         log_dashed_string('Downloading archive', self._platform)
199         archive_file = self._download_buildbot_archive()
200         _log.info('')
201         if not archive_file:
202             _log.error('No archive found.')
203             return False
204
205         log_dashed_string('Extracting and adding new baselines',
206                           self._platform)
207         if not self._extract_and_add_new_baselines(archive_file):
208             return False
209
210         log_dashed_string('Updating rebaselined tests in file',
211                           self._platform)
212         self._update_rebaselined_tests_in_file(backup)
213         _log.info('')
214
215         if len(self._rebaselining_tests) != len(self._rebaselined_tests):
216             _log.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN '
217                          'REBASELINED.')
218             _log.warning('  Total tests needing rebaselining: %d',
219                          len(self._rebaselining_tests))
220             _log.warning('  Total tests rebaselined: %d',
221                          len(self._rebaselined_tests))
222             return False
223
224         _log.warning('All tests needing rebaselining were successfully '
225                      'rebaselined.')
226
227         return True
228
229     def get_rebaselining_tests(self):
230         return self._rebaselining_tests
231
232     def _compile_rebaselining_tests(self):
233         """Compile list of tests that need rebaselining for the platform.
234
235         Returns:
236           List of tests that need rebaselining or
237           None if there is no such test.
238         """
239
240         self._rebaselining_tests = \
241             self._test_expectations.get_rebaselining_failures()
242         if not self._rebaselining_tests:
243             _log.warn('No tests found that need rebaselining.')
244             return None
245
246         _log.info('Total number of tests needing rebaselining '
247                   'for "%s": "%d"', self._platform,
248                   len(self._rebaselining_tests))
249
250         test_no = 1
251         for test in self._rebaselining_tests:
252             _log.info('  %d: %s', test_no, test)
253             test_no += 1
254
255         return self._rebaselining_tests
256
257     def _get_latest_revision(self, url):
258         """Get the latest layout test revision number from buildbot.
259
260         Args:
261           url: Url to retrieve layout test revision numbers.
262
263         Returns:
264           latest revision or
265           None on failure.
266         """
267
268         _log.debug('Url to retrieve revision: "%s"', url)
269
270         f = urllib.urlopen(url)
271         content = f.read()
272         f.close()
273
274         revisions = re.findall(self.REVISION_REGEX, content)
275         if not revisions:
276             _log.error('Failed to find revision, content: "%s"', content)
277             return None
278
279         revisions.sort(key=int)
280         _log.info('Latest revision: "%s"', revisions[len(revisions) - 1])
281         return revisions[len(revisions) - 1]
282
283     def _get_archive_dir_name(self, platform, webkit_canary):
284         """Get name of the layout test archive directory.
285
286         Returns:
287           Directory name or
288           None on failure
289         """
290
291         if webkit_canary:
292             platform += '-canary'
293
294         if platform in ARCHIVE_DIR_NAME_DICT:
295             return ARCHIVE_DIR_NAME_DICT[platform]
296         else:
297             _log.error('Cannot find platform key %s in archive '
298                        'directory name dictionary', platform)
299             return None
300
301     def _get_archive_url(self):
302         """Generate the url to download latest layout test archive.
303
304         Returns:
305           Url to download archive or
306           None on failure
307         """
308
309         if self._options.force_archive_url:
310             return self._options.force_archive_url
311
312         dir_name = self._get_archive_dir_name(self._platform,
313                                               self._options.webkit_canary)
314         if not dir_name:
315             return None
316
317         _log.debug('Buildbot platform dir name: "%s"', dir_name)
318
319         url_base = '%s/%s/' % (self._options.archive_url, dir_name)
320         latest_revision = self._get_latest_revision(url_base)
321         if latest_revision is None or latest_revision <= 0:
322             return None
323         archive_url = ('%s%s/layout-test-results.zip' % (url_base,
324                                                          latest_revision))
325         _log.info('Archive url: "%s"', archive_url)
326         return archive_url
327
328     def _download_buildbot_archive(self):
329         """Download layout test archive file from buildbot.
330
331         Returns:
332           True if download succeeded or
333           False otherwise.
334         """
335
336         url = self._get_archive_url()
337         if url is None:
338             return None
339
340         fn = urllib.urlretrieve(url)[0]
341         _log.info('Archive downloaded and saved to file: "%s"', fn)
342         return fn
343
344     def _extract_and_add_new_baselines(self, archive_file):
345         """Extract new baselines from archive and add them to SVN repository.
346
347         Args:
348           archive_file: full path to the archive file.
349
350         Returns:
351           List of tests that have been rebaselined or
352           None on failure.
353         """
354
355         zip_file = zipfile.ZipFile(archive_file, 'r')
356         zip_namelist = zip_file.namelist()
357
358         _log.debug('zip file namelist:')
359         for name in zip_namelist:
360             _log.debug('  ' + name)
361
362         platform = self._rebaseline_port.test_platform_name_to_name(
363             self._platform)
364         _log.debug('Platform dir: "%s"', platform)
365
366         test_no = 1
367         self._rebaselined_tests = []
368         for test in self._rebaselining_tests:
369             _log.info('Test %d: %s', test_no, test)
370
371             found = False
372             scm_error = False
373             test_basename = os.path.splitext(test)[0]
374             for suffix in BASELINE_SUFFIXES:
375                 archive_test_name = ('layout-test-results/%s-actual%s' %
376                                       (test_basename, suffix))
377                 _log.debug('  Archive test file name: "%s"',
378                            archive_test_name)
379                 if not archive_test_name in zip_namelist:
380                     _log.info('  %s file not in archive.', suffix)
381                     continue
382
383                 found = True
384                 _log.info('  %s file found in archive.', suffix)
385
386                 # Extract new baseline from archive and save it to a temp file.
387                 data = zip_file.read(archive_test_name)
388                 temp_fd, temp_name = tempfile.mkstemp(suffix)
389                 f = os.fdopen(temp_fd, 'wb')
390                 f.write(data)
391                 f.close()
392
393                 expected_filename = '%s-expected%s' % (test_basename, suffix)
394                 expected_fullpath = os.path.join(
395                     self._rebaseline_port.baseline_path(), expected_filename)
396                 expected_fullpath = os.path.normpath(expected_fullpath)
397                 _log.debug('  Expected file full path: "%s"',
398                            expected_fullpath)
399
400                 # TODO(victorw): for now, the rebaselining tool checks whether
401                 # or not THIS baseline is duplicate and should be skipped.
402                 # We could improve the tool to check all baselines in upper
403                 # and lower
404                 # levels and remove all duplicated baselines.
405                 if self._is_dup_baseline(temp_name,
406                                         expected_fullpath,
407                                         test,
408                                         suffix,
409                                         self._platform):
410                     os.remove(temp_name)
411                     self._delete_baseline(expected_fullpath)
412                     continue
413
414                 # Create the new baseline directory if it doesn't already
415                 # exist.
416                 self._port.maybe_make_directory(
417                     os.path.dirname(expected_fullpath))
418
419                 shutil.move(temp_name, expected_fullpath)
420
421                 if 0 != self._scm.add(expected_fullpath, return_exit_code=True):
422                     # FIXME: print detailed diagnose messages
423                     scm_error = True
424                 elif suffix != '.checksum':
425                     self._create_html_baseline_files(expected_fullpath)
426
427             if not found:
428                 _log.warn('  No new baselines found in archive.')
429             else:
430                 if scm_error:
431                     _log.warn('  Failed to add baselines to your repository.')
432                 else:
433                     _log.info('  Rebaseline succeeded.')
434                     self._rebaselined_tests.append(test)
435
436             test_no += 1
437
438         zip_file.close()
439         os.remove(archive_file)
440
441         return self._rebaselined_tests
442
443     def _is_dup_baseline(self, new_baseline, baseline_path, test, suffix,
444                          platform):
445         """Check whether a baseline is duplicate and can fallback to same
446            baseline for another platform. For example, if a test has same
447            baseline on linux and windows, then we only store windows
448            baseline and linux baseline will fallback to the windows version.
449
450         Args:
451           expected_filename: baseline expectation file name.
452           test: test name.
453           suffix: file suffix of the expected results, including dot;
454                   e.g. '.txt' or '.png'.
455           platform: baseline platform 'mac', 'win' or 'linux'.
456
457         Returns:
458           True if the baseline is unnecessary.
459           False otherwise.
460         """
461         test_filepath = os.path.join(self._target_port.layout_tests_dir(),
462                                      test)
463         all_baselines = self._rebaseline_port.expected_baselines(
464             test_filepath, suffix, True)
465         for (fallback_dir, fallback_file) in all_baselines:
466             if fallback_dir and fallback_file:
467                 fallback_fullpath = os.path.normpath(
468                     os.path.join(fallback_dir, fallback_file))
469                 if fallback_fullpath.lower() != baseline_path.lower():
470                     with codecs.open(new_baseline, "r",
471                                      None) as file_handle1:
472                         new_output = file_handle1.read()
473                     with codecs.open(fallback_fullpath, "r",
474                                      None) as file_handle2:
475                         fallback_output = file_handle2.read()
476                     is_image = baseline_path.lower().endswith('.png')
477                     if not self._diff_baselines(new_output, fallback_output,
478                                                 is_image):
479                         _log.info('  Found same baseline at %s',
480                                   fallback_fullpath)
481                         return True
482                     else:
483                         return False
484
485         return False
486
487     def _diff_baselines(self, output1, output2, is_image):
488         """Check whether two baselines are different.
489
490         Args:
491           output1, output2: contents of the baselines to compare.
492
493         Returns:
494           True if two files are different or have different extensions.
495           False otherwise.
496         """
497
498         if is_image:
499             return self._port.diff_image(output1, output2, None)
500         else:
501             return self._port.compare_text(output1, output2)
502
503     def _delete_baseline(self, filename):
504         """Remove the file from repository and delete it from disk.
505
506         Args:
507           filename: full path of the file to delete.
508         """
509
510         if not filename or not os.path.isfile(filename):
511             return
512         self._scm.delete(filename)
513
514     def _update_rebaselined_tests_in_file(self, backup):
515         """Update the rebaselined tests in test expectations file.
516
517         Args:
518           backup: if True, backup the original test expectations file.
519
520         Returns:
521           no
522         """
523
524         if self._rebaselined_tests:
525             new_expectations = (
526                 self._test_expectations.remove_platform_from_expectations(
527                 self._rebaselined_tests, self._platform))
528             path = self._target_port.path_to_test_expectations_file()
529             if backup:
530                 date_suffix = time.strftime('%Y%m%d%H%M%S',
531                                             time.localtime(time.time()))
532                 backup_file = ('%s.orig.%s' % (path, date_suffix))
533                 if os.path.exists(backup_file):
534                     os.remove(backup_file)
535                 _log.info('Saving original file to "%s"', backup_file)
536                 os.rename(path, backup_file)
537             # FIXME: What encoding are these files?
538             # Or is new_expectations always a byte array?
539             with open(path, "w") as file:
540                 file.write(new_expectations)
541             # self._scm.add(path)
542         else:
543             _log.info('No test was rebaselined so nothing to remove.')
544
545     def _create_html_baseline_files(self, baseline_fullpath):
546         """Create baseline files (old, new and diff) in html directory.
547
548            The files are used to compare the rebaselining results.
549
550         Args:
551           baseline_fullpath: full path of the expected baseline file.
552         """
553
554         if not baseline_fullpath or not os.path.exists(baseline_fullpath):
555             return
556
557         # Copy the new baseline to html directory for result comparison.
558         baseline_filename = os.path.basename(baseline_fullpath)
559         new_file = get_result_file_fullpath(self._options.html_directory,
560                                             baseline_filename, self._platform,
561                                             'new')
562         shutil.copyfile(baseline_fullpath, new_file)
563         _log.info('  Html: copied new baseline file from "%s" to "%s".',
564                   baseline_fullpath, new_file)
565
566         # Get the old baseline from the repository and save to the html directory.
567         try:
568             output = self._scm.show_head(baseline_fullpath)
569         except ScriptError, e:
570             _log.info(e)
571             output = ""
572
573         if (not output) or (output.upper().rstrip().endswith(
574             'NO SUCH FILE OR DIRECTORY')):
575             _log.info('  No base file: "%s"', baseline_fullpath)
576             return
577         base_file = get_result_file_fullpath(self._options.html_directory,
578                                              baseline_filename, self._platform,
579                                              'old')
580         # We should be using an explicit encoding here.
581         with open(base_file, "wb") as file:
582             file.write(output)
583         _log.info('  Html: created old baseline file: "%s".',
584                   base_file)
585
586         # Get the diff between old and new baselines and save to the html dir.
587         if baseline_filename.upper().endswith('.TXT'):
588             output = self._scm.diff_for_file(baseline_fullpath, log=_log)
589             if output:
590                 diff_file = get_result_file_fullpath(
591                     self._options.html_directory, baseline_filename,
592                     self._platform, 'diff')
593                 with open(diff_file, 'wb') as file:
594                     file.write(output)
595                 _log.info('  Html: created baseline diff file: "%s".',
596                           diff_file)
597
598
599 class HtmlGenerator(object):
600     """Class to generate rebaselining result comparison html."""
601
602     HTML_REBASELINE = ('<html>'
603                        '<head>'
604                        '<style>'
605                        'body {font-family: sans-serif;}'
606                        '.mainTable {background: #666666;}'
607                        '.mainTable td , .mainTable th {background: white;}'
608                        '.detail {margin-left: 10px; margin-top: 3px;}'
609                        '</style>'
610                        '<title>Rebaselining Result Comparison (%(time)s)'
611                        '</title>'
612                        '</head>'
613                        '<body>'
614                        '<h2>Rebaselining Result Comparison (%(time)s)</h2>'
615                        '%(body)s'
616                        '</body>'
617                        '</html>')
618     HTML_NO_REBASELINING_TESTS = (
619         '<p>No tests found that need rebaselining.</p>')
620     HTML_TABLE_TEST = ('<table class="mainTable" cellspacing=1 cellpadding=5>'
621                        '%s</table><br>')
622     HTML_TR_TEST = ('<tr>'
623                     '<th style="background-color: #CDECDE; border-bottom: '
624                     '1px solid black; font-size: 18pt; font-weight: bold" '
625                     'colspan="5">'
626                     '<a href="%s">%s</a>'
627                     '</th>'
628                     '</tr>')
629     HTML_TEST_DETAIL = ('<div class="detail">'
630                         '<tr>'
631                         '<th width="100">Baseline</th>'
632                         '<th width="100">Platform</th>'
633                         '<th width="200">Old</th>'
634                         '<th width="200">New</th>'
635                         '<th width="150">Difference</th>'
636                         '</tr>'
637                         '%s'
638                         '</div>')
639     HTML_TD_NOLINK = '<td align=center><a>%s</a></td>'
640     HTML_TD_LINK = '<td align=center><a href="%(uri)s">%(name)s</a></td>'
641     HTML_TD_LINK_IMG = ('<td><a href="%(uri)s">'
642                         '<img style="width: 200" src="%(uri)s" /></a></td>')
643     HTML_TR = '<tr>%s</tr>'
644
645     def __init__(self, target_port, options, platforms, rebaselining_tests,
646                  executive):
647         self._html_directory = options.html_directory
648         self._target_port = target_port
649         self._platforms = platforms
650         self._rebaselining_tests = rebaselining_tests
651         self._executive = executive
652         self._html_file = os.path.join(options.html_directory,
653                                        'rebaseline.html')
654
655     def abspath_to_uri(self, filename):
656         """Converts an absolute path to a file: URI."""
657         return path.abspath_to_uri(filename, self._executive)
658
659     def generate_html(self):
660         """Generate html file for rebaselining result comparison."""
661
662         _log.info('Generating html file')
663
664         html_body = ''
665         if not self._rebaselining_tests:
666             html_body += self.HTML_NO_REBASELINING_TESTS
667         else:
668             tests = list(self._rebaselining_tests)
669             tests.sort()
670
671             test_no = 1
672             for test in tests:
673                 _log.info('Test %d: %s', test_no, test)
674                 html_body += self._generate_html_for_one_test(test)
675
676         html = self.HTML_REBASELINE % ({'time': time.asctime(),
677                                         'body': html_body})
678         _log.debug(html)
679
680         with codecs.open(self._html_file, "w", "utf-8") as file:
681             file.write(html)
682
683         _log.info('Baseline comparison html generated at "%s"',
684                   self._html_file)
685
686     def show_html(self):
687         """Launch the rebaselining html in brwoser."""
688
689         _log.info('Launching html: "%s"', self._html_file)
690         user.User().open_url(self._html_file)
691         _log.info('Html launched.')
692
693     def _generate_baseline_links(self, test_basename, suffix, platform):
694         """Generate links for baseline results (old, new and diff).
695
696         Args:
697           test_basename: base filename of the test
698           suffix: baseline file suffixes: '.txt', '.png'
699           platform: win, linux or mac
700
701         Returns:
702           html links for showing baseline results (old, new and diff)
703         """
704
705         baseline_filename = '%s-expected%s' % (test_basename, suffix)
706         _log.debug('    baseline filename: "%s"', baseline_filename)
707
708         new_file = get_result_file_fullpath(self._html_directory,
709                                             baseline_filename, platform, 'new')
710         _log.info('    New baseline file: "%s"', new_file)
711         if not os.path.exists(new_file):
712             _log.info('    No new baseline file: "%s"', new_file)
713             return ''
714
715         old_file = get_result_file_fullpath(self._html_directory,
716                                             baseline_filename, platform, 'old')
717         _log.info('    Old baseline file: "%s"', old_file)
718         if suffix == '.png':
719             html_td_link = self.HTML_TD_LINK_IMG
720         else:
721             html_td_link = self.HTML_TD_LINK
722
723         links = ''
724         if os.path.exists(old_file):
725             links += html_td_link % {
726                 'uri': self.abspath_to_uri(old_file),
727                 'name': baseline_filename}
728         else:
729             _log.info('    No old baseline file: "%s"', old_file)
730             links += self.HTML_TD_NOLINK % ''
731
732         links += html_td_link % {'uri': self.abspath_to_uri(new_file),
733                                  'name': baseline_filename}
734
735         diff_file = get_result_file_fullpath(self._html_directory,
736                                              baseline_filename, platform,
737                                              'diff')
738         _log.info('    Baseline diff file: "%s"', diff_file)
739         if os.path.exists(diff_file):
740             links += html_td_link % {'uri': self.abspath_to_uri(diff_file),
741                                      'name': 'Diff'}
742         else:
743             _log.info('    No baseline diff file: "%s"', diff_file)
744             links += self.HTML_TD_NOLINK % ''
745
746         return links
747
748     def _generate_html_for_one_test(self, test):
749         """Generate html for one rebaselining test.
750
751         Args:
752           test: layout test name
753
754         Returns:
755           html that compares baseline results for the test.
756         """
757
758         test_basename = os.path.basename(os.path.splitext(test)[0])
759         _log.info('  basename: "%s"', test_basename)
760         rows = []
761         for suffix in BASELINE_SUFFIXES:
762             if suffix == '.checksum':
763                 continue
764
765             _log.info('  Checking %s files', suffix)
766             for platform in self._platforms:
767                 links = self._generate_baseline_links(test_basename, suffix,
768                     platform)
769                 if links:
770                     row = self.HTML_TD_NOLINK % self._get_baseline_result_type(
771                         suffix)
772                     row += self.HTML_TD_NOLINK % platform
773                     row += links
774                     _log.debug('    html row: %s', row)
775
776                     rows.append(self.HTML_TR % row)
777
778         if rows:
779             test_path = os.path.join(self._target_port.layout_tests_dir(),
780                                      test)
781             html = self.HTML_TR_TEST % (self.abspath_to_uri(test_path), test)
782             html += self.HTML_TEST_DETAIL % ' '.join(rows)
783
784             _log.debug('    html for test: %s', html)
785             return self.HTML_TABLE_TEST % html
786
787         return ''
788
789     def _get_baseline_result_type(self, suffix):
790         """Name of the baseline result type."""
791
792         if suffix == '.png':
793             return 'Pixel'
794         elif suffix == '.txt':
795             return 'Render Tree'
796         else:
797             return 'Other'
798
799
800 def get_host_port_object(options):
801     """Return a port object for the platform we're running on."""
802     # The only thing we really need on the host is a way to diff
803     # text files and image files, which means we need to check that some
804     # version of ImageDiff has been built. We will look for either Debug
805     # or Release versions of the default port on the platform.
806     options.configuration = "Release"
807     port_obj = port.get(None, options)
808     if not port_obj.check_image_diff(override_step=None, logging=False):
809         _log.debug('No release version of the image diff binary was found.')
810         options.configuration = "Debug"
811         port_obj = port.get(None, options)
812         if not port_obj.check_image_diff(override_step=None, logging=False):
813             _log.error('No version of image diff was found. Check your build.')
814             return None
815         else:
816             _log.debug('Found the debug version of the image diff binary.')
817     else:
818         _log.debug('Found the release version of the image diff binary.')
819     return port_obj
820
821
822 def parse_options(args):
823     """Parse options and return a pair of host options and target options."""
824     option_parser = optparse.OptionParser()
825     option_parser.add_option('-v', '--verbose',
826                              action='store_true',
827                              default=False,
828                              help='include debug-level logging.')
829
830     option_parser.add_option('-q', '--quiet',
831                              action='store_true',
832                              help='Suppress result HTML viewing')
833
834     option_parser.add_option('-p', '--platforms',
835                              default='mac,win,win-xp,win-vista,linux',
836                              help=('Comma delimited list of platforms '
837                                    'that need rebaselining.'))
838
839     option_parser.add_option('-u', '--archive_url',
840                              default=('http://build.chromium.org/buildbot/'
841                                       'layout_test_results'),
842                              help=('Url to find the layout test result archive'
843                                    ' file.'))
844     option_parser.add_option('-U', '--force_archive_url',
845                              help=('Url of result zip file. This option is for debugging '
846                                    'purposes'))
847
848     option_parser.add_option('-w', '--webkit_canary',
849                              action='store_true',
850                              default=False,
851                              help=('If True, pull baselines from webkit.org '
852                                    'canary bot.'))
853
854     option_parser.add_option('-b', '--backup',
855                              action='store_true',
856                              default=False,
857                              help=('Whether or not to backup the original test'
858                                    ' expectations file after rebaseline.'))
859
860     option_parser.add_option('-d', '--html_directory',
861                              default='',
862                              help=('The directory that stores the results for '
863                                    'rebaselining comparison.'))
864
865     option_parser.add_option('', '--use_drt',
866                              action='store_true',
867                              default=False,
868                              help=('Use ImageDiff from DumpRenderTree instead '
869                                    'of image_diff for pixel tests.'))
870
871     option_parser.add_option('', '--target-platform',
872                              default='chromium',
873                              help=('The target platform to rebaseline '
874                                    '("mac", "chromium", "qt", etc.). Defaults '
875                                    'to "chromium".'))
876     options = option_parser.parse_args(args)[0]
877
878     target_options = copy.copy(options)
879     if options.target_platform == 'chromium':
880         target_options.chromium = True
881     options.tolerance = 0
882
883     return (options, target_options)
884
885
886 def main(executive=Executive()):
887     """Main function to produce new baselines."""
888
889     (options, target_options) = parse_options(sys.argv[1:])
890
891     # We need to create three different Port objects over the life of this
892     # script. |target_port_obj| is used to determine configuration information:
893     # location of the expectations file, names of ports to rebaseline, etc.
894     # |port_obj| is used for runtime functionality like actually diffing
895     # Then we create a rebaselining port to actual find and manage the
896     # baselines.
897     target_port_obj = port.get(None, target_options)
898
899     # Set up our logging format.
900     log_level = logging.INFO
901     if options.verbose:
902         log_level = logging.DEBUG
903     logging.basicConfig(level=log_level,
904                         format=('%(asctime)s %(filename)s:%(lineno)-3d '
905                                 '%(levelname)s %(message)s'),
906                         datefmt='%y%m%d %H:%M:%S')
907
908     host_port_obj = get_host_port_object(options)
909     if not host_port_obj:
910         sys.exit(1)
911
912     # Verify 'platforms' option is valid.
913     if not options.platforms:
914         _log.error('Invalid "platforms" option. --platforms must be '
915                    'specified in order to rebaseline.')
916         sys.exit(1)
917     platforms = [p.strip().lower() for p in options.platforms.split(',')]
918     for platform in platforms:
919         if not platform in REBASELINE_PLATFORM_ORDER:
920             _log.error('Invalid platform: "%s"' % (platform))
921             sys.exit(1)
922
923     # Adjust the platform order so rebaseline tool is running at the order of
924     # 'mac', 'win' and 'linux'. This is in same order with layout test baseline
925     # search paths. It simplifies how the rebaseline tool detects duplicate
926     # baselines. Check _IsDupBaseline method for details.
927     rebaseline_platforms = []
928     for platform in REBASELINE_PLATFORM_ORDER:
929         if platform in platforms:
930             rebaseline_platforms.append(platform)
931
932     options.html_directory = setup_html_directory(options.html_directory)
933
934     rebaselining_tests = set()
935     backup = options.backup
936     for platform in rebaseline_platforms:
937         rebaseliner = Rebaseliner(host_port_obj, target_port_obj,
938                                   platform, options)
939
940         _log.info('')
941         log_dashed_string('Rebaseline started', platform)
942         if rebaseliner.run(backup):
943             # Only need to backup one original copy of test expectation file.
944             backup = False
945             log_dashed_string('Rebaseline done', platform)
946         else:
947             log_dashed_string('Rebaseline failed', platform, logging.ERROR)
948
949         rebaselining_tests |= set(rebaseliner.get_rebaselining_tests())
950
951     _log.info('')
952     log_dashed_string('Rebaselining result comparison started', None)
953     html_generator = HtmlGenerator(target_port_obj,
954                                    options,
955                                    rebaseline_platforms,
956                                    rebaselining_tests,
957                                    executive=executive)
958     html_generator.generate_html()
959     if not options.quiet:
960         html_generator.show_html()
961     log_dashed_string('Rebaselining result comparison done', None)
962
963     sys.exit(0)
964
965 if '__main__' == __name__:
966     main()