2010-08-24 Dirk Pranke <dpranke@chromium.org>
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / rebaseline_chromium_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Rebaselining tool that automatically produces baselines for all platforms.
31
32 The script does the following for each platform specified:
33   1. Compile a list of tests that need rebaselining.
34   2. Download test result archive from buildbot for the platform.
35   3. Extract baselines from the archive file for all identified files.
36   4. Add new baselines to SVN repository.
37   5. For each test that has been rebaselined, remove this platform option from
38      the test in test_expectation.txt. If no other platforms remain after
39      removal, delete the rebaselined test from the file.
40
41 At the end, the script generates a html that compares old and new baselines.
42 """
43
44 from __future__ import with_statement
45
46 import codecs
47 import copy
48 import logging
49 import optparse
50 import os
51 import re
52 import shutil
53 import subprocess
54 import sys
55 import tempfile
56 import time
57 import urllib
58 import webbrowser
59 import zipfile
60
61 from webkitpy.common.system.executive import run_command, ScriptError
62 import webkitpy.common.checkout.scm as scm
63
64 import port
65 from layout_package import test_expectations
66
67 _log = logging.getLogger("webkitpy.layout_tests."
68                          "rebaseline_chromium_webkit_tests")
69
70 BASELINE_SUFFIXES = ['.txt', '.png', '.checksum']
71 REBASELINE_PLATFORM_ORDER = ['mac', 'win', 'win-xp', 'win-vista', 'linux']
72 ARCHIVE_DIR_NAME_DICT = {'win': 'webkit-rel',
73                          'win-vista': 'webkit-dbg-vista',
74                          'win-xp': 'webkit-rel',
75                          'mac': 'webkit-rel-mac5',
76                          'linux': 'webkit-rel-linux',
77                          'win-canary': 'webkit-rel-webkit-org',
78                          'win-vista-canary': 'webkit-dbg-vista',
79                          'win-xp-canary': 'webkit-rel-webkit-org',
80                          'mac-canary': 'webkit-rel-mac-webkit-org',
81                          'linux-canary': 'webkit-rel-linux-webkit-org'}
82
83
84 # FIXME: Should be rolled into webkitpy.Executive
85 def run_shell_with_return_code(command, print_output=False):
86     """Executes a command and returns the output and process return code.
87
88     Args:
89       command: program and arguments.
90       print_output: if true, print the command results to standard output.
91
92     Returns:
93       command output, return code
94     """
95
96     # Use a shell for subcommands on Windows to get a PATH search.
97     # FIXME: shell=True is a trail of tears, and should be removed.
98     use_shell = sys.platform.startswith('win')
99     # Note: Not thread safe: http://bugs.python.org/issue2320
100     p = subprocess.Popen(command, stdout=subprocess.PIPE,
101                          stderr=subprocess.STDOUT, shell=use_shell)
102     if print_output:
103         output_array = []
104         while True:
105             line = p.stdout.readline()
106             if not line:
107                 break
108             if print_output:
109                 print line.strip('\n')
110             output_array.append(line)
111         output = ''.join(output_array)
112     else:
113         output = p.stdout.read()
114     p.wait()
115     p.stdout.close()
116
117     return output, p.returncode
118
119
120 # FIXME: Should be rolled into webkitpy.Executive
121 def run_shell(command, print_output=False):
122     """Executes a command and returns the output.
123
124     Args:
125       command: program and arguments.
126       print_output: if true, print the command results to standard output.
127
128     Returns:
129       command output
130     """
131
132     output, return_code = run_shell_with_return_code(command, print_output)
133     return output
134
135
136 def log_dashed_string(text, platform, logging_level=logging.INFO):
137     """Log text message with dashes on both sides."""
138
139     msg = text
140     if platform:
141         msg += ': ' + platform
142     if len(msg) < 78:
143         dashes = '-' * ((78 - len(msg)) / 2)
144         msg = '%s %s %s' % (dashes, msg, dashes)
145
146     if logging_level == logging.ERROR:
147         _log.error(msg)
148     elif logging_level == logging.WARNING:
149         _log.warn(msg)
150     else:
151         _log.info(msg)
152
153
154 def setup_html_directory(html_directory):
155     """Setup the directory to store html results.
156
157        All html related files are stored in the "rebaseline_html" subdirectory.
158
159     Args:
160       html_directory: parent directory that stores the rebaselining results.
161                       If None, a temp directory is created.
162
163     Returns:
164       the directory that stores the html related rebaselining results.
165     """
166
167     if not html_directory:
168         html_directory = tempfile.mkdtemp()
169     elif not os.path.exists(html_directory):
170         os.mkdir(html_directory)
171
172     html_directory = os.path.join(html_directory, 'rebaseline_html')
173     _log.info('Html directory: "%s"', html_directory)
174
175     if os.path.exists(html_directory):
176         shutil.rmtree(html_directory, True)
177         _log.info('Deleted file at html directory: "%s"', html_directory)
178
179     if not os.path.exists(html_directory):
180         os.mkdir(html_directory)
181     return html_directory
182
183
184 def get_result_file_fullpath(html_directory, baseline_filename, platform,
185                              result_type):
186     """Get full path of the baseline result file.
187
188     Args:
189       html_directory: directory that stores the html related files.
190       baseline_filename: name of the baseline file.
191       platform: win, linux or mac
192       result_type: type of the baseline result: '.txt', '.png'.
193
194     Returns:
195       Full path of the baseline file for rebaselining result comparison.
196     """
197
198     base, ext = os.path.splitext(baseline_filename)
199     result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext)
200     fullpath = os.path.join(html_directory, result_filename)
201     _log.debug('  Result file full path: "%s".', fullpath)
202     return fullpath
203
204
205 class Rebaseliner(object):
206     """Class to produce new baselines for a given platform."""
207
208     REVISION_REGEX = r'<a href=\"(\d+)/\">'
209
210     def __init__(self, running_port, target_port, platform, options):
211         """
212         Args:
213             running_port: the Port the script is running on.
214             target_port: the Port the script uses to find port-specific
215                 configuration information like the test_expectations.txt
216                 file location and the list of test platforms.
217             platform: the test platform to rebaseline
218             options: the command-line options object."""
219         self._platform = platform
220         self._options = options
221         self._port = running_port
222         self._target_port = target_port
223         self._rebaseline_port = port.get(
224             self._target_port.test_platform_name_to_name(platform), options)
225         self._rebaselining_tests = []
226         self._rebaselined_tests = []
227
228         # Create tests and expectations helper which is used to:
229         #   -. compile list of tests that need rebaselining.
230         #   -. update the tests in test_expectations file after rebaseline
231         #      is done.
232         expectations_str = self._rebaseline_port.test_expectations()
233         self._test_expectations = \
234             test_expectations.TestExpectations(self._rebaseline_port,
235                                                None,
236                                                expectations_str,
237                                                self._platform,
238                                                False,
239                                                False)
240         self._scm = scm.default_scm()
241
242     def run(self, backup):
243         """Run rebaseline process."""
244
245         log_dashed_string('Compiling rebaselining tests', self._platform)
246         if not self._compile_rebaselining_tests():
247             return True
248
249         log_dashed_string('Downloading archive', self._platform)
250         archive_file = self._download_buildbot_archive()
251         _log.info('')
252         if not archive_file:
253             _log.error('No archive found.')
254             return False
255
256         log_dashed_string('Extracting and adding new baselines',
257                           self._platform)
258         if not self._extract_and_add_new_baselines(archive_file):
259             return False
260
261         log_dashed_string('Updating rebaselined tests in file',
262                           self._platform)
263         self._update_rebaselined_tests_in_file(backup)
264         _log.info('')
265
266         if len(self._rebaselining_tests) != len(self._rebaselined_tests):
267             _log.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN '
268                          'REBASELINED.')
269             _log.warning('  Total tests needing rebaselining: %d',
270                          len(self._rebaselining_tests))
271             _log.warning('  Total tests rebaselined: %d',
272                          len(self._rebaselined_tests))
273             return False
274
275         _log.warning('All tests needing rebaselining were successfully '
276                      'rebaselined.')
277
278         return True
279
280     def get_rebaselining_tests(self):
281         return self._rebaselining_tests
282
283     def _compile_rebaselining_tests(self):
284         """Compile list of tests that need rebaselining for the platform.
285
286         Returns:
287           List of tests that need rebaselining or
288           None if there is no such test.
289         """
290
291         self._rebaselining_tests = \
292             self._test_expectations.get_rebaselining_failures()
293         if not self._rebaselining_tests:
294             _log.warn('No tests found that need rebaselining.')
295             return None
296
297         _log.info('Total number of tests needing rebaselining '
298                   'for "%s": "%d"', self._platform,
299                   len(self._rebaselining_tests))
300
301         test_no = 1
302         for test in self._rebaselining_tests:
303             _log.info('  %d: %s', test_no, test)
304             test_no += 1
305
306         return self._rebaselining_tests
307
308     def _get_latest_revision(self, url):
309         """Get the latest layout test revision number from buildbot.
310
311         Args:
312           url: Url to retrieve layout test revision numbers.
313
314         Returns:
315           latest revision or
316           None on failure.
317         """
318
319         _log.debug('Url to retrieve revision: "%s"', url)
320
321         f = urllib.urlopen(url)
322         content = f.read()
323         f.close()
324
325         revisions = re.findall(self.REVISION_REGEX, content)
326         if not revisions:
327             _log.error('Failed to find revision, content: "%s"', content)
328             return None
329
330         revisions.sort(key=int)
331         _log.info('Latest revision: "%s"', revisions[len(revisions) - 1])
332         return revisions[len(revisions) - 1]
333
334     def _get_archive_dir_name(self, platform, webkit_canary):
335         """Get name of the layout test archive directory.
336
337         Returns:
338           Directory name or
339           None on failure
340         """
341
342         if webkit_canary:
343             platform += '-canary'
344
345         if platform in ARCHIVE_DIR_NAME_DICT:
346             return ARCHIVE_DIR_NAME_DICT[platform]
347         else:
348             _log.error('Cannot find platform key %s in archive '
349                        'directory name dictionary', platform)
350             return None
351
352     def _get_archive_url(self):
353         """Generate the url to download latest layout test archive.
354
355         Returns:
356           Url to download archive or
357           None on failure
358         """
359
360         if self._options.force_archive_url:
361             return self._options.force_archive_url
362
363         dir_name = self._get_archive_dir_name(self._platform,
364                                               self._options.webkit_canary)
365         if not dir_name:
366             return None
367
368         _log.debug('Buildbot platform dir name: "%s"', dir_name)
369
370         url_base = '%s/%s/' % (self._options.archive_url, dir_name)
371         latest_revision = self._get_latest_revision(url_base)
372         if latest_revision is None or latest_revision <= 0:
373             return None
374         archive_url = ('%s%s/layout-test-results.zip' % (url_base,
375                                                          latest_revision))
376         _log.info('Archive url: "%s"', archive_url)
377         return archive_url
378
379     def _download_buildbot_archive(self):
380         """Download layout test archive file from buildbot.
381
382         Returns:
383           True if download succeeded or
384           False otherwise.
385         """
386
387         url = self._get_archive_url()
388         if url is None:
389             return None
390
391         fn = urllib.urlretrieve(url)[0]
392         _log.info('Archive downloaded and saved to file: "%s"', fn)
393         return fn
394
395     def _extract_and_add_new_baselines(self, archive_file):
396         """Extract new baselines from archive and add them to SVN repository.
397
398         Args:
399           archive_file: full path to the archive file.
400
401         Returns:
402           List of tests that have been rebaselined or
403           None on failure.
404         """
405
406         zip_file = zipfile.ZipFile(archive_file, 'r')
407         zip_namelist = zip_file.namelist()
408
409         _log.debug('zip file namelist:')
410         for name in zip_namelist:
411             _log.debug('  ' + name)
412
413         platform = self._rebaseline_port.test_platform_name_to_name(
414             self._platform)
415         _log.debug('Platform dir: "%s"', platform)
416
417         test_no = 1
418         self._rebaselined_tests = []
419         for test in self._rebaselining_tests:
420             _log.info('Test %d: %s', test_no, test)
421
422             found = False
423             scm_error = False
424             test_basename = os.path.splitext(test)[0]
425             for suffix in BASELINE_SUFFIXES:
426                 archive_test_name = ('layout-test-results/%s-actual%s' %
427                                       (test_basename, suffix))
428                 _log.debug('  Archive test file name: "%s"',
429                            archive_test_name)
430                 if not archive_test_name in zip_namelist:
431                     _log.info('  %s file not in archive.', suffix)
432                     continue
433
434                 found = True
435                 _log.info('  %s file found in archive.', suffix)
436
437                 # Extract new baseline from archive and save it to a temp file.
438                 data = zip_file.read(archive_test_name)
439                 temp_fd, temp_name = tempfile.mkstemp(suffix)
440                 f = os.fdopen(temp_fd, 'wb')
441                 f.write(data)
442                 f.close()
443
444                 expected_filename = '%s-expected%s' % (test_basename, suffix)
445                 expected_fullpath = os.path.join(
446                     self._rebaseline_port.baseline_path(), expected_filename)
447                 expected_fullpath = os.path.normpath(expected_fullpath)
448                 _log.debug('  Expected file full path: "%s"',
449                            expected_fullpath)
450
451                 # TODO(victorw): for now, the rebaselining tool checks whether
452                 # or not THIS baseline is duplicate and should be skipped.
453                 # We could improve the tool to check all baselines in upper
454                 # and lower
455                 # levels and remove all duplicated baselines.
456                 if self._is_dup_baseline(temp_name,
457                                         expected_fullpath,
458                                         test,
459                                         suffix,
460                                         self._platform):
461                     os.remove(temp_name)
462                     self._delete_baseline(expected_fullpath)
463                     continue
464
465                 # Create the new baseline directory if it doesn't already
466                 # exist.
467                 self._port.maybe_make_directory(
468                     os.path.dirname(expected_fullpath))
469
470                 shutil.move(temp_name, expected_fullpath)
471
472                 if 0 != self._scm.add(expected_fullpath, return_exit_code=True):
473                     # FIXME: print detailed diagnose messages
474                     scm_error = True
475                 elif suffix != '.checksum':
476                     self._create_html_baseline_files(expected_fullpath)
477
478             if not found:
479                 _log.warn('  No new baselines found in archive.')
480             else:
481                 if scm_error:
482                     _log.warn('  Failed to add baselines to your repository.')
483                 else:
484                     _log.info('  Rebaseline succeeded.')
485                     self._rebaselined_tests.append(test)
486
487             test_no += 1
488
489         zip_file.close()
490         os.remove(archive_file)
491
492         return self._rebaselined_tests
493
494     def _is_dup_baseline(self, new_baseline, baseline_path, test, suffix,
495                          platform):
496         """Check whether a baseline is duplicate and can fallback to same
497            baseline for another platform. For example, if a test has same
498            baseline on linux and windows, then we only store windows
499            baseline and linux baseline will fallback to the windows version.
500
501         Args:
502           expected_filename: baseline expectation file name.
503           test: test name.
504           suffix: file suffix of the expected results, including dot;
505                   e.g. '.txt' or '.png'.
506           platform: baseline platform 'mac', 'win' or 'linux'.
507
508         Returns:
509           True if the baseline is unnecessary.
510           False otherwise.
511         """
512         test_filepath = os.path.join(self._target_port.layout_tests_dir(),
513                                      test)
514         all_baselines = self._rebaseline_port.expected_baselines(
515             test_filepath, suffix, True)
516         for (fallback_dir, fallback_file) in all_baselines:
517             if fallback_dir and fallback_file:
518                 fallback_fullpath = os.path.normpath(
519                     os.path.join(fallback_dir, fallback_file))
520                 if fallback_fullpath.lower() != baseline_path.lower():
521                     if not self._diff_baselines(new_baseline,
522                                                 fallback_fullpath):
523                         _log.info('  Found same baseline at %s',
524                                   fallback_fullpath)
525                         return True
526                     else:
527                         return False
528
529         return False
530
531     def _diff_baselines(self, file1, file2):
532         """Check whether two baselines are different.
533
534         Args:
535           file1, file2: full paths of the baselines to compare.
536
537         Returns:
538           True if two files are different or have different extensions.
539           False otherwise.
540         """
541
542         ext1 = os.path.splitext(file1)[1].upper()
543         ext2 = os.path.splitext(file2)[1].upper()
544         if ext1 != ext2:
545             _log.warn('Files to compare have different ext. '
546                       'File1: %s; File2: %s', file1, file2)
547             return True
548
549         if ext1 == '.PNG':
550             return self._port.diff_image(file1, file2)
551         else:
552             with codecs.open(file1, "r", "utf8") as file_handle1:
553                 output1 = file_handle1.read()
554             with codecs.open(file2, "r", "utf8") as file_handle2:
555                 output2 = file_handle2.read()
556             return self._port.compare_text(output1, output2)
557
558     def _delete_baseline(self, filename):
559         """Remove the file from repository and delete it from disk.
560
561         Args:
562           filename: full path of the file to delete.
563         """
564
565         if not filename or not os.path.isfile(filename):
566             return
567         self._scm.delete(filename)
568
569     def _update_rebaselined_tests_in_file(self, backup):
570         """Update the rebaselined tests in test expectations file.
571
572         Args:
573           backup: if True, backup the original test expectations file.
574
575         Returns:
576           no
577         """
578
579         if self._rebaselined_tests:
580             new_expectations = (
581                 self._test_expectations.remove_platform_from_expectations(
582                 self._rebaselined_tests, self._platform))
583             path = self._target_port.path_to_test_expectations_file()
584             if backup:
585                 date_suffix = time.strftime('%Y%m%d%H%M%S',
586                                             time.localtime(time.time()))
587                 backup_file = ('%s.orig.%s' % (path, date_suffix))
588                 if os.path.exists(backup_file):
589                     os.remove(backup_file)
590                 _log.info('Saving original file to "%s"', backup_file)
591                 os.rename(path, backup_file)
592             # FIXME: What encoding are these files?
593             # Or is new_expectations always a byte array?
594             with open(path, "w") as file:
595                 file.write(new_expectations)
596             self._scm.add(path)
597         else:
598             _log.info('No test was rebaselined so nothing to remove.')
599
600     def _create_html_baseline_files(self, baseline_fullpath):
601         """Create baseline files (old, new and diff) in html directory.
602
603            The files are used to compare the rebaselining results.
604
605         Args:
606           baseline_fullpath: full path of the expected baseline file.
607         """
608
609         if not baseline_fullpath or not os.path.exists(baseline_fullpath):
610             return
611
612         # Copy the new baseline to html directory for result comparison.
613         baseline_filename = os.path.basename(baseline_fullpath)
614         new_file = get_result_file_fullpath(self._options.html_directory,
615                                             baseline_filename, self._platform,
616                                             'new')
617         shutil.copyfile(baseline_fullpath, new_file)
618         _log.info('  Html: copied new baseline file from "%s" to "%s".',
619                   baseline_fullpath, new_file)
620
621         # Get the old baseline from the repository and save to the html directory.
622         try:
623             output = self._scm.show_head(baseline_fullpath)
624         except ScriptError, e:
625             _log.info(e)
626             output = ""
627
628         if (not output) or (output.upper().rstrip().endswith(
629             'NO SUCH FILE OR DIRECTORY')):
630             _log.info('  No base file: "%s"', baseline_fullpath)
631             return
632         base_file = get_result_file_fullpath(self._options.html_directory,
633                                              baseline_filename, self._platform,
634                                              'old')
635         # FIXME: This assumes run_shell returns a byte array.
636         # We should be using an explicit encoding here.
637         with open(base_file, "wb") as file:
638             file.write(output)
639         _log.info('  Html: created old baseline file: "%s".',
640                   base_file)
641
642         # Get the diff between old and new baselines and save to the html dir.
643         if baseline_filename.upper().endswith('.TXT'):
644             output = self._scm.diff_for_file(baseline_fullpath, log=_log)
645             if output:
646                 diff_file = get_result_file_fullpath(
647                     self._options.html_directory, baseline_filename,
648                     self._platform, 'diff')
649                 # FIXME: This assumes run_shell returns a byte array, not unicode()
650                 with open(diff_file, 'wb') as file:
651                     file.write(output)
652                 _log.info('  Html: created baseline diff file: "%s".',
653                           diff_file)
654
655 class HtmlGenerator(object):
656     """Class to generate rebaselining result comparison html."""
657
658     HTML_REBASELINE = ('<html>'
659                        '<head>'
660                        '<style>'
661                        'body {font-family: sans-serif;}'
662                        '.mainTable {background: #666666;}'
663                        '.mainTable td , .mainTable th {background: white;}'
664                        '.detail {margin-left: 10px; margin-top: 3px;}'
665                        '</style>'
666                        '<title>Rebaselining Result Comparison (%(time)s)'
667                        '</title>'
668                        '</head>'
669                        '<body>'
670                        '<h2>Rebaselining Result Comparison (%(time)s)</h2>'
671                        '%(body)s'
672                        '</body>'
673                        '</html>')
674     HTML_NO_REBASELINING_TESTS = (
675         '<p>No tests found that need rebaselining.</p>')
676     HTML_TABLE_TEST = ('<table class="mainTable" cellspacing=1 cellpadding=5>'
677                        '%s</table><br>')
678     HTML_TR_TEST = ('<tr>'
679                     '<th style="background-color: #CDECDE; border-bottom: '
680                     '1px solid black; font-size: 18pt; font-weight: bold" '
681                     'colspan="5">'
682                     '<a href="%s">%s</a>'
683                     '</th>'
684                     '</tr>')
685     HTML_TEST_DETAIL = ('<div class="detail">'
686                         '<tr>'
687                         '<th width="100">Baseline</th>'
688                         '<th width="100">Platform</th>'
689                         '<th width="200">Old</th>'
690                         '<th width="200">New</th>'
691                         '<th width="150">Difference</th>'
692                         '</tr>'
693                         '%s'
694                         '</div>')
695     HTML_TD_NOLINK = '<td align=center><a>%s</a></td>'
696     HTML_TD_LINK = '<td align=center><a href="%(uri)s">%(name)s</a></td>'
697     HTML_TD_LINK_IMG = ('<td><a href="%(uri)s">'
698                         '<img style="width: 200" src="%(uri)s" /></a></td>')
699     HTML_TR = '<tr>%s</tr>'
700
701     def __init__(self, target_port, options, platforms, rebaselining_tests):
702         self._html_directory = options.html_directory
703         self._target_port = target_port
704         self._platforms = platforms
705         self._rebaselining_tests = rebaselining_tests
706         self._html_file = os.path.join(options.html_directory,
707                                        'rebaseline.html')
708
709     def generate_html(self):
710         """Generate html file for rebaselining result comparison."""
711
712         _log.info('Generating html file')
713
714         html_body = ''
715         if not self._rebaselining_tests:
716             html_body += self.HTML_NO_REBASELINING_TESTS
717         else:
718             tests = list(self._rebaselining_tests)
719             tests.sort()
720
721             test_no = 1
722             for test in tests:
723                 _log.info('Test %d: %s', test_no, test)
724                 html_body += self._generate_html_for_one_test(test)
725
726         html = self.HTML_REBASELINE % ({'time': time.asctime(),
727                                         'body': html_body})
728         _log.debug(html)
729
730         with codecs.open(self._html_file, "w", "utf-8") as file:
731             file.write(html)
732
733         _log.info('Baseline comparison html generated at "%s"',
734                   self._html_file)
735
736     def show_html(self):
737         """Launch the rebaselining html in brwoser."""
738
739         _log.info('Launching html: "%s"', self._html_file)
740
741         html_uri = self._target_port.filename_to_uri(self._html_file)
742         webbrowser.open(html_uri, 1)
743
744         _log.info('Html launched.')
745
746     def _generate_baseline_links(self, test_basename, suffix, platform):
747         """Generate links for baseline results (old, new and diff).
748
749         Args:
750           test_basename: base filename of the test
751           suffix: baseline file suffixes: '.txt', '.png'
752           platform: win, linux or mac
753
754         Returns:
755           html links for showing baseline results (old, new and diff)
756         """
757
758         baseline_filename = '%s-expected%s' % (test_basename, suffix)
759         _log.debug('    baseline filename: "%s"', baseline_filename)
760
761         new_file = get_result_file_fullpath(self._html_directory,
762                                             baseline_filename, platform, 'new')
763         _log.info('    New baseline file: "%s"', new_file)
764         if not os.path.exists(new_file):
765             _log.info('    No new baseline file: "%s"', new_file)
766             return ''
767
768         old_file = get_result_file_fullpath(self._html_directory,
769                                             baseline_filename, platform, 'old')
770         _log.info('    Old baseline file: "%s"', old_file)
771         if suffix == '.png':
772             html_td_link = self.HTML_TD_LINK_IMG
773         else:
774             html_td_link = self.HTML_TD_LINK
775
776         links = ''
777         if os.path.exists(old_file):
778             links += html_td_link % {
779                 'uri': self._target_port.filename_to_uri(old_file),
780                 'name': baseline_filename}
781         else:
782             _log.info('    No old baseline file: "%s"', old_file)
783             links += self.HTML_TD_NOLINK % ''
784
785         links += html_td_link % {'uri': self._target_port.filename_to_uri(
786                                      new_file),
787                                  'name': baseline_filename}
788
789         diff_file = get_result_file_fullpath(self._html_directory,
790                                              baseline_filename, platform,
791                                              'diff')
792         _log.info('    Baseline diff file: "%s"', diff_file)
793         if os.path.exists(diff_file):
794             links += html_td_link % {'uri': self._target_port.filename_to_uri(
795                 diff_file), 'name': 'Diff'}
796         else:
797             _log.info('    No baseline diff file: "%s"', diff_file)
798             links += self.HTML_TD_NOLINK % ''
799
800         return links
801
802     def _generate_html_for_one_test(self, test):
803         """Generate html for one rebaselining test.
804
805         Args:
806           test: layout test name
807
808         Returns:
809           html that compares baseline results for the test.
810         """
811
812         test_basename = os.path.basename(os.path.splitext(test)[0])
813         _log.info('  basename: "%s"', test_basename)
814         rows = []
815         for suffix in BASELINE_SUFFIXES:
816             if suffix == '.checksum':
817                 continue
818
819             _log.info('  Checking %s files', suffix)
820             for platform in self._platforms:
821                 links = self._generate_baseline_links(test_basename, suffix,
822                     platform)
823                 if links:
824                     row = self.HTML_TD_NOLINK % self._get_baseline_result_type(
825                         suffix)
826                     row += self.HTML_TD_NOLINK % platform
827                     row += links
828                     _log.debug('    html row: %s', row)
829
830                     rows.append(self.HTML_TR % row)
831
832         if rows:
833             test_path = os.path.join(self._target_port.layout_tests_dir(),
834                                      test)
835             html = self.HTML_TR_TEST % (
836                 self._target_port.filename_to_uri(test_path), test)
837             html += self.HTML_TEST_DETAIL % ' '.join(rows)
838
839             _log.debug('    html for test: %s', html)
840             return self.HTML_TABLE_TEST % html
841
842         return ''
843
844     def _get_baseline_result_type(self, suffix):
845         """Name of the baseline result type."""
846
847         if suffix == '.png':
848             return 'Pixel'
849         elif suffix == '.txt':
850             return 'Render Tree'
851         else:
852             return 'Other'
853
854
855 def get_host_port_object(options):
856     """Return a port object for the platform we're running on."""
857     # The only thing we really need on the host is a way to diff
858     # text files and image files, which means we need to check that some
859     # version of ImageDiff has been built. We will look for either Debug
860     # or Release versions of the default port on the platform.
861     options.configuration = "Release"
862     port_obj = port.get(None, options)
863     if not port_obj.check_image_diff(override_step=None, logging=False):
864         _log.debug('No release version of the image diff binary was found.')
865         options.configuration = "Debug"
866         port_obj = port.get(None, options)
867         if not port_obj.check_image_diff(override_step=None, logging=False):
868             _log.error('No version of image diff was found. Check your build.')
869             return None
870         else:
871             _log.debug('Found the debug version of the image diff binary.')
872     else:
873         _log.debug('Found the release version of the image diff binary.')
874     return port_obj
875
876
877 def main():
878     """Main function to produce new baselines."""
879
880     option_parser = optparse.OptionParser()
881     option_parser.add_option('-v', '--verbose',
882                              action='store_true',
883                              default=False,
884                              help='include debug-level logging.')
885
886     option_parser.add_option('-q', '--quiet',
887                              action='store_true',
888                              help='Suppress result HTML viewing')
889
890     option_parser.add_option('-p', '--platforms',
891                              default='mac,win,win-xp,win-vista,linux',
892                              help=('Comma delimited list of platforms '
893                                    'that need rebaselining.'))
894
895     option_parser.add_option('-u', '--archive_url',
896                              default=('http://build.chromium.org/buildbot/'
897                                       'layout_test_results'),
898                              help=('Url to find the layout test result archive'
899                                    ' file.'))
900     option_parser.add_option('-U', '--force_archive_url',
901                              help=('Url of result zip file. This option is for debugging '
902                                    'purposes'))
903
904     option_parser.add_option('-w', '--webkit_canary',
905                              action='store_true',
906                              default=False,
907                              help=('If True, pull baselines from webkit.org '
908                                    'canary bot.'))
909
910     option_parser.add_option('-b', '--backup',
911                              action='store_true',
912                              default=False,
913                              help=('Whether or not to backup the original test'
914                                    ' expectations file after rebaseline.'))
915
916     option_parser.add_option('-d', '--html_directory',
917                              default='',
918                              help=('The directory that stores the results for '
919                                    'rebaselining comparison.'))
920
921     option_parser.add_option('', '--use_drt',
922                              action='store_true',
923                              default=False,
924                              help=('Use ImageDiff from DumpRenderTree instead '
925                                    'of image_diff for pixel tests.'))
926
927     option_parser.add_option('', '--target-platform',
928                              default='chromium',
929                              help=('The target platform to rebaseline '
930                                    '("mac", "chromium", "qt", etc.). Defaults '
931                                    'to "chromium".'))
932     options = option_parser.parse_args()[0]
933
934     # We need to create three different Port objects over the life of this
935     # script. |target_port_obj| is used to determine configuration information:
936     # location of the expectations file, names of ports to rebaseline, etc.
937     # |port_obj| is used for runtime functionality like actually diffing
938     # Then we create a rebaselining port to actual find and manage the
939     # baselines.
940     target_options = copy.copy(options)
941     if options.target_platform == 'chromium':
942         target_options.chromium = True
943     target_port_obj = port.get(None, target_options)
944
945     # Set up our logging format.
946     log_level = logging.INFO
947     if options.verbose:
948         log_level = logging.DEBUG
949     logging.basicConfig(level=log_level,
950                         format=('%(asctime)s %(filename)s:%(lineno)-3d '
951                                 '%(levelname)s %(message)s'),
952                         datefmt='%y%m%d %H:%M:%S')
953
954     host_port_obj = get_host_port_object(options)
955     if not host_port_obj:
956         sys.exit(1)
957
958     # Verify 'platforms' option is valid.
959     if not options.platforms:
960         _log.error('Invalid "platforms" option. --platforms must be '
961                    'specified in order to rebaseline.')
962         sys.exit(1)
963     platforms = [p.strip().lower() for p in options.platforms.split(',')]
964     for platform in platforms:
965         if not platform in REBASELINE_PLATFORM_ORDER:
966             _log.error('Invalid platform: "%s"' % (platform))
967             sys.exit(1)
968
969     # Adjust the platform order so rebaseline tool is running at the order of
970     # 'mac', 'win' and 'linux'. This is in same order with layout test baseline
971     # search paths. It simplifies how the rebaseline tool detects duplicate
972     # baselines. Check _IsDupBaseline method for details.
973     rebaseline_platforms = []
974     for platform in REBASELINE_PLATFORM_ORDER:
975         if platform in platforms:
976             rebaseline_platforms.append(platform)
977
978     options.html_directory = setup_html_directory(options.html_directory)
979
980     rebaselining_tests = set()
981     backup = options.backup
982     for platform in rebaseline_platforms:
983         rebaseliner = Rebaseliner(host_port_obj, target_port_obj,
984                                   platform, options)
985
986         _log.info('')
987         log_dashed_string('Rebaseline started', platform)
988         if rebaseliner.run(backup):
989             # Only need to backup one original copy of test expectation file.
990             backup = False
991             log_dashed_string('Rebaseline done', platform)
992         else:
993             log_dashed_string('Rebaseline failed', platform, logging.ERROR)
994
995         rebaselining_tests |= set(rebaseliner.get_rebaselining_tests())
996
997     _log.info('')
998     log_dashed_string('Rebaselining result comparison started', None)
999     html_generator = HtmlGenerator(target_port_obj,
1000                                    options,
1001                                    rebaseline_platforms,
1002                                    rebaselining_tests)
1003     html_generator.generate_html()
1004     if not options.quiet:
1005         html_generator.show_html()
1006     log_dashed_string('Rebaselining result comparison done', None)
1007
1008     sys.exit(0)
1009
1010 if '__main__' == __name__:
1011     main()