Tools/Scripts/webkitpy/performance_tests/perftest.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2012 Google Inc. All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30
  31 import errno
  32 import logging
  33 import math
  34 import re
  35 import os
  36 import signal
  37 import socket
  38 import subprocess
  39 import sys
  40 import time
  41
  42 # Import for auto-install
  43 if sys.platform not in ('cygwin', 'win32'):
  44     # FIXME: webpagereplay doesn't work on win32. See https://bugs.webkit.org/show_bug.cgi?id=88279.
  45     import webkitpy.thirdparty.autoinstalled.webpagereplay.replay
  46
  47 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
  48 from webkitpy.layout_tests.port.driver import DriverInput
  49 from webkitpy.layout_tests.port.driver import DriverOutput
  50
  51
  52 _log = logging.getLogger(__name__)
  53
  54
  55 class PerfTest(object):
  56     def __init__(self, port, test_name, path_or_url):
  57         self._port = port
  58         self._test_name = test_name
  59         self._path_or_url = path_or_url
  60
  61     def test_name(self):
  62         return self._test_name
  63
  64     def path_or_url(self):
  65         return self._path_or_url
  66
  67     def prepare(self, time_out_ms):
  68         return True
  69
  70     def run(self, driver, time_out_ms):
  71         output = self.run_single(driver, self.path_or_url(), time_out_ms)
  72         if self.run_failed(output):
  73             return None
  74         return self.parse_output(output)
  75
  76     def run_single(self, driver, path_or_url, time_out_ms, should_run_pixel_test=False):
  77         return driver.run_test(DriverInput(path_or_url, time_out_ms, image_hash=None, should_run_pixel_test=should_run_pixel_test), stop_when_done=False)
  78
  79     def run_failed(self, output):
  80         if output.text == None or output.error:
  81             pass
  82         elif output.timeout:
  83             _log.error('timeout: %s' % self.test_name())
  84         elif output.crash:
  85             _log.error('crash: %s' % self.test_name())
  86         else:
  87             return False
  88
  89         if output.error:
  90             _log.error('error: %s\n%s' % (self.test_name(), output.error))
  91
  92         return True
  93
  94     _lines_to_ignore_in_parser_result = [
  95         re.compile(r'^Running \d+ times$'),
  96         re.compile(r'^Ignoring warm-up '),
  97         re.compile(r'^Info:'),
  98         re.compile(r'^\d+(.\d+)?(\s*(runs\/s|ms|fps))?$'),
  99         # Following are for handle existing test like Dromaeo
 100         re.compile(re.escape("""main frame - has 1 onunload handler(s)""")),
 101         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->-->" - has 1 onunload handler(s)""")),
 102         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->/<!--frame0-->-->" - has 1 onunload handler(s)""")),
 103         # Following is for html5.html
 104         re.compile(re.escape("""Blocked access to external URL http://www.whatwg.org/specs/web-apps/current-work/"""))]
 105
 106     def _should_ignore_line_in_parser_test_result(self, line):
 107         if not line:
 108             return True
 109         for regex in self._lines_to_ignore_in_parser_result:
 110             if regex.search(line):
 111                 return True
 112         return False
 113
 114     _description_regex = re.compile(r'^Description: (?P<description>.*)$', re.IGNORECASE)
 115     _result_classes = ['Time', 'JS Heap', 'Malloc']
 116     _result_class_regex = re.compile(r'^(?P<resultclass>' + r'|'.join(_result_classes) + '):')
 117     _statistics_keys = ['avg', 'median', 'stdev', 'min', 'max', 'unit', 'values']
 118     _score_regex = re.compile(r'^(?P<key>' + r'|'.join(_statistics_keys) + r')\s+(?P<value>([0-9\.]+(,\s+)?)+)\s*(?P<unit>.*)')
 119
 120     def parse_output(self, output):
 121         test_failed = False
 122         results = {}
 123         ordered_results_keys = []
 124         test_name = re.sub(r'\.\w+$', '', self._test_name)
 125         description_string = ""
 126         result_class = ""
 127         for line in re.split('\n', output.text):
 128             description = self._description_regex.match(line)
 129             if description:
 130                 description_string = description.group('description')
 131                 continue
 132
 133             result_class_match = self._result_class_regex.match(line)
 134             if result_class_match:
 135                 result_class = result_class_match.group('resultclass')
 136                 continue
 137
 138             score = self._score_regex.match(line)
 139             if score:
 140                 key = score.group('key')
 141                 if key == 'values':
 142                     value = [float(number) for number in score.group('value').split(', ')]
 143                 else:
 144                     value = float(score.group('value'))
 145                 unit = score.group('unit')
 146                 name = test_name
 147                 if result_class != 'Time':
 148                     name += ':' + result_class.replace(' ', '')
 149                 if name not in ordered_results_keys:
 150                     ordered_results_keys.append(name)
 151                 results.setdefault(name, {})
 152                 results[name]['unit'] = unit
 153                 results[name][key] = value
 154                 continue
 155
 156             if not self._should_ignore_line_in_parser_test_result(line):
 157                 test_failed = True
 158                 _log.error(line)
 159
 160         if test_failed:
 161             return None
 162
 163         if set(self._statistics_keys) != set(results[test_name].keys() + ['values']):
 164             # values is not provided by Dromaeo tests.
 165             _log.error("The test didn't report all statistics.")
 166             return None
 167
 168         for result_name in ordered_results_keys:
 169             if result_name == test_name:
 170                 self.output_statistics(result_name, results[result_name], description_string)
 171             else:
 172                 self.output_statistics(result_name, results[result_name])
 173         return results
 174
 175     def output_statistics(self, test_name, results, description_string=None):
 176         unit = results['unit']
 177         if description_string:
 178             _log.info('DESCRIPTION: %s' % description_string)
 179         _log.info('RESULT %s= %s %s' % (test_name.replace(':', ': ').replace('/', ': '), results['avg'], unit))
 180         _log.info(', '.join(['%s= %s %s' % (key, results[key], unit) for key in self._statistics_keys[1:5]]))
 181
 182
 183 class ChromiumStylePerfTest(PerfTest):
 184     _chromium_style_result_regex = re.compile(r'^RESULT\s+(?P<name>[^=]+)\s*=\s+(?P<value>\d+(\.\d+)?)\s*(?P<unit>\w+)$')
 185
 186     def __init__(self, port, test_name, path_or_url):
 187         super(ChromiumStylePerfTest, self).__init__(port, test_name, path_or_url)
 188
 189     def parse_output(self, output):
 190         test_failed = False
 191         results = {}
 192         for line in re.split('\n', output.text):
 193             resultLine = ChromiumStylePerfTest._chromium_style_result_regex.match(line)
 194             if resultLine:
 195                 # FIXME: Store the unit
 196                 results[self.test_name() + ':' + resultLine.group('name').replace(' ', '')] = float(resultLine.group('value'))
 197                 _log.info(line)
 198             elif not len(line) == 0:
 199                 test_failed = True
 200                 _log.error(line)
 201         return results if results and not test_failed else None
 202
 203
 204 class PageLoadingPerfTest(PerfTest):
 205     def __init__(self, port, test_name, path_or_url):
 206         super(PageLoadingPerfTest, self).__init__(port, test_name, path_or_url)
 207
 208     def run(self, driver, time_out_ms):
 209         test_times = []
 210
 211         for i in range(0, 20):
 212             output = self.run_single(driver, self.path_or_url(), time_out_ms)
 213             if not output or self.run_failed(output):
 214                 return None
 215             if i == 0:
 216                 continue
 217             test_times.append(output.test_time * 1000)
 218
 219         sorted_test_times = sorted(test_times)
 220
 221         # Compute the mean and variance using Knuth's online algorithm (has good numerical stability).
 222         squareSum = 0
 223         mean = 0
 224         for i, time in enumerate(sorted_test_times):
 225             delta = time - mean
 226             sweep = i + 1.0
 227             mean += delta / sweep
 228             squareSum += delta * (time - mean)
 229
 230         middle = int(len(test_times) / 2)
 231         results = {'values': test_times,
 232             'avg': mean,
 233             'min': sorted_test_times[0],
 234             'max': sorted_test_times[-1],
 235             'median': sorted_test_times[middle] if len(sorted_test_times) % 2 else (sorted_test_times[middle - 1] + sorted_test_times[middle]) / 2,
 236             'stdev': math.sqrt(squareSum / (len(sorted_test_times) - 1)),
 237             'unit': 'ms'}
 238         self.output_statistics(self.test_name(), results, '')
 239         return {self.test_name(): results}
 240
 241
 242 class ReplayServer(object):
 243     def __init__(self, archive, record):
 244         self._process = None
 245
 246         # FIXME: Should error if local proxy isn't set to forward requests to localhost:8080 and localhost:8443
 247
 248         replay_path = webkitpy.thirdparty.autoinstalled.webpagereplay.replay.__file__
 249         args = ['python', replay_path, '--no-dns_forwarding', '--port', '8080', '--ssl_port', '8443', '--use_closest_match', '--log_level', 'warning']
 250         if record:
 251             args.append('--record')
 252         args.append(archive)
 253
 254         self._process = subprocess.Popen(args)
 255
 256     def wait_until_ready(self):
 257         for i in range(0, 3):
 258             try:
 259                 connection = socket.create_connection(('localhost', '8080'), timeout=1)
 260                 connection.close()
 261                 return True
 262             except socket.error:
 263                 time.sleep(1)
 264                 continue
 265         return False
 266
 267     def stop(self):
 268         if self._process:
 269             self._process.send_signal(signal.SIGINT)
 270             self._process.wait()
 271         self._process = None
 272
 273     def __del__(self):
 274         self.stop()
 275
 276
 277 class ReplayPerfTest(PageLoadingPerfTest):
 278     def __init__(self, port, test_name, path_or_url):
 279         super(ReplayPerfTest, self).__init__(port, test_name, path_or_url)
 280
 281     def _start_replay_server(self, archive, record):
 282         try:
 283             return ReplayServer(archive, record)
 284         except OSError as error:
 285             if error.errno == errno.ENOENT:
 286                 _log.error("Replay tests require web-page-replay.")
 287             else:
 288                 raise error
 289
 290     def prepare(self, time_out_ms):
 291         filesystem = self._port.host.filesystem
 292         path_without_ext = filesystem.splitext(self.path_or_url())[0]
 293
 294         self._archive_path = filesystem.join(path_without_ext + '.wpr')
 295         self._expected_image_path = filesystem.join(path_without_ext + '-expected.png')
 296         self._url = filesystem.read_text_file(self.path_or_url()).split('\n')[0]
 297
 298         if filesystem.isfile(self._archive_path) and filesystem.isfile(self._expected_image_path):
 299             _log.info("Replay ready for %s" % self._archive_path)
 300             return True
 301
 302         _log.info("Preparing replay for %s" % self.test_name())
 303
 304         driver = self._port.create_driver(worker_number=1, no_timeout=True)
 305         try:
 306             output = self.run_single(driver, self._archive_path, time_out_ms, record=True)
 307         finally:
 308             driver.stop()
 309
 310         if not output or not filesystem.isfile(self._archive_path):
 311             _log.error("Failed to prepare a replay for %s" % self.test_name())
 312             return False
 313
 314         _log.info("Prepared replay for %s" % self.test_name())
 315
 316         return True
 317
 318     def run_single(self, driver, url, time_out_ms, record=False):
 319         server = self._start_replay_server(self._archive_path, record)
 320         if not server:
 321             _log.error("Web page replay didn't start.")
 322             return None
 323
 324         try:
 325             _log.debug("Waiting for Web page replay to start.")
 326             if not server.wait_until_ready():
 327                 _log.error("Web page replay didn't start.")
 328                 return None
 329
 330             _log.debug("Web page replay started. Loading the page.")
 331             output = super(ReplayPerfTest, self).run_single(driver, self._url, time_out_ms, should_run_pixel_test=True)
 332             if self.run_failed(output):
 333                 return None
 334
 335             if not output.image:
 336                 _log.error("Loading the page did not generate image results")
 337                 _log.error(output.text)
 338                 return None
 339
 340             filesystem = self._port.host.filesystem
 341             dirname = filesystem.dirname(self._archive_path)
 342             filename = filesystem.split(self._archive_path)[1]
 343             writer = TestResultWriter(filesystem, self._port, dirname, filename)
 344             if record:
 345                 writer.write_image_files(actual_image=None, expected_image=output.image)
 346             else:
 347                 writer.write_image_files(actual_image=output.image, expected_image=None)
 348
 349             return output
 350         finally:
 351             server.stop()
 352
 353
 354 class PerfTestFactory(object):
 355
 356     _pattern_map = [
 357         (re.compile(r'^inspector/'), ChromiumStylePerfTest),
 358         (re.compile(r'^PageLoad/'), PageLoadingPerfTest),
 359         (re.compile(r'(.+)\.replay$'), ReplayPerfTest),
 360     ]
 361
 362     @classmethod
 363     def create_perf_test(cls, port, test_name, path):
 364         for (pattern, test_class) in cls._pattern_map:
 365             if pattern.match(test_name):
 366                 return test_class(port, test_name, path)
 367         return PerfTest(port, test_name, path)