return base.DriverInput(self._filename, self._timeout, image_hash)
def _run(self, driver, test_input):
+ if self._options.new_baseline or self._options.reset_results:
+ return self._run_rebaseline(driver, test_input)
+ return self._run_compare_test(driver, test_input)
+
+ def _run_compare_test(self, driver, test_input):
driver_output = self._driver.run_test(self._driver_input(test_input))
return self._process_output(driver_output)
- def _process_output(self, driver_output):
- """Receives the output from a DumpRenderTree process, subjects it to a
- number of tests, and returns a list of failure types the test produced.
- Args:
- driver_output: a DriverOutput object containing the output from the driver
+ def _run_rebaseline(self, driver, test_input):
+ driver_output = self._driver.run_test(self._driver_input(test_input))
+ failures = self._handle_error(driver_output)
+ # FIXME: It the test crashed or timed out, it might be bettter to avoid
+ # to write new baselines.
+ self._save_baselines(driver_output)
+ return TestResult(self._filename, failures, driver_output.test_time)
+
+ def _save_baselines(self, driver_output):
+ # Although all test_shell/DumpRenderTree output should be utf-8,
+ # we do not ever decode it inside run-webkit-tests. For some tests
+ # DumpRenderTree may not output utf-8 text (e.g. webarchives).
+ self._save_baseline_data(driver_output.text, ".txt", encoding=None,
+ generate_new_baseline=self._options.new_baseline)
+ if self._options.pixel_tests and driver_output.image_hash:
+ self._save_baseline_data(driver_output.image, ".png", encoding=None,
+ generate_new_baseline=self._options.new_baseline)
+ self._save_baseline_data(driver_output.image_hash, ".checksum",
+ encoding="ascii",
+ generate_new_baseline=self._options.new_baseline)
+
+ def _save_baseline_data(self, data, modifier, encoding,
+ generate_new_baseline=True):
+ """Saves a new baseline file into the port's baseline directory.
+
+ The file will be named simply "<test>-expected<modifier>", suitable for
+ use as the expected results in a later run.
- Returns: a TestResult object
+ Args:
+ data: result to be saved as the new baseline
+ modifier: type of the result file, e.g. ".txt" or ".png"
+ encoding: file encoding (none, "utf-8", etc.)
+ generate_new_baseline: whether to enerate a new, platform-specific
+ baseline, or update the existing one
"""
+
+ port = self._port
+ fs = port._filesystem
+ if generate_new_baseline:
+ relative_dir = fs.dirname(self._testname)
+ baseline_path = port.baseline_path()
+ output_dir = fs.join(baseline_path, relative_dir)
+ output_file = fs.basename(fs.splitext(self._filename)[0] +
+ "-expected" + modifier)
+ fs.maybe_make_directory(output_dir)
+ output_path = fs.join(output_dir, output_file)
+ _log.debug('writing new baseline result "%s"' % (output_path))
+ else:
+ output_path = port.expected_filename(self._filename, modifier)
+ _log.debug('resetting baseline result "%s"' % output_path)
+
+ port.update_baseline(output_path, data, encoding)
+
+ def _handle_error(self, driver_output):
failures = []
fs = self._port._filesystem
-
- if driver_output.crash:
- failures.append(test_failures.FailureCrash())
if driver_output.timeout:
failures.append(test_failures.FailureTimeout())
-
if driver_output.crash:
+ failures.append(test_failures.FailureCrash())
_log.debug("%s Stacktrace for %s:\n%s" % (self._worker_name, self._testname,
driver_output.error))
stack_filename = fs.join(self._options.results_directory, self._testname)
elif driver_output.error:
_log.debug("%s %s output stderr lines:\n%s" % (self._worker_name, self._testname,
driver_output.error))
+ return failures
+
+ def _run_test(self):
+ driver_output = self._driver.run_test(self._driver_input())
+ return self._process_output(driver_output)
+
+ def _process_output(self, driver_output):
+ """Receives the output from a DumpRenderTree process, subjects it to a
+ number of tests, and returns a list of failure types the test produced.
+ Args:
+ driver_output: a DriverOutput object containing the output from the driver
+ Returns: a TestResult object
+ """
+ fs = self._port._filesystem
+ failures = self._handle_error(driver_output)
expected_driver_output = self._expected_driver_output()
# Check the output and save the results.
class ImageDiff(test_type_base.TestTypeBase):
- def _save_baseline_files(self, filename, image, image_hash,
- generate_new_baseline):
- """Saves new baselines for the PNG and checksum.
-
- Args:
- filename: test filename
- image: a image output
- image_hash: a checksum of the image
- generate_new_baseline: whether to generate a new, platform-specific
- baseline, or update the existing one
- """
- self._save_baseline_data(filename, image, ".png", encoding=None,
- generate_new_baseline=generate_new_baseline)
- self._save_baseline_data(filename, image_hash, ".checksum",
- encoding="ascii",
- generate_new_baseline=generate_new_baseline)
-
def _copy_image(self, filename, actual_image, expected_image):
self.write_output_files(filename, '.png',
output=actual_image, expected=expected_image,
if actual_driver_output.image_hash is None:
return failures
- # If we're generating a new baseline, we pass.
- if options.new_baseline or options.reset_results:
- self._save_baseline_files(filename, actual_driver_output.image,
- actual_driver_output.image_hash,
- options.new_baseline)
- return failures
-
if not expected_driver_output.image:
# Report a missing expected PNG file.
self._copy_image(filename, actual_driver_output.image, expected_image=None)
self._port.relative_test_filename(filename))
fs.maybe_make_directory(fs.dirname(output_filename))
- def _save_baseline_data(self, filename, data, modifier, encoding,
- generate_new_baseline=True):
- """Saves a new baseline file into the port's baseline directory.
-
- The file will be named simply "<test>-expected<modifier>", suitable for
- use as the expected results in a later run.
-
- Args:
- filename: path to the test file
- data: result to be saved as the new baseline
- modifier: type of the result file, e.g. ".txt" or ".png"
- encoding: file encoding (none, "utf-8", etc.)
- generate_new_baseline: whether to enerate a new, platform-specific
- baseline, or update the existing one
- """
-
- port = self._port
- fs = self._port._filesystem
- if generate_new_baseline:
- relative_dir = fs.dirname(port.relative_test_filename(filename))
- baseline_path = port.baseline_path()
- output_dir = fs.join(baseline_path, relative_dir)
- output_file = fs.basename(fs.splitext(filename)[0] +
- self.FILENAME_SUFFIX_EXPECTED + modifier)
- fs.maybe_make_directory(output_dir)
- output_path = fs.join(output_dir, output_file)
- _log.debug('writing new baseline result "%s"' % (output_path))
- else:
- output_path = port.expected_filename(filename, modifier)
- _log.debug('resetting baseline result "%s"' % output_path)
-
- port.update_baseline(output_path, data, encoding)
-
def output_filename(self, filename, modifier):
"""Returns a filename inside the output dir that contains modifier.
git://git.webkit.org / WebKit-https.git / commitdiff