Add a tool to update expected.txt files from EWS bot results
authorcommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Sun, 23 Apr 2017 19:52:07 +0000 (19:52 +0000)
committercommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Sun, 23 Apr 2017 19:52:07 +0000 (19:52 +0000)
https://bugs.webkit.org/show_bug.cgi?id=169538

Patch by Youenn Fablet <youenn@apple.com> on 2017-04-23
Reviewed by Ryosuke Niwa.

This script updates expected-txt files from bugzilla posted EWS results.
It uses mac-wk2 as the generic baseline and adds platform-specific results if other ports
have results different from the generic baseline.

* Scripts/update-test-expectations-from-bugzilla: Added.
* Scripts/webkitpy/common/net/bugzilla/test_expectation_updater.py: Added.
* Scripts/webkitpy/common/net/bugzilla/test_expectation_updater_unittest.py: Added.
* Scripts/webkitpy/layout_tests/controllers/test_result_writer.py: Adding helper routines.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215675 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/Scripts/update-test-expectations-from-bugzilla [new file with mode: 0755]
Tools/Scripts/webkitpy/common/net/bugzilla/test_expectation_updater.py [new file with mode: 0644]
Tools/Scripts/webkitpy/common/net/bugzilla/test_expectation_updater_unittest.py [new file with mode: 0644]
Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py

index d8c1927..428837f 100644 (file)
@@ -1,3 +1,19 @@
+2017-04-23  Youenn Fablet  <youenn@apple.com>
+
+        Add a tool to update expected.txt files from EWS bot results
+        https://bugs.webkit.org/show_bug.cgi?id=169538
+
+        Reviewed by Ryosuke Niwa.
+
+        This script updates expected-txt files from bugzilla posted EWS results.
+        It uses mac-wk2 as the generic baseline and adds platform-specific results if other ports
+        have results different from the generic baseline.
+
+        * Scripts/update-test-expectations-from-bugzilla: Added.
+        * Scripts/webkitpy/common/net/bugzilla/test_expectation_updater.py: Added.
+        * Scripts/webkitpy/common/net/bugzilla/test_expectation_updater_unittest.py: Added.
+        * Scripts/webkitpy/layout_tests/controllers/test_result_writer.py: Adding helper routines.
+
 2017-04-22  Youenn Fablet  <youenn@apple.com>
 
         Add an option to import tip-of-tree WPT tests
 2017-04-22  Youenn Fablet  <youenn@apple.com>
 
         Add an option to import tip-of-tree WPT tests
diff --git a/Tools/Scripts/update-test-expectations-from-bugzilla b/Tools/Scripts/update-test-expectations-from-bugzilla
new file mode 100755 (executable)
index 0000000..825453c
--- /dev/null
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2017 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above
+#    copyright notice, this list of conditions and the following
+#    disclaimer.
+# 2. Redistributions in binary form must reproduce the above
+#    copyright notice, this list of conditions and the following
+#    disclaimer in the documentation and/or other materials
+#    provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+import sys
+
+from webkitpy.common.net.bugzilla import test_expectation_updater
+
+
+sys.exit(test_expectation_updater.main(sys.argv[1:], sys.stdout, sys.stderr))
diff --git a/Tools/Scripts/webkitpy/common/net/bugzilla/test_expectation_updater.py b/Tools/Scripts/webkitpy/common/net/bugzilla/test_expectation_updater.py
new file mode 100644 (file)
index 0000000..c9124a8
--- /dev/null
@@ -0,0 +1,153 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2017 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above
+#    copyright notice, this list of conditions and the following
+#    disclaimer.
+# 2. Redistributions in binary form must reproduce the above
+#    copyright notice, this list of conditions and the following
+#    disclaimer in the documentation and/or other materials
+#    provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+import argparse
+import io
+import logging
+import zipfile
+
+from webkitpy.common.host import Host
+from webkitpy.common.net.bugzilla import bugzilla
+from webkitpy.common.net.layouttestresults import LayoutTestResults
+from webkitpy.common.webkit_finder import WebKitFinder
+from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
+from webkitpy.layout_tests.models import test_expectations
+
+_log = logging.getLogger(__name__)
+
+
+def configure_logging():
+    class LogHandler(logging.StreamHandler):
+
+        def format(self, record):
+            if record.levelno > logging.INFO:
+                return "%s: %s" % (record.levelname, record.getMessage())
+            return record.getMessage()
+
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    handler = LogHandler()
+    handler.setLevel(logging.INFO)
+    logger.addHandler(handler)
+    return handler
+
+
+def argument_parser():
+    description = """Update expected.txt files from patches submitted by EWS bots on bugzilla. Given id refers to a bug id by default."""
+    parser = argparse.ArgumentParser(prog='importupdate-test-expectations-from-bugzilla bugzilla_id', description=description, formatter_class=argparse.RawDescriptionHelpFormatter)
+
+    parser.add_argument('-a', '--is-attachment-id', dest='is_bug_id', action='store_false', default=True, help='Whether the given id is a bugzilla attachment and not a bug id')
+    parser.add_argument('-s', '--is-platform-specific', dest='is_attachment_platform_specific', action='store_true', default=False, help='Whether generic expected files should be updated or not')
+    return parser
+
+
+class TestExpectationUpdater(object):
+
+    def __init__(self, host, bugzilla_id, is_bug_id=True, is_attachment_platform_specific=False, attachment_fetcher=bugzilla.Bugzilla(), unzip=None):
+        self.host = host
+        self.filesystem = self.host.filesystem
+        self.unzip = unzip if unzip else lambda content: zipfile.ZipFile(io.BytesIO(content))
+        if is_bug_id:
+            self.platform_specific_attachments = {}
+            for attachment in attachment_fetcher.fetch_bug(bugzilla_id).attachments():
+                bot_type = self._bot_type(attachment)
+                if bot_type:
+                    self.platform_specific_attachments[bot_type] = attachment
+            self.generic_attachment = self.platform_specific_attachments.pop("mac-wk2")
+        else:
+            attachment = attachment_fetcher.fetch_attachment(bugzilla_id)
+            self.platform_specific_attachments = {self._bot_type(attachment): attachment} if is_attachment_platform_specific else {}
+            self.generic_attachment = attachment if not is_attachment_platform_specific else None
+
+        webkit_finder = WebKitFinder(self.filesystem)
+        self.layout_test_repository = webkit_finder.path_from_webkit_base("LayoutTests")
+
+    def _bot_type(self, attachment):
+        name = attachment.name()
+        if "mac" in name and name.endswith("-wk2"):
+            return "mac-wk2"
+        if "mac" in name and not name.endswith("-wk2"):
+            return "mac-wk1"
+        if "simulator" in name:
+            return "ios-wk2"
+        return None
+
+    def _tests_to_update(self, attachment, bot_type=None):
+        _log.info("Processing attachment " + str(attachment.id()))
+        zip_file = self.unzip(attachment.contents())
+        results = LayoutTestResults.results_from_string(zip_file.read("full_results.json"))
+        results_to_update = [result.test_name for result in results.failing_test_results() if result.type == test_expectations.TEXT]
+        return {result: zip_file.read(TestResultWriter.actual_filename(result, self.filesystem)) for result in results_to_update}
+
+    def _file_content_if_exists(self, filename):
+        return self.filesystem.read_text_file(filename) if self.filesystem.exists(filename) else ""
+
+    # FIXME: Investigate the possibility to align what is done there with what single_test_runner is doing.
+    # In particular the fact of not overwriting the file if content is the same.
+    def _update_from_generic_attachment(self):
+        for test_name, expected_content in self._tests_to_update(self.generic_attachment).iteritems():
+            expected_filename = self.filesystem.join(self.layout_test_repository, TestResultWriter.expected_filename(test_name, self.filesystem))
+            if expected_content != self._file_content_if_exists(expected_filename):
+                _log.info("Updating " + test_name + " (" + expected_filename + ")")
+                self.filesystem.write_text_file(expected_filename, expected_content)
+
+    # FIXME: Investigate the possibility to align what is done there with what single_test_runner is doing.
+    # In particular the ability to remove a new specific expectation if it is the same as the generic one.
+    def _update_from_platform_specific_attachment(self, attachment, bot_type):
+        for test_name, expected_content in self._tests_to_update(attachment, bot_type).iteritems():
+            expected_filename = self.filesystem.join(self.layout_test_repository, TestResultWriter.expected_filename(test_name, self.filesystem, bot_type))
+            generic_expected_filename = self.filesystem.join(self.layout_test_repository, TestResultWriter.expected_filename(test_name, self.filesystem))
+            if expected_content != self._file_content_if_exists(generic_expected_filename):
+                _log.info("Updating " + test_name + " for " + bot_type + " (" + expected_filename + ")")
+                self.filesystem.maybe_make_directory(self.filesystem.dirname(expected_filename))
+                self.filesystem.write_text_file(expected_filename, expected_content)
+            elif self.filesystem.exists(expected_filename):
+                self.filesystem.remove(expected_filename)
+
+    def do_update(self):
+        if not self.generic_attachment or not self.platform_specific_attachments:
+            _log.info("No attachment to process")
+            return
+        self._update_from_generic_attachment()
+        for bot_type, attachment in self.platform_specific_attachments.iteritems():
+            self._update_from_platform_specific_attachment(attachment, bot_type)
+
+
+def main(_argv, _stdout, _stderr):
+    parser = argument_parser()
+    options, args = parser.parse_known_args(_argv)
+
+    if not args:
+        raise Exception("Please provide a bug id or use -a option")
+
+    configure_logging()
+
+    bugzilla_id = args[0]
+    updater = TestExpectationUpdater(Host(), bugzilla_id, options.is_bug_id, options.is_attachment_platform_specific)
+    updater.do_update()
diff --git a/Tools/Scripts/webkitpy/common/net/bugzilla/test_expectation_updater_unittest.py b/Tools/Scripts/webkitpy/common/net/bugzilla/test_expectation_updater_unittest.py
new file mode 100644 (file)
index 0000000..fdd6ec8
--- /dev/null
@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2017 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above
+#    copyright notice, this list of conditions and the following
+#    disclaimer.
+# 2. Redistributions in binary form must reproduce the above
+#    copyright notice, this list of conditions and the following
+#    disclaimer in the documentation and/or other materials
+#    provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+import unittest
+
+from webkitpy.common.host_mock import MockHost
+from webkitpy.common.system.filesystem_mock import MockFileSystem
+from webkitpy.common.system.executive_mock import MockExecutive2
+from webkitpy.common.net.bugzilla.test_expectation_updater import TestExpectationUpdater
+from webkitpy.common.net.bugzilla.attachment import Attachment
+
+FAKE_FILES = {
+    '/tests/csswg/css-fake-1/empty_dir/README.txt': '',
+    '/mock-checkout/LayoutTests/w3c/css-fake-1/README.txt': '',
+}
+
+
+class TestExpectationUpdaterTest(unittest.TestCase):
+    def _exists(self, host, filename):
+        return host.filesystem.exists("/mock-checkout/LayoutTests/" + filename)
+
+    def _is_matching(self, host, filename, content):
+        return host.filesystem.read_text_file("/mock-checkout/LayoutTests/" + filename) == content
+
+    def test_update_test_expectations(self):
+        host = MockHost()
+        host.executive = MockExecutive2(exception=OSError())
+        host.filesystem = MockFileSystem(files={
+            '/mock-checkout/LayoutTests/platform/mac-wk1/imported/w3c/web-platform-tests/fetch/api/redirect/redirect-location-expected.txt': 'e-wk1',
+            '/mock-checkout/LayoutTests/imported/w3c/web-platform-tests/dom/events/EventTarget-dispatchEvent-expected.txt': "g",
+            '/mock-checkout/LayoutTests/imported/w3c/web-platform-tests/html/browsers/the-window-object/apis-for-creating-and-navigating-browsing-contexts-by-name/open-features-tokenization-001-expected.txt': "h",
+            '/mock-checkout/LayoutTests/imported/w3c/web-platform-tests/html/browsers/windows/browsing-context-expected.txt': "i",
+            '/mock-checkout/LayoutTests/imported/w3c/web-platform-tests/url/interfaces-expected.txt': "j-mac-wk2"})
+
+        class MockAttachment(Attachment):
+            def __init__(self, attachment_dictionary, contents):
+                Attachment.__init__(self, attachment_dictionary, self)
+                self._contents = contents
+
+            def contents(self):
+                return self._contents
+
+        class MockBugzilla():
+            def fetch_bug(self, id):
+                return self
+
+            def attachments(self):
+                attachment_mac_wk2 = MockAttachment({"id": 1, "name": "Archive of layout-test-results from ews103 for mac-elcapitan-wk2"}, "mac-wk2")
+                attachment_mac_wk1a = MockAttachment({"id": 2, "name": "Archive of layout-test-results from ews103 for mac-elcapitan"}, "mac-wk1a")
+                attachment_mac_wk1b = MockAttachment({"id": 3, "name": "Archive of layout-test-results from ews104 for mac-elcapitan"}, "mac-wk1b")
+                attachment_ios_sim = MockAttachment({"id": 4, "name": "Archive of layout-test-results from ews122 for ios-simulator-wk2"}, "ios-sim")
+                return [attachment_ios_sim, attachment_mac_wk1a, attachment_mac_wk2, attachment_mac_wk1b]
+
+        class MockZip():
+            def __init__(self):
+                self.content = None
+                mac_wk2_files = {
+                    "full_results.json": 'ADD_RESULTS({"tests":{"http":{"tests":{"media":{"hls":{"video-controls-live-stream.html":{"report":"FLAKY","expected":"PASS","actual":"TEXT PASS"},"video-duration-accessibility.html":{"report":"FLAKY","expected":"PASS","actual":"TEXT PASS"}}}}},"imported":{"w3c":{"web-platform-tests":{"html":{"browsers":{"windows":{"browsing-context.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}}}},"fetch":{"api":{"redirect":{"redirect-count.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"},"redirect-location.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"},"redirect-count-worker.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"},"redirect-count-cross-origin.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"},"redirect-location-worker.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}}}}}}},"media":{"track":{"track-in-band-style.html":{"report":"FLAKY","expected":"PASS","actual":"TEXT PASS"}}}},"skipped":3348,"num_regressions":6,"other_crashes":{},"interrupted":false,"num_missing":0,"layout_tests_dir":"/Volumes/Data/EWS/WebKit/LayoutTests","version":4,"num_passes":44738,"pixel_tests_enabled":false,"date":"07:18PM on April 08, 2017","has_pretty_patch":true,"fixable":3357,"num_flaky":3,"uses_expectations_file":true});',
+                    "imported/w3c/web-platform-tests/fetch/api/redirect/redirect-count-cross-origin-actual.txt": "a",
+                    "imported/w3c/web-platform-tests/fetch/api/redirect/redirect-count-worker-actual.txt": "b",
+                    "imported/w3c/web-platform-tests/fetch/api/redirect/redirect-count-actual.txt": "c",
+                    "imported/w3c/web-platform-tests/fetch/api/redirect/redirect-location-worker-actual.txt": "d",
+                    "imported/w3c/web-platform-tests/fetch/api/redirect/redirect-location-actual.txt": "e",
+                    "imported/w3c/web-platform-tests/html/browsers/windows/browsing-context-actual.txt": "f"}
+                mac_wk1a_files = {"full_results.json": 'ADD_RESULTS({"tests":{"http":{"tests":{"loading":{"resourceLoadStatistics":{"non-prevalent-resource-without-user-interaction.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS","has_stderr":true}}}}},"imported":{"w3c":{"web-platform-tests":{"IndexedDB":{"abort-in-initial-upgradeneeded.html":{"report":"FLAKY","expected":"PASS","actual":"TEXT PASS"}},"html":{"browsers":{"windows":{"browsing-context.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}}}},"fetch":{"api":{"redirect":{"redirect-count.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"},"redirect-location.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"},"redirect-count-worker.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"},"redirect-count-cross-origin.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"},"redirect-location-worker.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}}}}},"IndexedDB-private-browsing":{"idbfactory_open9.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS","has_stderr":true}}},"blink":{"storage":{"indexeddb":{"blob-delete-objectstore-db.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS","has_stderr":true}}}}}},"skipped":3537,"num_regressions":6,"other_crashes":{},"interrupted":false,"num_missing":0,"layout_tests_dir":"/Volumes/Data/EWS/WebKit/LayoutTests","version":4,"num_passes":44561,"pixel_tests_enabled":false,"date":"07:18PM on April 08, 2017","has_pretty_patch":true,"fixable":3547,"num_flaky":4,"uses_expectations_file":true});',
+                    "imported/w3c/web-platform-tests/fetch/api/redirect/redirect-count-cross-origin-actual.txt": "a",
+                    "imported/w3c/web-platform-tests/fetch/api/redirect/redirect-count-worker-actual.txt": "b",
+                    "imported/w3c/web-platform-tests/fetch/api/redirect/redirect-count-actual.txt": "c",
+                    "imported/w3c/web-platform-tests/fetch/api/redirect/redirect-location-worker-actual.txt": "d",
+                    "imported/w3c/web-platform-tests/fetch/api/redirect/redirect-location-actual.txt": "e",
+                    "imported/w3c/web-platform-tests/html/browsers/windows/browsing-context-actual.txt": "f-wk1a"}
+                mac_wk1b_files = {"full_results.json": 'ADD_RESULTS({"tests":{"http":{"tests":{"loading":{"resourceLoadStatistics":{"non-prevalent-resource-without-user-interaction.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS","has_stderr":true}}}}},"imported":{"w3c":{"web-platform-tests":{"IndexedDB":{"abort-in-initial-upgradeneeded.html":{"report":"FLAKY","expected":"PASS","actual":"TEXT PASS"}},"html":{"browsers":{"windows":{"browsing-context.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}}}},"fetch":{"api":{"redirect":{"redirect-count.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"},"redirect-location.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"},"redirect-count-worker.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"},"redirect-count-cross-origin.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"},"redirect-location-worker.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}}}}},"IndexedDB-private-browsing":{"idbfactory_open9.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS","has_stderr":true}}},"blink":{"storage":{"indexeddb":{"blob-delete-objectstore-db.html":{"report":"FLAKY","expected":"PASS","actual":"TIMEOUT PASS","has_stderr":true}}}}}},"skipped":3537,"num_regressions":6,"other_crashes":{},"interrupted":false,"num_missing":0,"layout_tests_dir":"/Volumes/Data/EWS/WebKit/LayoutTests","version":4,"num_passes":44561,"pixel_tests_enabled":false,"date":"07:18PM on April 08, 2017","has_pretty_patch":true,"fixable":3547,"num_flaky":4,"uses_expectations_file":true});',
+                    "imported/w3c/web-platform-tests/fetch/api/redirect/redirect-count-cross-origin-actual.txt": "a",
+                    "imported/w3c/web-platform-tests/fetch/api/redirect/redirect-count-worker-actual.txt": "b",
+                    "imported/w3c/web-platform-tests/fetch/api/redirect/redirect-count-actual.txt": "c",
+                    "imported/w3c/web-platform-tests/fetch/api/redirect/redirect-location-worker-actual.txt": "d",
+                    "imported/w3c/web-platform-tests/fetch/api/redirect/redirect-location-actual.txt": "e",
+                    "imported/w3c/web-platform-tests/html/browsers/windows/browsing-context-actual.txt": "f-wk1b"}
+                ios_sim_files = {"full_results.json": 'ADD_RESULTS({"tests":{"imported":{"w3c":{"web-platform-tests":{"url":{"interfaces.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}},"html":{"browsers":{"windows":{"browsing-context.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}},"the-window-object":{"apis-for-creating-and-navigating-browsing-contexts-by-name":{"open-features-tokenization-001.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}}}}},"dom":{"events":{"EventTarget-dispatchEvent.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT"}}}}}},"animations":{"trigger-container-scroll-empty.html":{"report":"FLAKY","expected":"PASS","actual":"TEXT PASS"}}},"skipped":9881,"num_regressions":4,"other_crashes":{},"interrupted":false,"num_missing":0,"layout_tests_dir":"/Volumes/Data/EWS/WebKit/LayoutTests","version":4,"num_passes":38225,"pixel_tests_enabled":false,"date":"07:33PM on April 08, 2017","has_pretty_patch":true,"fixable":48110,"num_flaky":1,"uses_expectations_file":true});',
+                    "imported/w3c/web-platform-tests/dom/events/EventTarget-dispatchEvent-actual.txt": "g",
+                    "imported/w3c/web-platform-tests/html/browsers/the-window-object/apis-for-creating-and-navigating-browsing-contexts-by-name/open-features-tokenization-001-actual.txt": "h",
+                    "imported/w3c/web-platform-tests/html/browsers/windows/browsing-context-actual.txt": "i",
+                    "imported/w3c/web-platform-tests/url/interfaces-actual.txt": "j"}
+                self.files = {"mac-wk2": mac_wk2_files, "mac-wk1a": mac_wk1a_files, "mac-wk1b": mac_wk1b_files, "ios-sim": ios_sim_files}
+
+            def unzip(self, content):
+                self.content = content
+                return self
+
+            def read(self, filename):
+                return self.files[self.content][filename]
+
+        mock_zip = MockZip()
+        updater = TestExpectationUpdater(host, "123456", True, False, MockBugzilla(), lambda content: mock_zip.unzip(content))
+        updater.do_update()
+        # mac-wk2 expectation
+        self.assertTrue(self._is_matching(host, "imported/w3c/web-platform-tests/fetch/api/redirect/redirect-count-cross-origin-expected.txt", "a"))
+        # no need to add mac-wk1 specific expectation
+        self.assertFalse(self._exists(host, "platform/mac-wk1/imported/w3c/web-platform-tests/fetch/api/redirect/redirect-count-cross-origin-expected.txt"))
+        # mac-wk1/ios-simulator-wk2 specific expectation
+        self.assertTrue(self._is_matching(host, "platform/mac-wk1/imported/w3c/web-platform-tests/html/browsers/windows/browsing-context-expected.txt", "f-wk1b"))
+        self.assertTrue(self._is_matching(host, "platform/ios-wk2/imported/w3c/web-platform-tests/url/interfaces-expected.txt", "j"))
+        # removal of mac-wk1 expectation since no longer different
+        self.assertFalse(self._exists(host, "platform/mac-wk1/imported/w3c/web-platform-tests/fetch/api/redirect/redirect-location-expected.txt"))
index 8fa3645..d47e6b9 100644 (file)
@@ -30,7 +30,6 @@
 import logging
 
 from webkitpy.common.wavediff import WaveDiff
 import logging
 
 from webkitpy.common.wavediff import WaveDiff
-from webkitpy.layout_tests.models import test_failures
 
 
 _log = logging.getLogger(__name__)
 
 
 _log = logging.getLogger(__name__)
@@ -63,6 +62,16 @@ class TestResultWriter(object):
     FILENAME_SUFFIX_IMAGE_DIFF = "-diff.png"
     FILENAME_SUFFIX_IMAGE_DIFFS_HTML = "-diffs.html"
 
     FILENAME_SUFFIX_IMAGE_DIFF = "-diff.png"
     FILENAME_SUFFIX_IMAGE_DIFFS_HTML = "-diffs.html"
 
+    @staticmethod
+    def expected_filename(test_name, filesystem, port_name=None, suffix='txt'):
+        if not port_name:
+            return filesystem.splitext(test_name)[0] + TestResultWriter.FILENAME_SUFFIX_EXPECTED + '.' + suffix
+        return filesystem.join("platform", port_name, filesystem.splitext(test_name)[0] + TestResultWriter.FILENAME_SUFFIX_EXPECTED + '.' + suffix)
+
+    @staticmethod
+    def actual_filename(test_name, filesystem, suffix='txt'):
+        return filesystem.splitext(test_name)[0] + TestResultWriter.FILENAME_SUFFIX_ACTUAL + '.' + suffix
+
     def __init__(self, filesystem, port, root_output_dir, test_name):
         self._filesystem = filesystem
         self._port = port
     def __init__(self, filesystem, port, root_output_dir, test_name):
         self._filesystem = filesystem
         self._port = port