Sync up w3c import script with changes in Blink
authorcommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 27 Jun 2013 22:43:27 +0000 (22:43 +0000)
committercommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 27 Jun 2013 22:43:27 +0000 (22:43 +0000)
https://bugs.webkit.org/show_bug.cgi?id=118149

Patch by Bem Jones-Bey <bjonesbe@adobe.com> on 2013-06-27
Reviewed by Dirk Pranke.

This is a port of the following blink changes:

https://codereview.chromium.org/15366004
https://chromiumcodereview.appspot.com/15592005
https://codereview.chromium.org/16021006
https://codereview.chromium.org/15590004
https://codereview.chromium.org/15901008
https://codereview.chromium.org/16171009
https://codereview.chromium.org/16465007
https://codereview.chromium.org/15981013

* Scripts/webkitpy/common/checkout/checkout_unittest.py:
(setUp):
(test_commit_message_for_this_commit):
* Scripts/webkitpy/port/base.py:
(Port):
* Scripts/webkitpy/port/test.py:
* Scripts/webkitpy/w3c/test_converter.py:
(W3CTestConverter.__init__):
(W3CTestConverter.read_webkit_prefixed_css_property_list):
(W3CTestConverter.convert_for_webkit):
(W3CTestConverter.convert_css):
(W3CTestConverter.convert_html):
(W3CTestConverter.convert_testharness_paths):
(W3CTestConverter.convert_prefixed_properties):
(W3CTestConverter.add_webkit_prefix_to_unprefixed_properties):
* Scripts/webkitpy/w3c/test_converter_unittest.py:
* Scripts/webkitpy/w3c/test_importer.py:
(main):
(configure_logging):
(configure_logging.LogHandler):
(configure_logging.LogHandler.format):
(parse_args):
(TestImporter.__init__):
(TestImporter.find_importable_tests):
(TestImporter.import_tests):
(TestImporter.setup_destination_directory):
(TestImporter.remove_deleted_files):
* Scripts/webkitpy/w3c/test_importer_unittest.py:
(TestImporterTest.test_import_dir_with_no_tests_and_no_hg):
(TestImporterTest.test_import_dir_with_no_tests):
(TestImporterTest):
* Scripts/webkitpy/w3c/test_parser.py:
(TestParser.load_file):
(TestParser.analyze_test):
(TestParser.support_files):
* Scripts/webkitpy/w3c/test_parser_unittest.py:

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@152133 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/Scripts/webkitpy/common/checkout/checkout_unittest.py
Tools/Scripts/webkitpy/port/base.py
Tools/Scripts/webkitpy/port/test.py
Tools/Scripts/webkitpy/w3c/test_converter.py
Tools/Scripts/webkitpy/w3c/test_converter_unittest.py
Tools/Scripts/webkitpy/w3c/test_importer.py
Tools/Scripts/webkitpy/w3c/test_importer_unittest.py
Tools/Scripts/webkitpy/w3c/test_parser.py
Tools/Scripts/webkitpy/w3c/test_parser_unittest.py

index d0ba557f3480d5813e1aabe439ac93d8223919f2..c76eb868469675150616a9ebf9bf73d7462b3464 100644 (file)
@@ -1,3 +1,58 @@
+2013-06-27  Bem Jones-Bey  <bjonesbe@adobe.com>
+
+        Sync up w3c import script with changes in Blink
+        https://bugs.webkit.org/show_bug.cgi?id=118149
+
+        Reviewed by Dirk Pranke.
+
+        This is a port of the following blink changes:
+
+        https://codereview.chromium.org/15366004
+        https://chromiumcodereview.appspot.com/15592005
+        https://codereview.chromium.org/16021006
+        https://codereview.chromium.org/15590004
+        https://codereview.chromium.org/15901008
+        https://codereview.chromium.org/16171009
+        https://codereview.chromium.org/16465007
+        https://codereview.chromium.org/15981013
+
+        * Scripts/webkitpy/common/checkout/checkout_unittest.py:
+        (setUp):
+        (test_commit_message_for_this_commit):
+        * Scripts/webkitpy/port/base.py:
+        (Port):
+        * Scripts/webkitpy/port/test.py:
+        * Scripts/webkitpy/w3c/test_converter.py:
+        (W3CTestConverter.__init__):
+        (W3CTestConverter.read_webkit_prefixed_css_property_list):
+        (W3CTestConverter.convert_for_webkit):
+        (W3CTestConverter.convert_css):
+        (W3CTestConverter.convert_html):
+        (W3CTestConverter.convert_testharness_paths):
+        (W3CTestConverter.convert_prefixed_properties):
+        (W3CTestConverter.add_webkit_prefix_to_unprefixed_properties):
+        * Scripts/webkitpy/w3c/test_converter_unittest.py:
+        * Scripts/webkitpy/w3c/test_importer.py:
+        (main):
+        (configure_logging):
+        (configure_logging.LogHandler):
+        (configure_logging.LogHandler.format):
+        (parse_args):
+        (TestImporter.__init__):
+        (TestImporter.find_importable_tests):
+        (TestImporter.import_tests):
+        (TestImporter.setup_destination_directory):
+        (TestImporter.remove_deleted_files):
+        * Scripts/webkitpy/w3c/test_importer_unittest.py:
+        (TestImporterTest.test_import_dir_with_no_tests_and_no_hg):
+        (TestImporterTest.test_import_dir_with_no_tests):
+        (TestImporterTest):
+        * Scripts/webkitpy/w3c/test_parser.py:
+        (TestParser.load_file):
+        (TestParser.analyze_test):
+        (TestParser.support_files):
+        * Scripts/webkitpy/w3c/test_parser_unittest.py:
+
 2013-06-27  Csaba Osztrogon√°c  <ossy@webkit.org>
 
         [webkitpy] AbstractPatchSequencingCommand should have _prepare_state
index b7aead97223100b309cf99b666c36bf94a597dad..587798e770680e8581aa694fba32358e56fb619e 100644 (file)
@@ -36,6 +36,7 @@ from .checkout import Checkout
 from .changelog import ChangeLogEntry
 from .scm import CommitMessage, SCMDetector
 from .scm.scm_mock import MockSCM
+from webkitpy.common.webkit_finder import WebKitFinder
 from webkitpy.common.system.executive import Executive, ScriptError
 from webkitpy.common.system.filesystem import FileSystem  # FIXME: This should not be needed.
 from webkitpy.common.system.filesystem_mock import MockFileSystem
@@ -102,6 +103,7 @@ Second part of this complicated change by me, Tor Arne Vestb\u00f8!
         self.temp_dir = str(self.filesystem.mkdtemp(suffix="changelogs"))
         self.old_cwd = self.filesystem.getcwd()
         self.filesystem.chdir(self.temp_dir)
+        self.webkit_base = WebKitFinder(self.filesystem).webkit_base()
 
         # Trick commit-log-editor into thinking we're in a Subversion working copy so it won't
         # complain about not being able to figure out what SCM is in use.
@@ -130,7 +132,7 @@ Second part of this complicated change by me, Tor Arne Vestb\u00f8!
             return executive.run_command(*args, **kwargs)
 
         detector = SCMDetector(self.filesystem, executive)
-        real_scm = detector.detect_scm_system(self.old_cwd)
+        real_scm = detector.detect_scm_system(self.webkit_base)
 
         mock_scm = MockSCM()
         mock_scm.run = mock_run
index 7200a7bf9421ba213f06ed6b31027a55ec05d9b6..c68597f2a4bac6a08e8e683607918655fb76c863 100644 (file)
@@ -606,8 +606,8 @@ class Port(object):
         return [self.relative_test_filename(f) for f in files]
 
     # When collecting test cases, we include any file with these extensions.
-    _supported_test_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.pl', '.htm', '.php', '.svg', '.mht'])
-    _supported_reference_extensions = set(['.html', '.xml', '.xhtml', '.htm', '.svg'])
+    _supported_test_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.pl', '.htm', '.php', '.svg', '.mht', '.xht'])
+    _supported_reference_extensions = set(['.html', '.xml', '.xhtml', '.htm', '.svg', '.xht'])
 
     @staticmethod
     # If any changes are made here be sure to update the isUsedInReftest method in old-run-webkit-tests as well.
index 7ad318dbfc229aef37b14a13ed7a000aa19609d5..76fea3ba182853f4d5badf54248a7a2d51a84557 100644 (file)
@@ -100,7 +100,7 @@ class TestList(object):
 #
 # These numbers may need to be updated whenever we add or delete tests.
 #
-TOTAL_TESTS = 104
+TOTAL_TESTS = 106
 TOTAL_SKIPS = 28
 TOTAL_RETRIES = 14
 
index bccb2be4d041f6a8157a01f517361475e5e1a12e..2e9bfcb9a35fb62e3af3c66fb18f69c65847c9ea 100644 (file)
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
+import logging
 import re
 
 from webkitpy.common.host import Host
+from webkitpy.common.webkit_finder import WebKitFinder
 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup, Tag
 
 
+_log = logging.getLogger(__name__)
+
+
 class W3CTestConverter(object):
 
     def __init__(self):
         self._host = Host()
         self._filesystem = self._host.filesystem
-        self._host.initialize_scm()
-        self._webkit_root = self._host.scm().checkout_root
+        self._webkit_root = WebKitFinder(self._filesystem).webkit_base()
+
+        # These settings might vary between WebKit and Blink
+        self._css_property_file = self.path_from_webkit_root('Source', 'WebCore', 'css', 'CSSPropertyNames.in')
+        self._css_property_split_string = '='
+
         self.prefixed_properties = self.read_webkit_prefixed_css_property_list()
 
     def path_from_webkit_root(self, *comps):
@@ -48,14 +57,14 @@ class W3CTestConverter(object):
     def read_webkit_prefixed_css_property_list(self):
         prefixed_properties = []
 
-        contents = self._filesystem.read_text_file(self.path_from_webkit_root('Source', 'WebCore', 'css', 'CSSPropertyNames.in'))
+        contents = self._filesystem.read_text_file(self._css_property_file)
         for line in contents.splitlines():
             # Find lines starting with the -webkit- prefix.
             match = re.match('-webkit-[\w|-]*', line)
             if match:
                 # Ignore lines where both the prefixed and non-prefixed property
                 # are supported - denoted by -webkit-some-property = some-property.
-                fields = line.split('=')
+                fields = line.split(self._css_property_split_string)
                 if len(fields) == 2 and fields[1].strip() in fields[0].strip():
                     continue
                 prefixed_properties.append(match.group(0))
@@ -66,21 +75,21 @@ class W3CTestConverter(object):
         """ Converts a file's |contents| so it will function correctly in its |new_path| in Webkit.
 
         Returns the list of modified properties and the modified text if the file was modifed, None otherwise."""
-        contents = self._filesystem.read_text_file(filename)
+        contents = self._filesystem.read_binary_file(filename)
         if filename.endswith('.css'):
-            return self.convert_css(contents)
-        return self.convert_html(new_path, contents)
+            return self.convert_css(contents, filename)
+        return self.convert_html(new_path, contents, filename)
 
-    def convert_css(self, contents):
-        return self.add_webkit_prefix_to_unprefixed_properties(contents)
+    def convert_css(self, contents, filename):
+        return self.add_webkit_prefix_to_unprefixed_properties(contents, filename)
 
-    def convert_html(self, new_path, contents):
+    def convert_html(self, new_path, contents, filename):
         doc = BeautifulSoup(contents)
-        did_modify_paths = self.convert_testharness_paths(doc, new_path)
-        converted_properties_and_content = self.convert_prefixed_properties(doc)
+        did_modify_paths = self.convert_testharness_paths(doc, new_path, filename)
+        converted_properties_and_content = self.convert_prefixed_properties(doc, filename)
         return converted_properties_and_content if (did_modify_paths or converted_properties_and_content[0]) else None
 
-    def convert_testharness_paths(self, doc, new_path):
+    def convert_testharness_paths(self, doc, new_path, filename):
         """ Update links to testharness.js in the BeautifulSoup |doc| to point to the copy in |new_path|.
 
         Returns whether the document was modified."""
@@ -98,10 +107,17 @@ class W3CTestConverter(object):
         resources_relpath = self._filesystem.relpath(resources_path, new_path)
 
         for tag in testharness_tags:
+            # FIXME: We need to handle img, audio, video tags also.
             attr = 'src'
             if tag.name != 'script':
                 attr = 'href'
 
+            if not attr in tag.attrMap:
+                # FIXME: Figure out what to do w/ invalid tags. For now, we return False
+                # and leave the document unmodified, which means that it'll probably fail to run.
+                _log.error("Missing an attr in %s" % filename)
+                return False
+
             old_path = tag[attr]
             new_tag = Tag(doc, tag.name, tag.attrs)
             new_tag[attr] = re.sub(pattern, resources_relpath + '/testharness', old_path)
@@ -110,7 +126,7 @@ class W3CTestConverter(object):
 
         return True
 
-    def convert_prefixed_properties(self, doc):
+    def convert_prefixed_properties(self, doc, filename):
         """ Searches a BeautifulSoup |doc| for any CSS properties requiring the -webkit- prefix and converts them.
 
         Returns the list of converted properties and the modified document as a string """
@@ -133,7 +149,7 @@ class W3CTestConverter(object):
             else:
                 style_text = tag['style']
 
-            updated_style_text = self.add_webkit_prefix_to_unprefixed_properties(style_text)
+            updated_style_text = self.add_webkit_prefix_to_unprefixed_properties(style_text, filename)
 
             # Rewrite tag only if changes were made.
             if updated_style_text[0]:
@@ -146,7 +162,7 @@ class W3CTestConverter(object):
 
         return (converted_properties, doc.prettify())
 
-    def add_webkit_prefix_to_unprefixed_properties(self, text):
+    def add_webkit_prefix_to_unprefixed_properties(self, text, filename):
         """ Searches |text| for instances of properties requiring the -webkit- prefix and adds the prefix to them.
 
         Returns the list of converted properties and the modified text."""
@@ -154,6 +170,8 @@ class W3CTestConverter(object):
         converted_properties = []
 
         for prefixed_property in self.prefixed_properties:
+            # FIXME: add in both the prefixed and unprefixed versions, rather than just replacing them?
+            # That might allow the imported test to work in other browsers more easily.
 
             unprefixed_property = prefixed_property.replace('-webkit-', '')
 
@@ -162,11 +180,11 @@ class W3CTestConverter(object):
             # or at the beginning of the string (for inline style attribute)
             pattern = '([\s{]|^)' + unprefixed_property + '(\s+:|:)'
             if re.search(pattern, text):
-                print 'converting %s -> %s' % (unprefixed_property, prefixed_property)
+                _log.info('converting %s -> %s' % (unprefixed_property, prefixed_property))
                 converted_properties.append(prefixed_property)
                 text = re.sub(pattern, prefixed_property + ':', text)
 
-        # FIXME: Handle the JS versions of these properties, too.
+        # FIXME: Handle the JS versions of these properties and GetComputedStyle, too.
         return (converted_properties, text)
 
     def replace_tag(self, old_tag, new_tag):
index 11e24d82fcb6a87fb008123e16729a6c909986fc..ff104abd50516a9cddc719955be1964c3e6d8ad9 100644 (file)
@@ -36,6 +36,8 @@ from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup
 from webkitpy.w3c.test_converter import W3CTestConverter
 
 
+DUMMY_FILENAME = 'dummy.html'
+
 class W3CTestConverterTest(unittest.TestCase):
 
     def fake_dir_path(self, converter, dirname):
@@ -77,7 +79,7 @@ CONTENT OF TEST
         oc = OutputCapture()
         oc.capture_output()
         try:
-            converted = converter.convert_html('/nothing/to/convert', test_html)
+            converted = converter.convert_html('/nothing/to/convert', test_html, DUMMY_FILENAME)
         finally:
             oc.restore_output()
 
@@ -94,7 +96,7 @@ CONTENT OF TEST
         converter = W3CTestConverter()
         fake_dir_path = self.fake_dir_path(converter, "harnessonly")
 
-        converted = converter.convert_html(fake_dir_path, test_html)
+        converted = converter.convert_html(fake_dir_path, test_html, DUMMY_FILENAME)
 
         self.verify_conversion_happened(converted)
         self.verify_test_harness_paths(converter, converted[1], fake_dir_path, 1, 1)
@@ -125,7 +127,7 @@ CONTENT OF TEST
         oc = OutputCapture()
         oc.capture_output()
         try:
-            converted = converter.convert_html(fake_dir_path, test_content[1])
+            converted = converter.convert_html(fake_dir_path, test_content[1], DUMMY_FILENAME)
         finally:
             oc.restore_output()
 
@@ -160,7 +162,7 @@ CONTENT OF TEST
         oc.capture_output()
         try:
             test_content = self.generate_test_content(converter.prefixed_properties, 2, test_html)
-            converted = converter.convert_html(fake_dir_path, test_content[1])
+            converted = converter.convert_html(fake_dir_path, test_content[1], DUMMY_FILENAME)
         finally:
             oc.restore_output()
 
@@ -185,7 +187,7 @@ CONTENT OF TEST
         oc = OutputCapture()
         oc.capture_output()
         try:
-            converted = converter.convert_testharness_paths(doc, fake_dir_path)
+            converted = converter.convert_testharness_paths(doc, fake_dir_path, DUMMY_FILENAME)
         finally:
             oc.restore_output()
 
@@ -262,7 +264,7 @@ CONTENT OF TEST
         oc = OutputCapture()
         oc.capture_output()
         try:
-            converted = converter.convert_prefixed_properties(BeautifulSoup(test_content[1]))
+            converted = converter.convert_prefixed_properties(BeautifulSoup(test_content[1]), DUMMY_FILENAME)
         finally:
             oc.restore_output()
 
index 03fdbe5945613acedac3453e6fb0ec6129211bf8..119bd7d92862c1e519ca0a4b3e9ce52fa9cfa628 100644 (file)
       This can also be overridden by a -n or --no-overwrite flag
 
     - All files are converted to work in WebKit:
-         1. .xht extensions are changed to .xhtml to make new-run-webkit-tests happy
-         2. Paths to testharness.js files are modified point to Webkit's copy of them in
+         1. Paths to testharness.js files are modified point to Webkit's copy of them in
             LayoutTests/resources, using the correct relative path from the new location
-         3. All CSS properties requiring the -webkit-vendor prefix are prefixed - this current
+         2. All CSS properties requiring the -webkit-vendor prefix are prefixed - this current
             list of what needs prefixes is read from Source/WebCore/CSS/CSSProperties.in
-         4. Each reftest has its own copy of its reference file following the naming conventions
+         3. Each reftest has its own copy of its reference file following the naming conventions
             new-run-webkit-tests expects
-         5. If a a reference files lives outside the directory of the test that uses it, it is checked
+         4. If a reference files lives outside the directory of the test that uses it, it is checked
             for paths to support files as it will be imported into a different relative position to the
             test file (in the same directory)
 
@@ -90,6 +89,7 @@
 # FIXME: Change this file to use the Host abstractions rather that os, sys, shutils, etc.
 
 import datetime
+import logging
 import mimetypes
 import optparse
 import os
@@ -97,6 +97,7 @@ import shutil
 import sys
 
 from webkitpy.common.host import Host
+from webkitpy.common.webkit_finder import WebKitFinder
 from webkitpy.common.system.executive import ScriptError
 from webkitpy.w3c.test_parser import TestParser
 from webkitpy.w3c.test_converter import W3CTestConverter
@@ -109,62 +110,81 @@ TEST_STATUS_SUBMITTED = 'submitted'
 CHANGESET_NOT_AVAILABLE = 'Not Available'
 
 
+_log = logging.getLogger(__name__)
+
+
 def main(_argv, _stdout, _stderr):
     options, args = parse_args()
-    import_dir = validate_import_directory(args[0])
-    test_importer = TestImporter(Host(), import_dir, options)
+    import_dir = args[0]
+    if len(args) == 1:
+        repo_dir = os.path.dirname(import_dir)
+    else:
+        repo_dir = args[1]
+
+    if not os.path.exists(import_dir):
+        sys.exit('Source directory %s not found!' % import_dir)
+
+    if not os.path.exists(repo_dir):
+        sys.exit('Repository directory %s not found!' % repo_dir)
+    if not repo_dir in import_dir:
+        sys.exit('Repository directory %s must be a parent of %s' % (repo_dir, import_dir))
+
+    configure_logging()
+
+    test_importer = TestImporter(Host(), import_dir, repo_dir, options)
     test_importer.do_import()
 
 
+def configure_logging():
+    class LogHandler(logging.StreamHandler):
+
+        def format(self, record):
+            if record.levelno > logging.INFO:
+                return "%s: %s" % (record.levelname, record.getMessage())
+            return record.getMessage()
+
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    handler = LogHandler()
+    handler.setLevel(logging.INFO)
+    logger.addHandler(handler)
+    return handler
+
+
 def parse_args():
-    parser = optparse.OptionParser(usage='usage: %prog [options] w3c_test_directory')
+    parser = optparse.OptionParser(usage='usage: %prog [options] w3c_test_directory [repo_directory]')
     parser.add_option('-n', '--no-overwrite', dest='overwrite', action='store_false', default=True,
         help='Flag to prevent duplicate test files from overwriting existing tests. By default, they will be overwritten')
     parser.add_option('-a', '--all', action='store_true', default=False,
         help='Import all tests including reftests, JS tests, and manual/pixel tests. By default, only reftests and JS tests are imported')
 
     options, args = parser.parse_args()
-    if len(args) != 1:
+    if len(args) not in (1, 2):
         parser.error('Incorrect number of arguments')
     return options, args
 
 
-def validate_import_directory(import_dir):
-    if not os.path.exists(import_dir):
-        sys.exit('Source directory %s not found!' % import_dir)
-
-    # Make sure the tests are officially submitted to the W3C, either approved or
-    # submitted following their directory naming conventions
-    if import_dir.find('approved') == -1 and import_dir.find('submitted') == -1:
-        # If not pointed directly to the approved directory or to any submitted
-        # directory, check for a submitted subdirectory and go with that
-        import_dir = os.path.join(import_dir, 'submitted')
-        if not os.path.exists(os.path.join(import_dir)):
-            sys.exit('Unable to import tests that aren\'t approved or submitted to the W3C')
-
-    return import_dir
-
-
 class TestImporter(object):
 
-    def __init__(self, host, source_directory, options):
+    def __init__(self, host, source_directory, repo_dir, options):
         self.host = host
         self.source_directory = source_directory
         self.options = options
 
         self.filesystem = self.host.filesystem
-        self._webkit_root = __file__.split(self.filesystem.sep + 'Tools')[0]
 
-        self.destination_directory = self.path_from_webkit_root("LayoutTests", "csswg")
+        webkit_finder = WebKitFinder(self.filesystem)
+        self._webkit_root = webkit_finder.webkit_base()
+        self.repo_dir = repo_dir
+        subdirs = os.path.dirname(os.path.relpath(source_directory, repo_dir))
+
+        self.destination_directory = webkit_finder.path_from_webkit_base("LayoutTests", 'w3c', *subdirs)
 
         self.changeset = CHANGESET_NOT_AVAILABLE
         self.test_status = TEST_STATUS_UNKNOWN
 
         self.import_list = []
 
-    def path_from_webkit_root(self, *comps):
-        return self.filesystem.abspath(self.filesystem.join(self._webkit_root, *comps))
-
     def do_import(self):
         self.find_importable_tests(self.source_directory)
         self.load_changeset()
@@ -180,22 +200,18 @@ class TestImporter(object):
     def find_importable_tests(self, directory):
         # FIXME: use filesystem
         for root, dirs, files in os.walk(directory):
-            print 'Scanning ' + root + '...'
+            _log.info('Scanning ' + root + '...')
             total_tests = 0
             reftests = 0
             jstests = 0
 
-            # Ignore any repo stuff
-            if '.git' in dirs:
-                dirs.remove('.git')
-            if '.hg' in dirs:
-                dirs.remove('.hg')
-
-            # archive and data dirs are internal csswg things that live in every approved directory
-            if 'data' in dirs:
-                dirs.remove('data')
-            if 'archive' in dirs:
-                dirs.remove('archive')
+            # "archive" and "data" dirs are internal csswg things that live in every approved directory.
+            # FIXME: skip 'incoming' tests for now, but we should rework the 'test_status' concept and
+            # support reading them as well.
+            DIRS_TO_SKIP = ('.git', '.hg', 'data', 'archive', 'incoming')
+            for d in DIRS_TO_SKIP:
+                if d in dirs:
+                    dirs.remove(d)
 
             copy_list = []
 
@@ -258,22 +274,19 @@ class TestImporter(object):
                 if 'support' in dirs:
                     dirs.remove('support')
 
-                if copy_list:
-                    # Only add this directory to the list if there's something to import
-                    self.import_list.append({'dirname': root, 'copy_list': copy_list,
-                        'reftests': reftests, 'jstests': jstests, 'total_tests': total_tests})
+            if copy_list:
+                # Only add this directory to the list if there's something to import
+                self.import_list.append({'dirname': root, 'copy_list': copy_list,
+                    'reftests': reftests, 'jstests': jstests, 'total_tests': total_tests})
 
     def import_tests(self):
-        if self.import_list:
-            self.setup_destination_directory()
-
         converter = W3CTestConverter()
         total_imported_tests = 0
         total_imported_reftests = 0
         total_imported_jstests = 0
+        total_prefixed_properties = {}
 
         for dir_to_copy in self.import_list:
-
             total_imported_tests += dir_to_copy['total_tests']
             total_imported_reftests += dir_to_copy['reftests']
             total_imported_jstests += dir_to_copy['jstests']
@@ -283,15 +296,11 @@ class TestImporter(object):
             if not dir_to_copy['copy_list']:
                 continue
 
-            # Build the subpath starting with the approved/submitted directory
             orig_path = dir_to_copy['dirname']
-            start = orig_path.find(self.test_status)
-            new_subpath = orig_path[start:len(orig_path)]
 
-            # Append the new subpath to the destination_directory
-            new_path = os.path.join(self.destination_directory, new_subpath)
+            subpath = os.path.relpath(orig_path, self.repo_dir)
+            new_path = os.path.join(self.destination_directory, subpath)
 
-            # Create the destination subdirectories if not there
             if not(os.path.exists(new_path)):
                 os.makedirs(new_path)
 
@@ -301,41 +310,44 @@ class TestImporter(object):
                 # FIXME: Split this block into a separate function.
                 orig_filepath = os.path.normpath(file_to_copy['src'])
 
-                assert(not os.path.isdir(orig_filepath))
+                if os.path.isdir(orig_filepath):
+                    # FIXME: Figure out what is triggering this and what to do about it.
+                    _log.error('%s refers to a directory' % orig_filepath)
+                    continue
 
                 if not(os.path.exists(orig_filepath)):
-                    print 'Warning: ' + orig_filepath + ' not found. Possible error in the test.'
+                    _log.warning('%s not found. Possible error in the test.', orig_filepath)
                     continue
 
                 new_filepath = os.path.join(new_path, file_to_copy['dest'])
 
-                # FIXME: we should just support '.xht' directly.
-                new_filepath = new_filepath.replace('.xht', '.xhtml')
-
                 if not(os.path.exists(os.path.dirname(new_filepath))):
                     os.makedirs(os.path.dirname(new_filepath))
 
                 if not self.options.overwrite and os.path.exists(new_filepath):
-                    print 'Skipping import of existing file ' + new_filepath
+                    _log.info('Skipping import of existing file ' + new_filepath)
                 else:
                     # FIXME: Maybe doing a file diff is in order here for existing files?
                     # In other words, there's no sense in overwriting identical files, but
                     # there's no harm in copying the identical thing.
-                    print 'Importing:', orig_filepath
-                    print '       As:', new_filepath
+                    _log.info('Importing: %s', orig_filepath)
+                    _log.info('       As: %s', new_filepath)
 
                 # Only html, xml, or css should be converted
                 # FIXME: Eventually, so should js when support is added for this type of conversion
                 mimetype = mimetypes.guess_type(orig_filepath)
                 if 'html' in str(mimetype[0]) or 'xml' in str(mimetype[0])  or 'css' in str(mimetype[0]):
-
                     converted_file = converter.convert_for_webkit(new_path, filename=orig_filepath)
 
                     if not converted_file:
                         shutil.copyfile(orig_filepath, new_filepath)  # The file was unmodified.
                     else:
+                        for prefixed_property in converted_file[0]:
+                            total_prefixed_properties.setdefault(prefixed_property, 0)
+                            total_prefixed_properties[prefixed_property] += 1
+
                         prefixed_properties.extend(set(converted_file[0]) - set(prefixed_properties))
-                        outfile = open(new_filepath, 'w')
+                        outfile = open(new_filepath, 'wb')
                         outfile.write(converted_file[1])
                         outfile.close()
                 else:
@@ -346,12 +358,16 @@ class TestImporter(object):
             self.remove_deleted_files(new_path, copied_files)
             self.write_import_log(new_path, copied_files, prefixed_properties)
 
-        print 'Import complete'
+        _log.info('Import complete')
 
-        print 'IMPORTED ' + str(total_imported_tests) + ' TOTAL TESTS'
-        print 'Imported ' + str(total_imported_reftests) + ' reftests'
-        print 'Imported ' + str(total_imported_jstests) + ' JS tests'
-        print 'Imported ' + str(total_imported_tests - total_imported_jstests - total_imported_reftests) + ' pixel/manual tests'
+        _log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
+        _log.info('Imported %d reftests', total_imported_reftests)
+        _log.info('Imported %d JS tests', total_imported_jstests)
+        _log.info('Imported %d pixel/manual tests', total_imported_tests - total_imported_jstests - total_imported_reftests)
+        _log.info('')
+        _log.info('Properties needing prefixes (by count):')
+        for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
+            _log.info('  %s: %s', prefixed_property, total_prefixed_properties[prefixed_property])
 
     def setup_destination_directory(self):
         """ Creates a destination directory that mirrors that of the source approved or submitted directory """
@@ -359,14 +375,14 @@ class TestImporter(object):
         self.update_test_status()
 
         start = self.source_directory.find(self.test_status)
-        new_subpath = self.source_directory[start:len(self.source_directory)]
+        new_subpath = self.source_directory[len(self.repo_dir):]
 
         destination_directory = os.path.join(self.destination_directory, new_subpath)
 
         if not os.path.exists(destination_directory):
             os.makedirs(destination_directory)
 
-        print 'Tests will be imported into: ' + destination_directory
+        _log.info('Tests will be imported into: %s', destination_directory)
 
     def update_test_status(self):
         """ Sets the test status to either 'approved' or 'submitted' """
@@ -398,7 +414,7 @@ class TestImporter(object):
 
         deleted_files = set(previous_file_list) - set(new_file_list)
         for deleted_file in deleted_files:
-            print 'Deleting file removed from the W3C repo:' + deleted_file
+            _log.info('Deleting file removed from the W3C repo: %s', deleted_file)
             deleted_file = os.path.join(self._webkit_root, deleted_file)
             os.remove(deleted_file)
 
index 08ea72d79bc761c21a9a15797378a7c55eaef72c..9103623dc79deb0c3a35851ffc76c7de363fe125 100644 (file)
@@ -32,22 +32,29 @@ import shutil
 import tempfile
 import unittest2 as unittest
 
-from webkitpy.common.host import Host
+from webkitpy.common.host_mock import MockHost
+from webkitpy.common.system.filesystem_mock import MockFileSystem
 from webkitpy.common.system.executive_mock import MockExecutive2, ScriptError
 from webkitpy.common.system.outputcapture import OutputCapture
 from webkitpy.w3c.test_importer import TestImporter
 
 
+FAKE_SOURCE_DIR = '/blink/w3c'
+FAKE_REPO_DIR = '/blink'
+
+FAKE_FILES = {
+    '/blink/w3c/empty_dir/README.txt': '',
+    '/mock-checkout/LayoutTests/w3c/README.txt': '',
+}
+
 class TestImporterTest(unittest.TestCase):
 
     def test_import_dir_with_no_tests_and_no_hg(self):
-        # FIXME: Use MockHosts instead.
-        host = Host()
+        host = MockHost()
         host.executive = MockExecutive2(exception=OSError())
+        host.filesystem = MockFileSystem(files=FAKE_FILES)
 
-        importer = TestImporter(host, None, optparse.Values({"overwrite": False}))
-        importer.source_directory = importer.path_from_webkit_root("Tools", "Scripts", "webkitpy", "w3c")
-        importer.destination_directory = tempfile.mkdtemp(prefix='csswg')
+        importer = TestImporter(host, FAKE_SOURCE_DIR, FAKE_REPO_DIR, optparse.Values({"overwrite": False}))
 
         oc = OutputCapture()
         oc.capture_output()
@@ -55,23 +62,18 @@ class TestImporterTest(unittest.TestCase):
             importer.do_import()
         finally:
             oc.restore_output()
-            shutil.rmtree(importer.destination_directory, ignore_errors=True)
 
     def test_import_dir_with_no_tests(self):
-        # FIXME: Use MockHosts instead.
-        host = Host()
+        host = MockHost()
         host.executive = MockExecutive2(exception=ScriptError("abort: no repository found in '/Volumes/Source/src/wk/Tools/Scripts/webkitpy/w3c' (.hg not found)!"))
+        host.filesystem = MockFileSystem(files=FAKE_FILES)
 
-        importer = TestImporter(host, None, optparse.Values({"overwrite": False}))
-        importer.source_directory = importer.path_from_webkit_root("Tools", "Scripts", "webkitpy", "w3c")
-        importer.destination_directory = tempfile.mkdtemp(prefix='csswg')
-
+        importer = TestImporter(host, FAKE_SOURCE_DIR, FAKE_REPO_DIR, optparse.Values({"overwrite": False}))
         oc = OutputCapture()
         oc.capture_output()
         try:
             importer.do_import()
         finally:
             oc.restore_output()
-            shutil.rmtree(importer.destination_directory, ignore_errors=True)
 
-    # FIXME: Need more tests, but need to add a mock filesystem w/ sample data.
+    # FIXME: Needs more tests.
index bdb24dfd656da1dbe11bedd8adcbe49743633313..bb66fdae63b7f25b06aab6866af3d1ef09ffa870 100644 (file)
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
+import logging
 import re
 
 from webkitpy.common.host import Host
 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup as Parser
 
 
+_log = logging.getLogger(__name__)
+
+
 class TestParser(object):
 
     def __init__(self, options, filename):
@@ -46,9 +50,17 @@ class TestParser(object):
         self.load_file(filename)
 
     def load_file(self, filename):
-        if self.filesystem.exists(filename):
-            self.test_doc = Parser(self.filesystem.read_text_file(filename))
+        if self.filesystem.isfile(filename):
+            try:
+                self.test_doc = Parser(self.filesystem.read_binary_file(filename))
+            except:
+                # FIXME: Figure out what to do if we can't parse the file.
+                _log.error("Failed to parse %s", filename)
+                self.test_doc is None
         else:
+            if self.filesystem.isdir(filename):
+                # FIXME: Figure out what is triggering this and what to do about it.
+                _log.error("Trying to load %s, which is a directory", filename)
             self.test_doc = None
         self.ref_doc = None
 
@@ -71,9 +83,17 @@ class TestParser(object):
         matches = self.reference_links_of_type('match') + self.reference_links_of_type('mismatch')
         if matches:
             if len(matches) > 1:
-                print 'Warning: Webkit does not support multiple references. Importing the first ref defined in ' + self.filesystem.basename(self.filename)
+                # FIXME: Is this actually true? We should fix this.
+                _log.warning('Multiple references are not supported. Importing the first ref defined in %s',
+                             self.filesystem.basename(self.filename))
+
+            try:
+                ref_file = self.filesystem.join(self.filesystem.dirname(self.filename), matches[0]['href'])
+            except KeyError as e:
+                # FIXME: Figure out what to do w/ invalid test files.
+                _log.error('%s has a reference link but is missing the "href"', self.filesystem)
+                return None
 
-            ref_file = self.filesystem.join(self.filesystem.dirname(self.filename), matches[0]['href'])
             if self.ref_doc is None:
                 self.ref_doc = self.load_file(ref_file)
 
@@ -92,7 +112,7 @@ class TestParser(object):
             # support files.
             #
             # *But*, there is exactly one case in the entire css2.1 suite where
-            # at test depends on a file that lives in a different directory,
+            # a test depends on a file that lives in a different directory,
             # which depends on another file that lives outside of its
             # directory. This code covers that case :)
             if matches[0]['href'].startswith('..'):
@@ -127,8 +147,8 @@ class TestParser(object):
         urls = []
         for url in doc.findAll(text=url_pattern):
             url = re.search(url_pattern, url)
-            url = re.sub('url\([\'\"]', '', url.group(0))
-            url = re.sub('[\'\"]\)', '', url)
+            url = re.sub('url\([\'\"]?', '', url.group(0))
+            url = re.sub('[\'\"]?\)', '', url)
             urls.append(url)
 
         src_paths = [src_tag['src'] for src_tag in elements_with_src_attributes]
index 3b697267c68a3d52b12a9361caa417aa43cbff89..7fb0c5b04a74bb01bccfd40aa2d6fe9cf16d1df7 100644 (file)
@@ -69,7 +69,7 @@ class TestParserTest(unittest.TestCase):
             parser = TestParser(options, test_path + 'somefile.html')
             test_info = parser.analyze_test(test_contents=test_html)
         finally:
-            output, _, _ = oc.restore_output()
+            _, _, logs = oc.restore_output()
 
         self.assertNotEqual(test_info, None, 'did not find a test')
         self.assertTrue('test' in test_info.keys(), 'did not find a test file')
@@ -78,7 +78,7 @@ class TestParserTest(unittest.TestCase):
         self.assertFalse('refsupport' in test_info.keys(), 'there should be no refsupport files for this test')
         self.assertFalse('jstest' in test_info.keys(), 'test should not have been analyzed as a jstest')
 
-        self.assertTrue(output.startswith('Warning'), 'no warning about multiple matches')
+        self.assertEqual(logs, 'Multiple references are not supported. Importing the first ref defined in somefile.html\n')
 
     def test_analyze_test_reftest_match_and_mismatch(self):
         test_html = """<head>
@@ -95,7 +95,7 @@ class TestParserTest(unittest.TestCase):
             parser = TestParser(options, test_path + 'somefile.html')
             test_info = parser.analyze_test(test_contents=test_html)
         finally:
-            output, _, _ = oc.restore_output()
+            _, _, logs = oc.restore_output()
 
         self.assertNotEqual(test_info, None, 'did not find a test')
         self.assertTrue('test' in test_info.keys(), 'did not find a test file')
@@ -104,8 +104,7 @@ class TestParserTest(unittest.TestCase):
         self.assertFalse('refsupport' in test_info.keys(), 'there should be no refsupport files for this test')
         self.assertFalse('jstest' in test_info.keys(), 'test should not have been analyzed as a jstest')
 
-        warnings = output.splitlines()
-        self.assertEquals(warnings, ["Warning: Webkit does not support multiple references. Importing the first ref defined in somefile.html"])
+        self.assertEqual(logs, 'Multiple references are not supported. Importing the first ref defined in somefile.html\n')
 
     def test_analyze_test_reftest_with_ref_support_Files(self):
         """ Tests analyze_test() using a reftest that has refers to a reference file outside of the tests directory and the reference file has paths to other support files """