3 # Copyright (C) 2013 Adobe Systems Incorporated. All rights reserved.
4 # Copyright (C) 2015 Canon Inc. All rights reserved.
6 # Redistribution and use in source and binary forms, with or without
7 # modification, are permitted provided that the following conditions
10 # 1. Redistributions of source code must retain the above
11 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER "AS IS" AND ANY
19 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE
22 # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
23 # OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
25 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
27 # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
28 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 This script imports W3C tests into WebKit, either from a local folder or by cloning W3C CSS and WPT repositories.
34 This script will import the tests into WebKit following these rules:
36 - All tests are by default imported into LayoutTests/imported/w3c
38 - Tests will be imported into a directory tree that
39 mirrors the CSS and WPT repositories. For example, <csswg_repo_root>/css2.1 should be brought in
40 as LayoutTests/imported/w3c/csswg-tests/css2.1, maintaining the entire directory structure under that
42 - By default, only reftests and jstest are imported. This can be overridden with a -a or --all
45 - Also by default, if test files by the same name already exist in the destination directory,
46 they are overwritten with the idea that running this script would refresh files periodically.
47 This can also be overridden by a -n or --no-overwrite flag
49 - If no import_directory is provided, the script will download the tests from the W3C github repositories.
50 The selection of tests and folders to import will be based on the following files:
51 1. LayoutTests/imported/w3c/resources/TestRepositories lists the repositories to clone, the corresponding revision to checkout and the infrastructure folders that need to be imported/skipped.
52 2. LayoutTests/imported/w3c/resources/ImportExpectations list the test suites or tests to NOT import.
54 - All files are converted to work in WebKit:
55 1. Paths to testharness.js files are modified to point to Webkit's copy of them in
56 LayoutTests/resources, using the correct relative path from the new location.
57 This is applied to CSS tests but not to WPT tests.
58 2. All CSS properties requiring the -webkit-vendor prefix are prefixed - this current
59 list of what needs prefixes is read from Source/WebCore/CSS/CSSProperties.in
60 3. Each reftest has its own copy of its reference file following the naming conventions
61 new-run-webkit-tests expects
62 4. If a reference files lives outside the directory of the test that uses it, it is checked
63 for paths to support files as it will be imported into a different relative position to the
64 test file (in the same directory)
66 - Upon completion, script outputs the total number tests imported, broken down by test type
68 - Also upon completion, each directory where files are imported will have w3c-import.log written
69 with a timestamp, the list of CSS properties used that require prefixes, the list of imported files,
70 and guidance for future test modification and maintenance.
72 - On subsequent imports, this file is read to determine if files have been removed in the newer changesets.
73 The script removes these files accordingly.
83 from webkitpy.common.host import Host
84 from webkitpy.common.system.filesystem import FileSystem
85 from webkitpy.common.webkit_finder import WebKitFinder
86 from webkitpy.common.system.executive import ScriptError
87 from webkitpy.w3c.test_parser import TestParser
88 from webkitpy.w3c.test_converter import convert_for_webkit
89 from webkitpy.w3c.test_downloader import TestDownloader
91 CHANGESET_NOT_AVAILABLE = 'Not Available'
93 _log = logging.getLogger(__name__)
96 def main(_argv, _stdout, _stderr):
97 options, args = parse_args(_argv)
98 import_dir = args[0] if args else None
99 filesystem = FileSystem()
100 if import_dir and not filesystem.exists(import_dir):
101 sys.exit('Source directory %s not found!' % import_dir)
105 test_importer = TestImporter(Host(), import_dir, options)
106 test_importer.do_import()
109 def configure_logging():
110 class LogHandler(logging.StreamHandler):
112 def format(self, record):
113 if record.levelno > logging.INFO:
114 return "%s: %s" % (record.levelname, record.getMessage())
115 return record.getMessage()
117 logger = logging.getLogger()
118 logger.setLevel(logging.INFO)
119 handler = LogHandler()
120 handler.setLevel(logging.INFO)
121 logger.addHandler(handler)
125 def parse_args(args):
126 parser = argparse.ArgumentParser(prog='import-w3c-tests [w3c_test_source_directory]')
128 parser.add_argument('-n', '--no-overwrite', dest='overwrite', action='store_false', default=True,
129 help='Flag to prevent duplicate test files from overwriting existing tests. By default, they will be overwritten')
130 parser.add_argument('-l', '--no-links-conversion', dest='convert_test_harness_links', action='store_false', default=True,
131 help='Do not change links (testharness js or css e.g.). This option only applies when providing a source directory, in which case by default, links are converted to point to WebKit testharness files. When tests are downloaded from W3C repository, links are converted for CSS tests and remain unchanged for WPT tests')
133 parser.add_argument('-a', '--all', action='store_true', default=False,
134 help='Import all tests including reftests, JS tests, and manual/pixel tests. By default, only reftests and JS tests are imported')
136 parser.add_argument('-d', '--dest-dir', dest='destination', default=fs.join('imported', 'w3c'),
137 help='Import into a specified directory relative to the LayoutTests root. By default, imports into imported/w3c')
139 list_of_repositories = ' or '.join([test_repository['name'] for test_repository in TestDownloader.load_test_repositories()])
140 parser.add_argument('-t', '--test-path', action='append', dest='test_paths', default=[],
141 help='Import only tests in the supplied subdirectory of the source directory. Can be supplied multiple times to give multiple paths. For tests directly cloned from W3C repositories, use ' + list_of_repositories + ' prefixes to filter specific tests')
143 parser.add_argument('-v', '--verbose', action='store_true', default=False,
144 help='Print maximal log')
145 parser.add_argument('--no-fetch', action='store_false', dest='fetch', default=True,
146 help='Do not fetch the repositories. By default, repositories are fetched if a source directory is not provided')
147 parser.add_argument('--import-all', action='store_true', default=False,
148 help='Ignore the ImportExpectations file. All tests will be imported. This option only applies when tests are downloaded from W3C repository')
150 options, args = parser.parse_known_args(args)
152 parser.error('Incorrect number of arguments')
156 class TestImporter(object):
158 def __init__(self, host, source_directory, options):
160 self.source_directory = source_directory
161 self.options = options
163 self.filesystem = self.host.filesystem
165 webkit_finder = WebKitFinder(self.filesystem)
166 self._webkit_root = webkit_finder.webkit_base()
168 self.destination_directory = webkit_finder.path_from_webkit_base("LayoutTests", options.destination)
169 self.layout_tests_w3c_path = webkit_finder.path_from_webkit_base('LayoutTests', 'imported', 'w3c')
170 self.tests_download_path = webkit_finder.path_from_webkit_base('WebKitBuild', 'w3c-tests')
172 self._test_downloader = None
174 self.import_list = []
175 self._importing_downloaded_tests = source_directory is None
178 if not self.source_directory:
179 _log.info('Downloading W3C test repositories')
180 self.source_directory = self.filesystem.join(self.tests_download_path, 'to-be-imported')
181 self.filesystem.maybe_make_directory(self.tests_download_path)
182 self.filesystem.maybe_make_directory(self.source_directory)
183 self.test_downloader().download_tests(self.source_directory, self.options.test_paths)
185 if not self.options.test_paths or self._importing_downloaded_tests:
186 self.find_importable_tests(self.source_directory)
188 for test_path in self.options.test_paths:
189 self.find_importable_tests(self.filesystem.join(self.source_directory, test_path))
193 if self._importing_downloaded_tests:
194 self.generate_git_submodules_description_for_all_repositories()
196 def generate_git_submodules_description_for_all_repositories(self):
197 for test_repository in self._test_downloader.test_repositories:
198 if 'generate_git_submodules_description' in test_repository['import_options']:
199 self.filesystem.maybe_make_directory(self.filesystem.join(self.destination_directory, 'resources'))
200 self._test_downloader.generate_git_submodules_description(test_repository, self.filesystem.join(self.destination_directory, 'resources', test_repository['name'] + '-modules.json'))
201 # FIXME: Generate WPT .gitignore and main __init__.py
203 def test_downloader(self):
204 if not self._test_downloader:
205 download_options = TestDownloader.default_options()
206 download_options.fetch = self.options.fetch
207 download_options.verbose = self.options.verbose
208 download_options.import_all = self.options.import_all
209 self._test_downloader = TestDownloader(self.tests_download_path, self.host, download_options)
210 return self._test_downloader
212 def should_skip_file(self, filename):
213 # For some reason the w3c repo contains random perl scripts we don't care about.
214 if filename.endswith('.pl'):
216 if filename.startswith('.'):
217 return not filename == '.htaccess'
220 def find_importable_tests(self, directory):
221 def should_keep_subdir(filesystem, path):
222 if self._importing_downloaded_tests:
224 subdir = path[len(directory):]
225 DIRS_TO_SKIP = ('work-in-progress', 'tools', 'support')
226 should_skip = filesystem.basename(subdir).startswith('.') or (subdir in DIRS_TO_SKIP)
227 return not should_skip
229 directories = self.filesystem.dirs_under(directory, should_keep_subdir)
230 for root in directories:
231 _log.info('Scanning ' + root + '...')
238 for filename in self.filesystem.listdir(root):
239 if self.filesystem.isdir(self.filesystem.join(root, filename)):
241 # FIXME: This block should really be a separate function, but the early-continues make that difficult.
243 if self.should_skip_file(filename):
246 fullpath = self.filesystem.join(root, filename)
248 mimetype = mimetypes.guess_type(fullpath)
249 if not 'html' in str(mimetype[0]) and not 'application/xhtml+xml' in str(mimetype[0]) and not 'application/xml' in str(mimetype[0]):
250 copy_list.append({'src': fullpath, 'dest': filename})
253 test_parser = TestParser(vars(self.options), filename=fullpath, host=self.host)
254 test_info = test_parser.analyze_test()
255 if test_info is None:
256 # If html file is in a "resources" folder, it should be copied anyway
257 if self.filesystem.basename(self.filesystem.dirname(fullpath)) == "resources":
258 copy_list.append({'src': fullpath, 'dest': filename})
261 if 'reference' in test_info.keys():
264 test_basename = self.filesystem.basename(test_info['test'])
266 # Add the ref file, following WebKit style.
267 # FIXME: Ideally we'd support reading the metadata
268 # directly rather than relying on a naming convention.
269 # Using a naming convention creates duplicate copies of the
271 ref_file = self.filesystem.splitext(test_basename)[0] + '-expected'
272 ref_file += self.filesystem.splitext(test_info['reference'])[1]
274 copy_list.append({'src': test_info['reference'], 'dest': ref_file, 'reference_support_info': test_info['reference_support_info']})
275 copy_list.append({'src': test_info['test'], 'dest': filename})
277 elif 'jstest' in test_info.keys():
280 copy_list.append({'src': fullpath, 'dest': filename})
283 copy_list.append({'src': fullpath, 'dest': filename})
286 # Only add this directory to the list if there's something to import
287 self.import_list.append({'dirname': root, 'copy_list': copy_list,
288 'reftests': reftests, 'jstests': jstests, 'total_tests': total_tests})
290 def should_convert_test_harness_links(self, test):
291 if self._importing_downloaded_tests:
292 for test_repository in self.test_downloader().test_repositories:
293 if test.startswith(test_repository['name']):
294 return 'convert_test_harness_links' in test_repository['import_options']
296 return self.options.convert_test_harness_links
298 def import_tests(self):
299 total_imported_tests = 0
300 total_imported_reftests = 0
301 total_imported_jstests = 0
302 total_prefixed_properties = {}
303 total_prefixed_property_values = {}
305 failed_conversion_files = []
307 for dir_to_copy in self.import_list:
308 total_imported_tests += dir_to_copy['total_tests']
309 total_imported_reftests += dir_to_copy['reftests']
310 total_imported_jstests += dir_to_copy['jstests']
312 prefixed_properties = []
313 prefixed_property_values = []
315 if not dir_to_copy['copy_list']:
318 orig_path = dir_to_copy['dirname']
320 subpath = self.filesystem.relpath(orig_path, self.source_directory)
321 new_path = self.filesystem.join(self.destination_directory, subpath)
323 if not(self.filesystem.exists(new_path)):
324 self.filesystem.maybe_make_directory(new_path)
328 for file_to_copy in dir_to_copy['copy_list']:
329 # FIXME: Split this block into a separate function.
330 orig_filepath = self.filesystem.normpath(file_to_copy['src'])
332 if self.filesystem.isdir(orig_filepath):
333 # FIXME: Figure out what is triggering this and what to do about it.
334 _log.error('%s refers to a directory' % orig_filepath)
337 if not(self.filesystem.exists(orig_filepath)):
338 _log.warning('%s not found. Possible error in the test.', orig_filepath)
341 new_filepath = self.filesystem.join(new_path, file_to_copy['dest'])
342 if 'reference_support_info' in file_to_copy.keys() and file_to_copy['reference_support_info'] != {}:
343 reference_support_info = file_to_copy['reference_support_info']
345 reference_support_info = None
347 if not(self.filesystem.exists(self.filesystem.dirname(new_filepath))):
348 self.filesystem.maybe_make_directory(self.filesystem.dirname(new_filepath))
350 if not self.options.overwrite and self.filesystem.exists(new_filepath):
351 _log.info('Skipping import of existing file ' + new_filepath)
353 # FIXME: Maybe doing a file diff is in order here for existing files?
354 # In other words, there's no sense in overwriting identical files, but
355 # there's no harm in copying the identical thing.
356 _log.info('Importing: %s', orig_filepath)
357 _log.info(' As: %s', new_filepath)
359 # Only html, xml, or css should be converted
360 # FIXME: Eventually, so should js when support is added for this type of conversion
361 mimetype = mimetypes.guess_type(orig_filepath)
362 if 'html' in str(mimetype[0]) or 'xml' in str(mimetype[0]) or 'css' in str(mimetype[0]):
364 converted_file = convert_for_webkit(new_path, filename=orig_filepath, reference_support_info=reference_support_info, host=self.host, convert_test_harness_links=self.should_convert_test_harness_links(subpath))
366 _log.warn('Failed converting %s', orig_filepath)
367 failed_conversion_files.append(orig_filepath)
368 converted_file = None
370 if not converted_file:
371 self.filesystem.copyfile(orig_filepath, new_filepath) # The file was unmodified.
373 for prefixed_property in converted_file[0]:
374 total_prefixed_properties.setdefault(prefixed_property, 0)
375 total_prefixed_properties[prefixed_property] += 1
377 prefixed_properties.extend(set(converted_file[0]) - set(prefixed_properties))
379 for prefixed_value in converted_file[1]:
380 total_prefixed_property_values.setdefault(prefixed_value, 0)
381 total_prefixed_property_values[prefixed_value] += 1
383 prefixed_property_values.extend(set(converted_file[1]) - set(prefixed_property_values))
385 self.filesystem.write_binary_file(new_filepath, converted_file[2])
386 elif orig_filepath.endswith('__init__.py') and not self.filesystem.getsize(orig_filepath):
387 # Some bots dislike empty __init__.py.
388 self.filesystem.write_text_file(new_filepath, '# This file is required for Python to search this directory for modules.')
390 self.filesystem.copyfile(orig_filepath, new_filepath)
392 copied_files.append(new_filepath.replace(self._webkit_root, ''))
394 self.remove_deleted_files(new_path, copied_files)
395 self.write_import_log(new_path, copied_files, prefixed_properties, prefixed_property_values)
397 _log.info('Import complete')
399 _log.info('IMPORTED %d TOTAL TESTS', total_imported_tests)
400 _log.info('Imported %d reftests', total_imported_reftests)
401 _log.info('Imported %d JS tests', total_imported_jstests)
402 _log.info('Imported %d pixel/manual tests', total_imported_tests - total_imported_jstests - total_imported_reftests)
403 if len(failed_conversion_files):
404 _log.warn('Failed converting %d files (files copied without being converted)', len(failed_conversion_files))
406 _log.info('Properties needing prefixes (by count):')
408 for prefixed_property in sorted(total_prefixed_properties, key=lambda p: total_prefixed_properties[p]):
409 _log.info(' %s: %s', prefixed_property, total_prefixed_properties[prefixed_property])
411 _log.info('Property values needing prefixes (by count):')
413 for prefixed_value in sorted(total_prefixed_property_values, key=lambda p: total_prefixed_property_values[p]):
414 _log.info(' %s: %s', prefixed_value, total_prefixed_property_values[prefixed_value])
416 def remove_deleted_files(self, import_directory, new_file_list):
417 """ Reads an import log in |import_directory|, compares it to the |new_file_list|, and removes files not in the new list."""
419 previous_file_list = []
421 import_log_file = self.filesystem.join(import_directory, 'w3c-import.log')
422 if not self.filesystem.exists(import_log_file):
425 contents = self.filesystem.read_text_file(import_log_file).split('\n')
427 if 'List of files\n' in contents:
428 list_index = contents.index('List of files:\n') + 1
429 previous_file_list = [filename.strip() for filename in contents[list_index:]]
431 deleted_files = set(previous_file_list) - set(new_file_list)
432 for deleted_file in deleted_files:
433 _log.info('Deleting file removed from the W3C repo: %s', deleted_file)
434 deleted_file = self.filesystem.join(self._webkit_root, deleted_file)
435 self.filesystem.remove(deleted_file)
437 def write_import_log(self, import_directory, file_list, prop_list, property_values_list):
438 """ Writes a w3c-import.log file in each directory with imported files. """
441 import_log.append('The tests in this directory were imported from the W3C repository.\n')
442 import_log.append('Do NOT modify these tests directly in Webkit.\n')
443 import_log.append('Instead, create a pull request on the W3C CSS or WPT github:\n')
444 import_log.append('\thttps://github.com/w3c/csswg-test\n')
445 import_log.append('\thttps://github.com/w3c/web-platform-tests\n\n')
446 import_log.append('Then run the Tools/Scripts/import-w3c-tests in Webkit to reimport\n\n')
447 import_log.append('Do NOT modify or remove this file\n\n')
448 import_log.append('------------------------------------------------------------------------\n')
449 import_log.append('Properties requiring vendor prefixes:\n')
451 for prop in prop_list:
452 import_log.append(prop + '\n')
454 import_log.append('None\n')
455 import_log.append('Property values requiring vendor prefixes:\n')
456 if property_values_list:
457 for value in property_values_list:
458 import_log.append(value + '\n')
460 import_log.append('None\n')
461 import_log.append('------------------------------------------------------------------------\n')
462 import_log.append('List of files:\n')
463 for item in sorted(file_list):
464 import_log.append(item + '\n')
466 self.filesystem.write_text_file(self.filesystem.join(import_directory, 'w3c-import.log'), ''.join(import_log))