Unreviewed. Update W3C WebDriver imported tests.
[WebKit-https.git] / WebDriverTests / imported / w3c / tools / wptrunner / wptrunner / metadata.py
1 import os
2 import shutil
3 import sys
4 import tempfile
5 import types
6 import uuid
7 from collections import defaultdict
8
9 from mozlog import reader
10 from mozlog import structuredlog
11
12 import expected
13 import manifestupdate
14 import testloader
15 import wptmanifest
16 import wpttest
17 from vcs import git
18 manifest = None  # Module that will be imported relative to test_root
19 manifestitem = None
20
21 logger = structuredlog.StructuredLogger("web-platform-tests")
22
23
24 def load_test_manifests(serve_root, test_paths):
25     do_delayed_imports(serve_root)
26     manifest_loader = testloader.ManifestLoader(test_paths, False)
27     return manifest_loader.load()
28
29
30 def update_expected(test_paths, serve_root, log_file_names,
31                     rev_old=None, rev_new="HEAD", ignore_existing=False,
32                     sync_root=None, property_order=None, boolean_properties=None,
33                     stability=None):
34     """Update the metadata files for web-platform-tests based on
35     the results obtained in a previous run or runs
36
37     If stability is not None, assume log_file_names refers to logs from repeated
38     test jobs, disable tests that don't behave as expected on all runs"""
39
40     manifests = load_test_manifests(serve_root, test_paths)
41
42     change_data = {}
43
44     if sync_root is not None:
45         if rev_old is not None:
46             rev_old = git("rev-parse", rev_old, repo=sync_root).strip()
47         rev_new = git("rev-parse", rev_new, repo=sync_root).strip()
48
49         if rev_old is not None:
50             change_data = load_change_data(rev_old, rev_new, repo=sync_root)
51
52     expected_map_by_manifest = update_from_logs(manifests,
53                                                 *log_file_names,
54                                                 ignore_existing=ignore_existing,
55                                                 property_order=property_order,
56                                                 boolean_properties=boolean_properties,
57                                                 stability=stability)
58
59     for test_manifest, expected_map in expected_map_by_manifest.iteritems():
60         url_base = manifests[test_manifest]["url_base"]
61         metadata_path = test_paths[url_base]["metadata_path"]
62         write_changes(metadata_path, expected_map)
63         if stability is not None:
64             for tree in expected_map.itervalues():
65                 for test in tree.iterchildren():
66                     for subtest in test.iterchildren():
67                         if subtest.new_disabled:
68                             print "disabled: %s" % os.path.dirname(subtest.root.test_path) + "/" + subtest.name
69                     if test.new_disabled:
70                         print "disabled: %s" % test.root.test_path
71
72     results_changed = [item.test_path for item in expected_map.itervalues() if item.modified]
73
74     return unexpected_changes(manifests, change_data, results_changed)
75
76
77 def do_delayed_imports(serve_root):
78     global manifest, manifestitem
79     from manifest import manifest, item as manifestitem
80
81
82 def files_in_repo(repo_root):
83     return git("ls-tree", "-r", "--name-only", "HEAD").split("\n")
84
85
86 def rev_range(rev_old, rev_new, symmetric=False):
87     joiner = ".." if not symmetric else "..."
88     return "".join([rev_old, joiner, rev_new])
89
90
91 def paths_changed(rev_old, rev_new, repo):
92     data = git("diff", "--name-status", rev_range(rev_old, rev_new), repo=repo)
93     lines = [tuple(item.strip() for item in line.strip().split("\t", 1))
94              for line in data.split("\n") if line.strip()]
95     output = set(lines)
96     return output
97
98
99 def load_change_data(rev_old, rev_new, repo):
100     changes = paths_changed(rev_old, rev_new, repo)
101     rv = {}
102     status_keys = {"M": "modified",
103                    "A": "new",
104                    "D": "deleted"}
105     # TODO: deal with renames
106     for item in changes:
107         rv[item[1]] = status_keys[item[0]]
108     return rv
109
110
111 def unexpected_changes(manifests, change_data, files_changed):
112     files_changed = set(files_changed)
113
114     root_manifest = None
115     for manifest, paths in manifests.iteritems():
116         if paths["url_base"] == "/":
117             root_manifest = manifest
118             break
119     else:
120         return []
121
122     rv = []
123
124     return [fn for _, fn, _ in root_manifest if fn in files_changed and change_data.get(fn) != "M"]
125
126 # For each testrun
127 # Load all files and scan for the suite_start entry
128 # Build a hash of filename: properties
129 # For each different set of properties, gather all chunks
130 # For each chunk in the set of chunks, go through all tests
131 # for each test, make a map of {conditionals: [(platform, new_value)]}
132 # Repeat for each platform
133 # For each test in the list of tests:
134 #   for each conditional:
135 #      If all the new values match (or there aren't any) retain that conditional
136 #      If any new values mismatch:
137 #           If stability and any repeated values don't match, disable the test
138 #           else mark the test as needing human attention
139 #   Check if all the RHS values are the same; if so collapse the conditionals
140
141
142 def update_from_logs(manifests, *log_filenames, **kwargs):
143     ignore_existing = kwargs.get("ignore_existing", False)
144     property_order = kwargs.get("property_order")
145     boolean_properties = kwargs.get("boolean_properties")
146     stability = kwargs.get("stability")
147
148     expected_map = {}
149     id_test_map = {}
150
151     for test_manifest, paths in manifests.iteritems():
152         expected_map_manifest, id_path_map_manifest = create_test_tree(
153             paths["metadata_path"],
154             test_manifest,
155             property_order=property_order,
156             boolean_properties=boolean_properties)
157         expected_map[test_manifest] = expected_map_manifest
158         id_test_map.update(id_path_map_manifest)
159
160     updater = ExpectedUpdater(manifests, expected_map, id_test_map,
161                               ignore_existing=ignore_existing)
162     for log_filename in log_filenames:
163         with open(log_filename) as f:
164             updater.update_from_log(f)
165
166     for manifest_expected in expected_map.itervalues():
167         for tree in manifest_expected.itervalues():
168             for test in tree.iterchildren():
169                 for subtest in test.iterchildren():
170                     subtest.coalesce_expected(stability=stability)
171                 test.coalesce_expected(stability=stability)
172
173     return expected_map
174
175 def directory_manifests(metadata_path):
176     rv = []
177     for dirpath, dirname, filenames in os.walk(metadata_path):
178         if "__dir__.ini" in filenames:
179             rel_path = os.path.relpath(dirpath, metadata_path)
180             rv.append(os.path.join(rel_path, "__dir__.ini"))
181     return rv
182
183 def write_changes(metadata_path, expected_map):
184     # First write the new manifest files to a temporary directory
185     temp_path = tempfile.mkdtemp(dir=os.path.split(metadata_path)[0])
186     write_new_expected(temp_path, expected_map)
187
188     # Keep all __dir__.ini files (these are not in expected_map because they
189     # aren't associated with a specific test)
190     keep_files = directory_manifests(metadata_path)
191
192     # Copy all files in the root to the temporary location since
193     # these cannot be ini files
194     keep_files.extend(item for item in os.listdir(metadata_path) if
195                       not os.path.isdir(os.path.join(metadata_path, item)))
196
197     for item in keep_files:
198         dest_dir = os.path.dirname(os.path.join(temp_path, item))
199         if not os.path.exists(dest_dir):
200             os.makedirs(dest_dir)
201         shutil.copyfile(os.path.join(metadata_path, item),
202                         os.path.join(temp_path, item))
203
204     # Then move the old manifest files to a new location
205     temp_path_2 = metadata_path + str(uuid.uuid4())
206     os.rename(metadata_path, temp_path_2)
207     # Move the new files to the destination location and remove the old files
208     os.rename(temp_path, metadata_path)
209     shutil.rmtree(temp_path_2)
210
211
212 def write_new_expected(metadata_path, expected_map):
213     # Serialize the data back to a file
214     for tree in expected_map.itervalues():
215         if not tree.is_empty:
216             manifest_str = wptmanifest.serialize(tree.node, skip_empty_data=True)
217             assert manifest_str != ""
218             path = expected.expected_path(metadata_path, tree.test_path)
219             dir = os.path.split(path)[0]
220             if not os.path.exists(dir):
221                 os.makedirs(dir)
222             with open(path, "wb") as f:
223                 f.write(manifest_str)
224
225
226 class ExpectedUpdater(object):
227     def __init__(self, test_manifests, expected_tree, id_path_map, ignore_existing=False):
228         self.test_manifests = test_manifests
229         self.expected_tree = expected_tree
230         self.id_path_map = id_path_map
231         self.ignore_existing = ignore_existing
232         self.run_info = None
233         self.action_map = {"suite_start": self.suite_start,
234                            "test_start": self.test_start,
235                            "test_status": self.test_status,
236                            "test_end": self.test_end}
237         self.tests_visited = {}
238
239         self.test_cache = {}
240
241     def update_from_log(self, log_file):
242         self.run_info = None
243         log_reader = reader.read(log_file)
244         reader.each_log(log_reader, self.action_map)
245
246     def suite_start(self, data):
247         self.run_info = data["run_info"]
248
249     def test_type(self, path):
250         for manifest in self.test_manifests.iterkeys():
251             tests = list(manifest.iterpath(path))
252             if len(tests):
253                 assert all(test.item_type == tests[0].item_type for test in tests)
254                 return tests[0].item_type
255         assert False
256
257     def test_start(self, data):
258         test_id = data["test"]
259         try:
260             test_manifest, test = self.id_path_map[test_id]
261             expected_node = self.expected_tree[test_manifest][test].get_test(test_id)
262         except KeyError:
263             print "Test not found %s, skipping" % test_id
264             return
265         self.test_cache[test_id] = expected_node
266
267         if test_id not in self.tests_visited:
268             if self.ignore_existing:
269                 expected_node.clear_expected()
270             self.tests_visited[test_id] = set()
271
272     def test_status(self, data):
273         test = self.test_cache.get(data["test"])
274         if test is None:
275             return
276         test_cls = wpttest.manifest_test_cls[self.test_type(test.root.test_path)]
277
278         subtest = test.get_subtest(data["subtest"])
279
280         self.tests_visited[test.id].add(data["subtest"])
281
282         result = test_cls.subtest_result_cls(
283             data["subtest"],
284             data["status"],
285             data.get("message"))
286
287         subtest.set_result(self.run_info, result)
288
289     def test_end(self, data):
290         test_id = data["test"]
291         test = self.test_cache.get(test_id)
292         if test is None:
293             return
294         test_cls = wpttest.manifest_test_cls[self.test_type(test.root.test_path)]
295
296         if data["status"] == "SKIP":
297             return
298
299         result = test_cls.result_cls(
300             data["status"],
301             data.get("message"))
302         test.set_result(self.run_info, result)
303         del self.test_cache[test_id]
304
305
306 def create_test_tree(metadata_path, test_manifest, property_order=None,
307                      boolean_properties=None):
308     expected_map = {}
309     id_test_map = {}
310     exclude_types = frozenset(["stub", "helper", "manual", "support", "conformancechecker"])
311     all_types = [item.item_type for item in manifestitem.__dict__.itervalues()
312                  if type(item) == type and
313                  issubclass(item, manifestitem.ManifestItem) and
314                  item.item_type is not None]
315     include_types = set(all_types) - exclude_types
316     for _, test_path, tests in test_manifest.itertypes(*include_types):
317         expected_data = load_expected(test_manifest, metadata_path, test_path, tests,
318                                       property_order=property_order,
319                                       boolean_properties=boolean_properties)
320         if expected_data is None:
321             expected_data = create_expected(test_manifest,
322                                             test_path,
323                                             tests,
324                                             property_order=property_order,
325                                             boolean_properties=boolean_properties)
326
327         for test in tests:
328             id_test_map[test.id] = (test_manifest, test)
329             expected_map[test] = expected_data
330
331     return expected_map, id_test_map
332
333
334 def create_expected(test_manifest, test_path, tests, property_order=None,
335                     boolean_properties=None):
336     expected = manifestupdate.ExpectedManifest(None, test_path, test_manifest.url_base,
337                                                property_order=property_order,
338                                                boolean_properties=boolean_properties)
339     for test in tests:
340         expected.append(manifestupdate.TestNode.create(test.id))
341     return expected
342
343
344 def load_expected(test_manifest, metadata_path, test_path, tests, property_order=None,
345                   boolean_properties=None):
346     expected_manifest = manifestupdate.get_manifest(metadata_path,
347                                                     test_path,
348                                                     test_manifest.url_base,
349                                                     property_order=property_order,
350                                                     boolean_properties=boolean_properties)
351     if expected_manifest is None:
352         return
353
354     tests_by_id = {item.id: item for item in tests}
355
356     # Remove expected data for tests that no longer exist
357     for test in expected_manifest.iterchildren():
358         if test.id not in tests_by_id:
359             test.remove()
360
361     # Add tests that don't have expected data
362     for test in tests:
363         if not expected_manifest.has_test(test.id):
364             expected_manifest.append(manifestupdate.TestNode.create(test.id))
365
366     return expected_manifest