Unreviewed. Update W3C WebDriver imported tests.
[WebKit-https.git] / WebDriverTests / imported / w3c / tools / wptrunner / wptrunner / executors / base.py
1 import hashlib
2 import httplib
3 import os
4 import threading
5 import traceback
6 import socket
7 import urlparse
8 from abc import ABCMeta, abstractmethod
9
10 from ..testrunner import Stop
11
12 here = os.path.split(__file__)[0]
13
14 # Extra timeout to use after internal test timeout at which the harness
15 # should force a timeout
16 extra_timeout = 5 # seconds
17
18
19 def executor_kwargs(test_type, server_config, cache_manager, **kwargs):
20     timeout_multiplier = kwargs["timeout_multiplier"]
21     if timeout_multiplier is None:
22         timeout_multiplier = 1
23
24     executor_kwargs = {"server_config": server_config,
25                        "timeout_multiplier": timeout_multiplier,
26                        "debug_info": kwargs["debug_info"]}
27
28     if test_type == "reftest":
29         executor_kwargs["screenshot_cache"] = cache_manager.dict()
30
31     if test_type == "wdspec":
32         executor_kwargs["binary"] = kwargs.get("binary")
33         executor_kwargs["webdriver_binary"] = kwargs.get("webdriver_binary")
34         executor_kwargs["webdriver_args"] = kwargs.get("webdriver_args")
35
36     return executor_kwargs
37
38
39 def strip_server(url):
40     """Remove the scheme and netloc from a url, leaving only the path and any query
41     or fragment.
42
43     url - the url to strip
44
45     e.g. http://example.org:8000/tests?id=1#2 becomes /tests?id=1#2"""
46
47     url_parts = list(urlparse.urlsplit(url))
48     url_parts[0] = ""
49     url_parts[1] = ""
50     return urlparse.urlunsplit(url_parts)
51
52
53 class TestharnessResultConverter(object):
54     harness_codes = {0: "OK",
55                      1: "ERROR",
56                      2: "TIMEOUT"}
57
58     test_codes = {0: "PASS",
59                   1: "FAIL",
60                   2: "TIMEOUT",
61                   3: "NOTRUN"}
62
63     def __call__(self, test, result):
64         """Convert a JSON result into a (TestResult, [SubtestResult]) tuple"""
65         result_url, status, message, stack, subtest_results = result
66         assert result_url == test.url, ("Got results from %s, expected %s" %
67                                       (result_url, test.url))
68         harness_result = test.result_cls(self.harness_codes[status], message)
69         return (harness_result,
70                 [test.subtest_result_cls(name, self.test_codes[status], message, stack)
71                  for name, status, message, stack in subtest_results])
72
73
74 testharness_result_converter = TestharnessResultConverter()
75
76
77 def reftest_result_converter(self, test, result):
78     return (test.result_cls(result["status"], result["message"],
79                             extra=result.get("extra")), [])
80
81
82 def pytest_result_converter(self, test, data):
83     harness_data, subtest_data = data
84
85     if subtest_data is None:
86         subtest_data = []
87
88     harness_result = test.result_cls(*harness_data)
89     subtest_results = [test.subtest_result_cls(*item) for item in subtest_data]
90
91     return (harness_result, subtest_results)
92
93
94 class ExecutorException(Exception):
95     def __init__(self, status, message):
96         self.status = status
97         self.message = message
98
99
100 class TestExecutor(object):
101     __metaclass__ = ABCMeta
102
103     test_type = None
104     convert_result = None
105     supports_testdriver = False
106
107     def __init__(self, browser, server_config, timeout_multiplier=1,
108                  debug_info=None, **kwargs):
109         """Abstract Base class for object that actually executes the tests in a
110         specific browser. Typically there will be a different TestExecutor
111         subclass for each test type and method of executing tests.
112
113         :param browser: ExecutorBrowser instance providing properties of the
114                         browser that will be tested.
115         :param server_config: Dictionary of wptserve server configuration of the
116                               form stored in TestEnvironment.external_config
117         :param timeout_multiplier: Multiplier relative to base timeout to use
118                                    when setting test timeout.
119         """
120         self.runner = None
121         self.browser = browser
122         self.server_config = server_config
123         self.timeout_multiplier = timeout_multiplier
124         self.debug_info = debug_info
125         self.last_environment = {"protocol": "http",
126                                  "prefs": {}}
127         self.protocol = None # This must be set in subclasses
128
129     @property
130     def logger(self):
131         """StructuredLogger for this executor"""
132         if self.runner is not None:
133             return self.runner.logger
134
135     def setup(self, runner):
136         """Run steps needed before tests can be started e.g. connecting to
137         browser instance
138
139         :param runner: TestRunner instance that is going to run the tests"""
140         self.runner = runner
141         if self.protocol is not None:
142             self.protocol.setup(runner)
143
144     def teardown(self):
145         """Run cleanup steps after tests have finished"""
146         if self.protocol is not None:
147             self.protocol.teardown()
148
149     def run_test(self, test):
150         """Run a particular test.
151
152         :param test: The test to run"""
153         if test.environment != self.last_environment:
154             self.on_environment_change(test.environment)
155
156         try:
157             result = self.do_test(test)
158         except Exception as e:
159             result = self.result_from_exception(test, e)
160
161         if result is Stop:
162             return result
163
164         # log result of parent test
165         if result[0].status == "ERROR":
166             self.logger.debug(result[0].message)
167
168         self.last_environment = test.environment
169
170         self.runner.send_message("test_ended", test, result)
171
172     def server_url(self, protocol):
173         return "%s://%s:%s" % (protocol,
174                                self.server_config["host"],
175                                self.server_config["ports"][protocol][0])
176
177     def test_url(self, test):
178         return urlparse.urljoin(self.server_url(test.environment["protocol"]), test.url)
179
180     @abstractmethod
181     def do_test(self, test):
182         """Test-type and protocol specific implementation of running a
183         specific test.
184
185         :param test: The test to run."""
186         pass
187
188     def on_environment_change(self, new_environment):
189         pass
190
191     def result_from_exception(self, test, e):
192         if hasattr(e, "status") and e.status in test.result_cls.statuses:
193             status = e.status
194         else:
195             status = "ERROR"
196         message = unicode(getattr(e, "message", ""))
197         if message:
198             message += "\n"
199         message += traceback.format_exc(e)
200         return test.result_cls(status, message), []
201
202
203 class TestharnessExecutor(TestExecutor):
204     convert_result = testharness_result_converter
205
206
207 class RefTestExecutor(TestExecutor):
208     convert_result = reftest_result_converter
209
210     def __init__(self, browser, server_config, timeout_multiplier=1, screenshot_cache=None,
211                  debug_info=None, **kwargs):
212         TestExecutor.__init__(self, browser, server_config,
213                               timeout_multiplier=timeout_multiplier,
214                               debug_info=debug_info)
215
216         self.screenshot_cache = screenshot_cache
217
218
219 class RefTestImplementation(object):
220     def __init__(self, executor):
221         self.timeout_multiplier = executor.timeout_multiplier
222         self.executor = executor
223         # Cache of url:(screenshot hash, screenshot). Typically the
224         # screenshot is None, but we set this value if a test fails
225         # and the screenshot was taken from the cache so that we may
226         # retrieve the screenshot from the cache directly in the future
227         self.screenshot_cache = self.executor.screenshot_cache
228         self.message = None
229
230     def setup(self):
231         pass
232
233     def teardown(self):
234         pass
235
236     @property
237     def logger(self):
238         return self.executor.logger
239
240     def get_hash(self, test, viewport_size, dpi):
241         timeout = test.timeout * self.timeout_multiplier
242         key = (test.url, viewport_size, dpi)
243
244         if key not in self.screenshot_cache:
245             success, data = self.executor.screenshot(test, viewport_size, dpi)
246
247             if not success:
248                 return False, data
249
250             screenshot = data
251             hash_value = hashlib.sha1(screenshot).hexdigest()
252
253             self.screenshot_cache[key] = (hash_value, None)
254
255             rv = (hash_value, screenshot)
256         else:
257             rv = self.screenshot_cache[key]
258
259         self.message.append("%s %s" % (test.url, rv[0]))
260         return True, rv
261
262     def is_pass(self, lhs_hash, rhs_hash, relation):
263         assert relation in ("==", "!=")
264         self.message.append("Testing %s %s %s" % (lhs_hash, relation, rhs_hash))
265         return ((relation == "==" and lhs_hash == rhs_hash) or
266                 (relation == "!=" and lhs_hash != rhs_hash))
267
268     def run_test(self, test):
269         viewport_size = test.viewport_size
270         dpi = test.dpi
271         self.message = []
272
273         # Depth-first search of reference tree, with the goal
274         # of reachings a leaf node with only pass results
275
276         stack = list(((test, item[0]), item[1]) for item in reversed(test.references))
277         while stack:
278             hashes = [None, None]
279             screenshots = [None, None]
280
281             nodes, relation = stack.pop()
282
283             for i, node in enumerate(nodes):
284                 success, data = self.get_hash(node, viewport_size, dpi)
285                 if success is False:
286                     return {"status": data[0], "message": data[1]}
287
288                 hashes[i], screenshots[i] = data
289
290             if self.is_pass(hashes[0], hashes[1], relation):
291                 if nodes[1].references:
292                     stack.extend(list(((nodes[1], item[0]), item[1]) for item in reversed(nodes[1].references)))
293                 else:
294                     # We passed
295                     return {"status":"PASS", "message": None}
296
297         # We failed, so construct a failure message
298
299         for i, (node, screenshot) in enumerate(zip(nodes, screenshots)):
300             if screenshot is None:
301                 success, screenshot = self.retake_screenshot(node, viewport_size, dpi)
302                 if success:
303                     screenshots[i] = screenshot
304
305         log_data = [{"url": nodes[0].url, "screenshot": screenshots[0]}, relation,
306                     {"url": nodes[1].url, "screenshot": screenshots[1]}]
307
308         return {"status": "FAIL",
309                 "message": "\n".join(self.message),
310                 "extra": {"reftest_screenshots": log_data}}
311
312     def retake_screenshot(self, node, viewport_size, dpi):
313         success, data = self.executor.screenshot(node, viewport_size, dpi)
314         if not success:
315             return False, data
316
317         key = (node.url, viewport_size, dpi)
318         hash_val, _ = self.screenshot_cache[key]
319         self.screenshot_cache[key] = hash_val, data
320         return True, data
321
322
323 class WdspecExecutor(TestExecutor):
324     convert_result = pytest_result_converter
325     protocol_cls = None
326
327     def __init__(self, browser, server_config, webdriver_binary,
328                  webdriver_args, timeout_multiplier=1, capabilities=None,
329                  debug_info=None, **kwargs):
330         self.do_delayed_imports()
331         TestExecutor.__init__(self, browser, server_config,
332                               timeout_multiplier=timeout_multiplier,
333                               debug_info=debug_info)
334         self.webdriver_binary = webdriver_binary
335         self.webdriver_args = webdriver_args
336         self.timeout_multiplier = timeout_multiplier
337         self.capabilities = capabilities
338         self.protocol = self.protocol_cls(self, browser)
339
340     def is_alive(self):
341         return self.protocol.is_alive
342
343     def on_environment_change(self, new_environment):
344         pass
345
346     def do_test(self, test):
347         timeout = test.timeout * self.timeout_multiplier + extra_timeout
348
349         success, data = WdspecRun(self.do_wdspec,
350                                   self.protocol.session_config,
351                                   test.abs_path,
352                                   timeout).run()
353
354         if success:
355             return self.convert_result(test, data)
356
357         return (test.result_cls(*data), [])
358
359     def do_wdspec(self, session_config, path, timeout):
360         harness_result = ("OK", None)
361         subtest_results = pytestrunner.run(path,
362                                            self.server_config,
363                                            session_config,
364                                            timeout=timeout)
365         return (harness_result, subtest_results)
366
367     def do_delayed_imports(self):
368         global pytestrunner
369         from . import pytestrunner
370
371
372 class Protocol(object):
373     def __init__(self, executor, browser):
374         self.executor = executor
375         self.browser = browser
376
377     @property
378     def logger(self):
379         return self.executor.logger
380
381     def setup(self, runner):
382         pass
383
384     def teardown(self):
385         pass
386
387     def wait(self):
388         pass
389
390
391 class WdspecRun(object):
392     def __init__(self, func, session, path, timeout):
393         self.func = func
394         self.result = (None, None)
395         self.session = session
396         self.path = path
397         self.timeout = timeout
398         self.result_flag = threading.Event()
399
400     def run(self):
401         """Runs function in a thread and interrupts it if it exceeds the
402         given timeout.  Returns (True, (Result, [SubtestResult ...])) in
403         case of success, or (False, (status, extra information)) in the
404         event of failure.
405         """
406
407         executor = threading.Thread(target=self._run)
408         executor.start()
409
410         flag = self.result_flag.wait(self.timeout)
411         if self.result[1] is None:
412             self.result = False, ("EXTERNAL-TIMEOUT", None)
413
414         return self.result
415
416     def _run(self):
417         try:
418             self.result = True, self.func(self.session, self.path, self.timeout)
419         except (socket.timeout, IOError):
420             self.result = False, ("CRASH", None)
421         except Exception as e:
422             message = getattr(e, "message")
423             if message:
424                 message += "\n"
425             message += traceback.format_exc(e)
426             self.result = False, ("ERROR", message)
427         finally:
428             self.result_flag.set()
429
430
431 class WebDriverProtocol(Protocol):
432     server_cls = None
433
434     def __init__(self, executor, browser):
435         Protocol.__init__(self, executor, browser)
436         self.webdriver_binary = executor.webdriver_binary
437         self.webdriver_args = executor.webdriver_args
438         self.capabilities = self.executor.capabilities
439         self.session_config = None
440         self.server = None
441
442     def setup(self, runner):
443         """Connect to browser via the HTTP server."""
444         try:
445             self.server = self.server_cls(
446                 self.logger,
447                 binary=self.webdriver_binary,
448                 args=self.webdriver_args)
449             self.server.start(block=False)
450             self.logger.info(
451                 "WebDriver HTTP server listening at %s" % self.server.url)
452             self.session_config = {"host": self.server.host,
453                                    "port": self.server.port,
454                                    "capabilities": self.capabilities}
455         except Exception:
456             self.logger.error(traceback.format_exc())
457             self.executor.runner.send_message("init_failed")
458         else:
459             self.executor.runner.send_message("init_succeeded")
460
461     def teardown(self):
462         if self.server is not None and self.server.is_alive:
463             self.server.stop()
464
465     @property
466     def is_alive(self):
467         """Test that the connection is still alive.
468
469         Because the remote communication happens over HTTP we need to
470         make an explicit request to the remote.  It is allowed for
471         WebDriver spec tests to not have a WebDriver session, since this
472         may be what is tested.
473
474         An HTTP request to an invalid path that results in a 404 is
475         proof enough to us that the server is alive and kicking.
476         """
477         conn = httplib.HTTPConnection(self.server.host, self.server.port)
478         conn.request("HEAD", self.server.base_path + "invalid")
479         res = conn.getresponse()
480         return res.status == 404