Unreviewed. Update W3C WebDriver imported tests.
[WebKit-https.git] / WebDriverTests / imported / w3c / tools / wptrunner / wptrunner / executors / executormarionette.py
1 import os
2 import socket
3 import threading
4 import traceback
5 import urlparse
6 import uuid
7
8 errors = None
9 marionette = None
10 pytestrunner = None
11
12 here = os.path.join(os.path.split(__file__)[0])
13
14 from .base import (ExecutorException,
15                    Protocol,
16                    RefTestExecutor,
17                    RefTestImplementation,
18                    TestExecutor,
19                    TestharnessExecutor,
20                    WdspecExecutor,
21                    WdspecRun,
22                    WebDriverProtocol,
23                    extra_timeout,
24                    testharness_result_converter,
25                    reftest_result_converter,
26                    strip_server)
27
28 from ..testrunner import Stop
29 from ..webdriver_server import GeckoDriverServer
30
31
32
33 def do_delayed_imports():
34     global errors, marionette
35
36     # Marionette client used to be called marionette, recently it changed
37     # to marionette_driver for unfathomable reasons
38     try:
39         import marionette
40         from marionette import errors
41     except ImportError:
42         from marionette_driver import marionette, errors
43
44
45 class MarionetteProtocol(Protocol):
46     def __init__(self, executor, browser, capabilities=None, timeout_multiplier=1):
47         do_delayed_imports()
48
49         Protocol.__init__(self, executor, browser)
50         self.marionette = None
51         self.marionette_port = browser.marionette_port
52         self.capabilities = capabilities
53         self.timeout_multiplier = timeout_multiplier
54         self.timeout = None
55         self.runner_handle = None
56
57     def setup(self, runner):
58         """Connect to browser via Marionette."""
59         Protocol.setup(self, runner)
60
61         self.logger.debug("Connecting to Marionette on port %i" % self.marionette_port)
62         startup_timeout = marionette.Marionette.DEFAULT_STARTUP_TIMEOUT * self.timeout_multiplier
63         self.marionette = marionette.Marionette(host='localhost',
64                                                 port=self.marionette_port,
65                                                 socket_timeout=None,
66                                                 startup_timeout=startup_timeout)
67         try:
68             self.logger.debug("Waiting for Marionette connection")
69             while True:
70                 try:
71                     self.marionette.raise_for_port()
72                     break
73                 except IOError:
74                     # When running in a debugger wait indefinitely for Firefox to start
75                     if self.executor.debug_info is None:
76                         raise
77
78             self.logger.debug("Starting Marionette session")
79             self.marionette.start_session()
80             self.logger.debug("Marionette session started")
81
82         except Exception as e:
83             self.logger.warning("Failed to start a Marionette session: %s" % e)
84             self.executor.runner.send_message("init_failed")
85
86         else:
87             try:
88                 self.after_connect()
89             except Exception:
90                 self.logger.warning("Post-connection steps failed")
91                 self.logger.error(traceback.format_exc())
92                 self.executor.runner.send_message("init_failed")
93             else:
94                 self.executor.runner.send_message("init_succeeded")
95
96     def teardown(self):
97         try:
98             self.marionette._request_in_app_shutdown()
99             self.marionette.delete_session(send_request=False, reset_session_id=True)
100         except Exception:
101             # This is typically because the session never started
102             pass
103         if self.marionette is not None:
104             del self.marionette
105
106     @property
107     def is_alive(self):
108         """Check if the Marionette connection is still active."""
109         try:
110             self.marionette.current_window_handle
111         except Exception:
112             return False
113         return True
114
115     def after_connect(self):
116         self.load_runner(self.executor.last_environment["protocol"])
117
118     def set_timeout(self, timeout):
119         """Set the Marionette script timeout.
120
121         :param timeout: Script timeout in seconds
122
123         """
124         self.marionette.timeout.script = timeout
125         self.timeout = timeout
126
127     def load_runner(self, protocol):
128         # Check if we previously had a test window open, and if we did make sure it's closed
129         self.marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
130         url = urlparse.urljoin(self.executor.server_url(protocol), "/testharness_runner.html")
131         self.logger.debug("Loading %s" % url)
132         self.runner_handle = self.marionette.current_window_handle
133         try:
134             self.marionette.navigate(url)
135         except Exception as e:
136             self.logger.critical(
137                 "Loading initial page %s failed. Ensure that the "
138                 "there are no other programs bound to this port and "
139                 "that your firewall rules or network setup does not "
140                 "prevent access.\e%s" % (url, traceback.format_exc(e)))
141         self.marionette.execute_script(
142             "document.title = '%s'" % threading.current_thread().name.replace("'", '"'))
143
144     def close_old_windows(self, protocol):
145         handles = self.marionette.window_handles
146         runner_handle = None
147         try:
148             handles.remove(self.runner_handle)
149             runner_handle = self.runner_handle
150         except ValueError:
151             # The runner window probably changed id but we can restore it
152             # This isn't supposed to happen, but marionette ids are not yet stable
153             # We assume that the first handle returned corresponds to the runner,
154             # but it hopefully doesn't matter too much if that assumption is
155             # wrong since we reload the runner in that tab anyway.
156             runner_handle = handles.pop(0)
157
158         for handle in handles:
159             try:
160                 self.marionette.switch_to_window(handle)
161                 self.marionette.close()
162             except errors.NoSuchWindowException:
163                 # We might have raced with the previous test to close this
164                 # window, skip it.
165                 pass
166
167         self.marionette.switch_to_window(runner_handle)
168         if runner_handle != self.runner_handle:
169             self.load_runner(protocol)
170
171     def wait(self):
172         try:
173             socket_timeout = self.marionette.client.socket_timeout
174         except AttributeError:
175             # This can happen if there was a crash
176             return
177         if socket_timeout:
178             self.marionette.timeout.script = socket_timeout / 2
179
180         self.marionette.switch_to_window(self.runner_handle)
181         while True:
182             try:
183                 self.marionette.execute_async_script("")
184             except errors.NoSuchWindowException:
185                 # The window closed
186                 break
187             except errors.ScriptTimeoutException:
188                 self.logger.debug("Script timed out")
189                 pass
190             except (socket.timeout, IOError):
191                 self.logger.debug("Socket closed")
192                 break
193             except Exception as e:
194                 self.logger.warning(traceback.format_exc(e))
195                 break
196
197     def on_environment_change(self, old_environment, new_environment):
198         #Unset all the old prefs
199         for name in old_environment.get("prefs", {}).iterkeys():
200             value = self.executor.original_pref_values[name]
201             if value is None:
202                 self.clear_user_pref(name)
203             else:
204                 self.set_pref(name, value)
205
206         for name, value in new_environment.get("prefs", {}).iteritems():
207             self.executor.original_pref_values[name] = self.get_pref(name)
208             self.set_pref(name, value)
209
210     def set_pref(self, name, value):
211         if value.lower() not in ("true", "false"):
212             try:
213                 int(value)
214             except ValueError:
215                 value = "'%s'" % value
216         else:
217             value = value.lower()
218
219         self.logger.info("Setting pref %s (%s)" % (name, value))
220
221         script = """
222             let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
223                                           .getService(Components.interfaces.nsIPrefBranch);
224             let pref = '%s';
225             let type = prefInterface.getPrefType(pref);
226             let value = %s;
227             switch(type) {
228                 case prefInterface.PREF_STRING:
229                     prefInterface.setCharPref(pref, value);
230                     break;
231                 case prefInterface.PREF_BOOL:
232                     prefInterface.setBoolPref(pref, value);
233                     break;
234                 case prefInterface.PREF_INT:
235                     prefInterface.setIntPref(pref, value);
236                     break;
237             }
238             """ % (name, value)
239         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
240             self.marionette.execute_script(script)
241
242     def clear_user_pref(self, name):
243         self.logger.info("Clearing pref %s" % (name))
244         script = """
245             let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
246                                           .getService(Components.interfaces.nsIPrefBranch);
247             let pref = '%s';
248             prefInterface.clearUserPref(pref);
249             """ % name
250         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
251             self.marionette.execute_script(script)
252
253     def get_pref(self, name):
254         script = """
255             let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
256                                           .getService(Components.interfaces.nsIPrefBranch);
257             let pref = '%s';
258             let type = prefInterface.getPrefType(pref);
259             switch(type) {
260                 case prefInterface.PREF_STRING:
261                     return prefInterface.getCharPref(pref);
262                 case prefInterface.PREF_BOOL:
263                     return prefInterface.getBoolPref(pref);
264                 case prefInterface.PREF_INT:
265                     return prefInterface.getIntPref(pref);
266                 case prefInterface.PREF_INVALID:
267                     return null;
268             }
269             """ % name
270         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
271             self.marionette.execute_script(script)
272
273     def clear_origin(self, url):
274         self.logger.info("Clearing origin %s" % (url))
275         script = """
276             let url = '%s';
277             let uri = Components.classes["@mozilla.org/network/io-service;1"]
278                                 .getService(Ci.nsIIOService)
279                                 .newURI(url);
280             let ssm = Components.classes["@mozilla.org/scriptsecuritymanager;1"]
281                                 .getService(Ci.nsIScriptSecurityManager);
282             let principal = ssm.createCodebasePrincipal(uri, {});
283             let qms = Components.classes["@mozilla.org/dom/quota-manager-service;1"]
284                                 .getService(Components.interfaces.nsIQuotaManagerService);
285             qms.clearStoragesForPrincipal(principal, "default", true);
286             """ % url
287         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
288             self.marionette.execute_script(script)
289
290
291 class ExecuteAsyncScriptRun(object):
292     def __init__(self, logger, func, protocol, url, timeout):
293         self.logger = logger
294         self.result = (None, None)
295         self.protocol = protocol
296         self.marionette = protocol.marionette
297         self.func = func
298         self.url = url
299         self.timeout = timeout
300         self.result_flag = threading.Event()
301
302     def run(self):
303         index = self.url.rfind("/storage/");
304         if index != -1:
305             # Clear storage
306             self.protocol.clear_origin(self.url)
307
308         timeout = self.timeout
309
310         try:
311             if timeout is not None:
312                 if timeout + extra_timeout != self.protocol.timeout:
313                     self.protocol.set_timeout(timeout + extra_timeout)
314             else:
315                 # We just want it to never time out, really, but marionette doesn't
316                 # make that possible. It also seems to time out immediately if the
317                 # timeout is set too high. This works at least.
318                 self.protocol.set_timeout(2**28 - 1)
319         except IOError:
320             self.logger.error("Lost marionette connection before starting test")
321             return Stop
322
323         executor = threading.Thread(target = self._run)
324         executor.start()
325
326         if timeout is not None:
327             wait_timeout = timeout + 2 * extra_timeout
328         else:
329             wait_timeout = None
330
331         flag = self.result_flag.wait(wait_timeout)
332
333         if self.result == (None, None):
334             self.logger.debug("Timed out waiting for a result")
335             self.result = False, ("EXTERNAL-TIMEOUT", None)
336         elif self.result[1] is None:
337             # We didn't get any data back from the test, so check if the
338             # browser is still responsive
339             if self.protocol.is_alive:
340                 self.result = False, ("ERROR", None)
341             else:
342                 self.result = False, ("CRASH", None)
343         return self.result
344
345     def _run(self):
346         try:
347             self.result = True, self.func(self.marionette, self.url, self.timeout)
348         except errors.ScriptTimeoutException:
349             self.logger.debug("Got a marionette timeout")
350             self.result = False, ("EXTERNAL-TIMEOUT", None)
351         except (socket.timeout, IOError):
352             # This can happen on a crash
353             # Also, should check after the test if the firefox process is still running
354             # and otherwise ignore any other result and set it to crash
355             self.result = False, ("CRASH", None)
356         except Exception as e:
357             message = getattr(e, "message", "")
358             if message:
359                 message += "\n"
360             message += traceback.format_exc(e)
361             self.result = False, ("ERROR", e)
362
363         finally:
364             self.result_flag.set()
365
366
367 class MarionetteTestharnessExecutor(TestharnessExecutor):
368     def __init__(self, browser, server_config, timeout_multiplier=1,
369                  close_after_done=True, debug_info=None, capabilities=None,
370                  **kwargs):
371         """Marionette-based executor for testharness.js tests"""
372         TestharnessExecutor.__init__(self, browser, server_config,
373                                      timeout_multiplier=timeout_multiplier,
374                                      debug_info=debug_info)
375
376         self.protocol = MarionetteProtocol(self, browser, capabilities, timeout_multiplier)
377         self.script = open(os.path.join(here, "testharness_marionette.js")).read()
378         self.close_after_done = close_after_done
379         self.window_id = str(uuid.uuid4())
380
381         self.original_pref_values = {}
382
383         if marionette is None:
384             do_delayed_imports()
385
386     def is_alive(self):
387         return self.protocol.is_alive
388
389     def on_environment_change(self, new_environment):
390         self.protocol.on_environment_change(self.last_environment, new_environment)
391
392         if new_environment["protocol"] != self.last_environment["protocol"]:
393             self.protocol.load_runner(new_environment["protocol"])
394
395     def do_test(self, test):
396         timeout = (test.timeout * self.timeout_multiplier if self.debug_info is None
397                    else None)
398
399         success, data = ExecuteAsyncScriptRun(self.logger,
400                                               self.do_testharness,
401                                               self.protocol,
402                                               self.test_url(test),
403                                               timeout).run()
404         if success:
405             return self.convert_result(test, data)
406
407         return (test.result_cls(*data), [])
408
409     def do_testharness(self, marionette, url, timeout):
410         if self.close_after_done:
411             marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
412             self.protocol.close_old_windows(self.protocol)
413
414         if timeout is not None:
415             timeout_ms = str(timeout * 1000)
416         else:
417             timeout_ms = "null"
418
419         script = self.script % {"abs_url": url,
420                                 "url": strip_server(url),
421                                 "window_id": self.window_id,
422                                 "timeout_multiplier": self.timeout_multiplier,
423                                 "timeout": timeout_ms,
424                                 "explicit_timeout": timeout is None}
425
426         rv = marionette.execute_async_script(script, new_sandbox=False)
427         return rv
428
429
430 class MarionetteRefTestExecutor(RefTestExecutor):
431     def __init__(self, browser, server_config, timeout_multiplier=1,
432                  screenshot_cache=None, close_after_done=True,
433                  debug_info=None, reftest_internal=False,
434                  reftest_screenshot="unexpected",
435                  group_metadata=None, capabilities=None, **kwargs):
436         """Marionette-based executor for reftests"""
437         RefTestExecutor.__init__(self,
438                                  browser,
439                                  server_config,
440                                  screenshot_cache=screenshot_cache,
441                                  timeout_multiplier=timeout_multiplier,
442                                  debug_info=debug_info)
443         self.protocol = MarionetteProtocol(self, browser, capabilities,
444                                            timeout_multiplier)
445         self.implementation = (InternalRefTestImplementation
446                                if reftest_internal
447                                else RefTestImplementation)(self)
448         self.implementation_kwargs = ({"screenshot": reftest_screenshot} if
449                                       reftest_internal else {})
450
451         self.close_after_done = close_after_done
452         self.has_window = False
453         self.original_pref_values = {}
454         self.group_metadata = group_metadata
455
456         with open(os.path.join(here, "reftest.js")) as f:
457             self.script = f.read()
458         with open(os.path.join(here, "reftest-wait_marionette.js")) as f:
459             self.wait_script = f.read()
460
461     def setup(self, runner):
462         super(self.__class__, self).setup(runner)
463         self.implementation.setup(**self.implementation_kwargs)
464
465     def teardown(self):
466         try:
467             self.implementation.teardown()
468             handle = self.protocol.marionette.window_handles[0]
469             self.protocol.marionette.switch_to_window(handle)
470             super(self.__class__, self).teardown()
471         except Exception as e:
472             # Ignore errors during teardown
473             self.logger.warning(traceback.format_exc(e))
474
475     def is_alive(self):
476         return self.protocol.is_alive
477
478     def on_environment_change(self, new_environment):
479         self.protocol.on_environment_change(self.last_environment, new_environment)
480
481     def do_test(self, test):
482         if not isinstance(self.implementation, InternalRefTestImplementation):
483             if self.close_after_done and self.has_window:
484                 self.protocol.marionette.close()
485                 self.protocol.marionette.switch_to_window(
486                     self.protocol.marionette.window_handles[-1])
487                 self.has_window = False
488
489             if not self.has_window:
490                 self.protocol.marionette.execute_script(self.script)
491                 self.protocol.marionette.switch_to_window(self.protocol.marionette.window_handles[-1])
492                 self.has_window = True
493
494         result = self.implementation.run_test(test)
495         return self.convert_result(test, result)
496
497     def screenshot(self, test, viewport_size, dpi):
498         # https://github.com/w3c/wptrunner/issues/166
499         assert viewport_size is None
500         assert dpi is None
501
502         timeout =  self.timeout_multiplier * test.timeout if self.debug_info is None else None
503
504         test_url = self.test_url(test)
505
506         return ExecuteAsyncScriptRun(self.logger,
507                                      self._screenshot,
508                                      self.protocol,
509                                      test_url,
510                                      timeout).run()
511
512     def _screenshot(self, marionette, url, timeout):
513         marionette.navigate(url)
514
515         marionette.execute_async_script(self.wait_script)
516
517         screenshot = marionette.screenshot(full=False)
518         # strip off the data:img/png, part of the url
519         if screenshot.startswith("data:image/png;base64,"):
520             screenshot = screenshot.split(",", 1)[1]
521
522         return screenshot
523
524
525 class InternalRefTestImplementation(object):
526     def __init__(self, executor):
527         self.timeout_multiplier = executor.timeout_multiplier
528         self.executor = executor
529
530     @property
531     def logger(self):
532         return self.executor.logger
533
534     def setup(self, screenshot="unexpected"):
535         data = {"screenshot": screenshot}
536         if self.executor.group_metadata is not None:
537             data["urlCount"] = {urlparse.urljoin(self.executor.server_url(key[0]), key[1]):value
538                                 for key, value in self.executor.group_metadata.get("url_count", {}).iteritems()
539                                 if value > 1}
540         self.executor.protocol.marionette.set_context(self.executor.protocol.marionette.CONTEXT_CHROME)
541         self.executor.protocol.marionette._send_message("reftest:setup", data)
542
543     def run_test(self, test):
544         viewport_size = test.viewport_size
545         dpi = test.dpi
546
547         references = self.get_references(test)
548         rv = self.executor.protocol.marionette._send_message("reftest:run",
549                                                              {"test": self.executor.test_url(test),
550                                                               "references": references,
551                                                               "expected": test.expected(),
552                                                               "timeout": test.timeout * 1000})["value"]
553         return rv
554
555     def get_references(self, node):
556         rv = []
557         for item, relation in node.references:
558             rv.append([self.executor.test_url(item), self.get_references(item), relation])
559         return rv
560
561     def teardown(self):
562         try:
563             self.executor.protocol.marionette._send_message("reftest:teardown", {})
564             self.executor.protocol.marionette.set_context(self.executor.protocol.marionette.CONTEXT_CONTENT)
565         except Exception as e:
566             # Ignore errors during teardown
567             self.logger.warning(traceback.traceback.format_exc(e))
568
569
570
571 class GeckoDriverProtocol(WebDriverProtocol):
572     server_cls = GeckoDriverServer
573
574
575 class MarionetteWdspecExecutor(WdspecExecutor):
576     protocol_cls = GeckoDriverProtocol