Unreviewed. Update W3C WebDriver imported tests.
[WebKit-https.git] / WebDriverTests / imported / w3c / tools / wptrunner / wptrunner / executors / executormarionette.py
1 import os
2 import socket
3 import threading
4 import traceback
5 import urlparse
6 import uuid
7
8 errors = None
9 marionette = None
10 pytestrunner = None
11
12 here = os.path.join(os.path.split(__file__)[0])
13
14 from .base import (ExecutorException,
15                    Protocol,
16                    RefTestExecutor,
17                    RefTestImplementation,
18                    TestExecutor,
19                    TestharnessExecutor,
20                    WdspecExecutor,
21                    WdspecRun,
22                    WebDriverProtocol,
23                    extra_timeout,
24                    testharness_result_converter,
25                    reftest_result_converter,
26                    strip_server)
27
28 from ..testrunner import Stop
29 from ..webdriver_server import GeckoDriverServer
30
31
32
33 def do_delayed_imports():
34     global errors, marionette
35
36     # Marionette client used to be called marionette, recently it changed
37     # to marionette_driver for unfathomable reasons
38     try:
39         import marionette
40         from marionette import errors
41     except ImportError:
42         from marionette_driver import marionette, errors
43
44
45 class MarionetteProtocol(Protocol):
46     def __init__(self, executor, browser, capabilities=None, timeout_multiplier=1):
47         do_delayed_imports()
48
49         Protocol.__init__(self, executor, browser)
50         self.marionette = None
51         self.marionette_port = browser.marionette_port
52         self.capabilities = capabilities
53         self.timeout_multiplier = timeout_multiplier
54         self.timeout = None
55         self.runner_handle = None
56
57     def setup(self, runner):
58         """Connect to browser via Marionette."""
59         Protocol.setup(self, runner)
60
61         self.logger.debug("Connecting to Marionette on port %i" % self.marionette_port)
62         startup_timeout = marionette.Marionette.DEFAULT_STARTUP_TIMEOUT * self.timeout_multiplier
63         self.marionette = marionette.Marionette(host='localhost',
64                                                 port=self.marionette_port,
65                                                 socket_timeout=None,
66                                                 startup_timeout=startup_timeout)
67         try:
68             self.logger.debug("Waiting for Marionette connection")
69             while True:
70                 try:
71                     self.marionette.raise_for_port()
72                     break
73                 except IOError:
74                     # When running in a debugger wait indefinitely for Firefox to start
75                     if self.executor.debug_info is None:
76                         raise
77
78             self.logger.debug("Starting Marionette session")
79             self.marionette.start_session()
80             self.logger.debug("Marionette session started")
81
82         except Exception as e:
83             self.logger.warning("Failed to start a Marionette session: %s" % e)
84             self.executor.runner.send_message("init_failed")
85
86         else:
87             try:
88                 self.after_connect()
89             except Exception:
90                 self.logger.warning("Post-connection steps failed")
91                 self.logger.error(traceback.format_exc())
92                 self.executor.runner.send_message("init_failed")
93             else:
94                 self.executor.runner.send_message("init_succeeded")
95
96     def teardown(self):
97         try:
98             self.marionette._request_in_app_shutdown()
99             self.marionette.delete_session(send_request=False)
100         except Exception:
101             # This is typically because the session never started
102             pass
103         if self.marionette is not None:
104             del self.marionette
105
106     @property
107     def is_alive(self):
108         """Check if the Marionette connection is still active."""
109         try:
110             self.marionette.current_window_handle
111         except Exception:
112             return False
113         return True
114
115     def after_connect(self):
116         self.load_runner(self.executor.last_environment["protocol"])
117
118     def set_timeout(self, timeout):
119         """Set the Marionette script timeout.
120
121         :param timeout: Script timeout in seconds
122
123         """
124         self.marionette.timeout.script = timeout
125         self.timeout = timeout
126
127     def load_runner(self, protocol):
128         # Check if we previously had a test window open, and if we did make sure it's closed
129         self.marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
130         url = urlparse.urljoin(self.executor.server_url(protocol), "/testharness_runner.html")
131         self.logger.debug("Loading %s" % url)
132         self.runner_handle = self.marionette.current_window_handle
133         try:
134             self.dismiss_alert(lambda: self.marionette.navigate(url))
135         except Exception as e:
136             self.logger.critical(
137                 "Loading initial page %s failed. Ensure that the "
138                 "there are no other programs bound to this port and "
139                 "that your firewall rules or network setup does not "
140                 "prevent access.\e%s" % (url, traceback.format_exc(e)))
141             raise
142         self.marionette.execute_script(
143             "document.title = '%s'" % threading.current_thread().name.replace("'", '"'))
144
145     def close_old_windows(self, protocol):
146         handles = self.marionette.window_handles
147         runner_handle = None
148         try:
149             handles.remove(self.runner_handle)
150             runner_handle = self.runner_handle
151         except ValueError:
152             # The runner window probably changed id but we can restore it
153             # This isn't supposed to happen, but marionette ids are not yet stable
154             # We assume that the first handle returned corresponds to the runner,
155             # but it hopefully doesn't matter too much if that assumption is
156             # wrong since we reload the runner in that tab anyway.
157             runner_handle = handles.pop(0)
158
159         for handle in handles:
160             try:
161                 self.dismiss_alert(lambda: self.marionette.switch_to_window(handle))
162                 self.marionette.switch_to_window(handle)
163                 self.marionette.close()
164             except errors.NoSuchWindowException:
165                 # We might have raced with the previous test to close this
166                 # window, skip it.
167                 pass
168
169         self.marionette.switch_to_window(runner_handle)
170         if runner_handle != self.runner_handle:
171             self.load_runner(protocol)
172
173     def dismiss_alert(self, f):
174         while True:
175             try:
176                 f()
177             except errors.UnexpectedAlertOpen:
178                 alert = self.marionette.switch_to_alert()
179                 try:
180                     alert.dismiss()
181                 except errors.NoAlertPresentException:
182                     pass
183             else:
184                 break
185
186     def wait(self):
187         try:
188             socket_timeout = self.marionette.client.socket_timeout
189         except AttributeError:
190             # This can happen if there was a crash
191             return
192         if socket_timeout:
193             try:
194                 self.marionette.timeout.script = socket_timeout / 2
195             except (socket.error, IOError):
196                 self.logger.debug("Socket closed")
197                 return
198
199         self.marionette.switch_to_window(self.runner_handle)
200         while True:
201             try:
202                 self.marionette.execute_async_script("")
203             except errors.NoSuchWindowException:
204                 # The window closed
205                 break
206             except errors.ScriptTimeoutException:
207                 self.logger.debug("Script timed out")
208                 pass
209             except (socket.timeout, IOError):
210                 self.logger.debug("Socket closed")
211                 break
212             except Exception as e:
213                 self.logger.warning(traceback.format_exc(e))
214                 break
215
216     def on_environment_change(self, old_environment, new_environment):
217         #Unset all the old prefs
218         for name in old_environment.get("prefs", {}).iterkeys():
219             value = self.executor.original_pref_values[name]
220             if value is None:
221                 self.clear_user_pref(name)
222             else:
223                 self.set_pref(name, value)
224
225         for name, value in new_environment.get("prefs", {}).iteritems():
226             self.executor.original_pref_values[name] = self.get_pref(name)
227             self.set_pref(name, value)
228
229     def set_pref(self, name, value):
230         if value.lower() not in ("true", "false"):
231             try:
232                 int(value)
233             except ValueError:
234                 value = "'%s'" % value
235         else:
236             value = value.lower()
237
238         self.logger.info("Setting pref %s (%s)" % (name, value))
239
240         script = """
241             let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
242                                           .getService(Components.interfaces.nsIPrefBranch);
243             let pref = '%s';
244             let type = prefInterface.getPrefType(pref);
245             let value = %s;
246             switch(type) {
247                 case prefInterface.PREF_STRING:
248                     prefInterface.setCharPref(pref, value);
249                     break;
250                 case prefInterface.PREF_BOOL:
251                     prefInterface.setBoolPref(pref, value);
252                     break;
253                 case prefInterface.PREF_INT:
254                     prefInterface.setIntPref(pref, value);
255                     break;
256             }
257             """ % (name, value)
258         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
259             self.marionette.execute_script(script)
260
261     def clear_user_pref(self, name):
262         self.logger.info("Clearing pref %s" % (name))
263         script = """
264             let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
265                                           .getService(Components.interfaces.nsIPrefBranch);
266             let pref = '%s';
267             prefInterface.clearUserPref(pref);
268             """ % name
269         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
270             self.marionette.execute_script(script)
271
272     def get_pref(self, name):
273         script = """
274             let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
275                                           .getService(Components.interfaces.nsIPrefBranch);
276             let pref = '%s';
277             let type = prefInterface.getPrefType(pref);
278             switch(type) {
279                 case prefInterface.PREF_STRING:
280                     return prefInterface.getCharPref(pref);
281                 case prefInterface.PREF_BOOL:
282                     return prefInterface.getBoolPref(pref);
283                 case prefInterface.PREF_INT:
284                     return prefInterface.getIntPref(pref);
285                 case prefInterface.PREF_INVALID:
286                     return null;
287             }
288             """ % name
289         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
290             self.marionette.execute_script(script)
291
292     def clear_origin(self, url):
293         self.logger.info("Clearing origin %s" % (url))
294         script = """
295             let url = '%s';
296             let uri = Components.classes["@mozilla.org/network/io-service;1"]
297                                 .getService(Ci.nsIIOService)
298                                 .newURI(url);
299             let ssm = Components.classes["@mozilla.org/scriptsecuritymanager;1"]
300                                 .getService(Ci.nsIScriptSecurityManager);
301             let principal = ssm.createCodebasePrincipal(uri, {});
302             let qms = Components.classes["@mozilla.org/dom/quota-manager-service;1"]
303                                 .getService(Components.interfaces.nsIQuotaManagerService);
304             qms.clearStoragesForPrincipal(principal, "default", true);
305             """ % url
306         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
307             self.marionette.execute_script(script)
308
309
310 class ExecuteAsyncScriptRun(object):
311     def __init__(self, logger, func, protocol, url, timeout):
312         self.logger = logger
313         self.result = (None, None)
314         self.protocol = protocol
315         self.marionette = protocol.marionette
316         self.func = func
317         self.url = url
318         self.timeout = timeout
319         self.result_flag = threading.Event()
320
321     def run(self):
322         index = self.url.rfind("/storage/")
323         if index != -1:
324             # Clear storage
325             self.protocol.clear_origin(self.url)
326
327         timeout = self.timeout
328
329         try:
330             if timeout is not None:
331                 if timeout + extra_timeout != self.protocol.timeout:
332                     self.protocol.set_timeout(timeout + extra_timeout)
333             else:
334                 # We just want it to never time out, really, but marionette doesn't
335                 # make that possible. It also seems to time out immediately if the
336                 # timeout is set too high. This works at least.
337                 self.protocol.set_timeout(2**28 - 1)
338         except IOError:
339             self.logger.error("Lost marionette connection before starting test")
340             return Stop
341
342         executor = threading.Thread(target = self._run)
343         executor.start()
344
345         if timeout is not None:
346             wait_timeout = timeout + 2 * extra_timeout
347         else:
348             wait_timeout = None
349
350         flag = self.result_flag.wait(wait_timeout)
351
352         if self.result == (None, None):
353             self.logger.debug("Timed out waiting for a result")
354             self.result = False, ("EXTERNAL-TIMEOUT", None)
355         elif self.result[1] is None:
356             # We didn't get any data back from the test, so check if the
357             # browser is still responsive
358             if self.protocol.is_alive:
359                 self.result = False, ("ERROR", None)
360             else:
361                 self.result = False, ("CRASH", None)
362         return self.result
363
364     def _run(self):
365         try:
366             self.result = True, self.func(self.marionette, self.url, self.timeout)
367         except errors.ScriptTimeoutException:
368             self.logger.debug("Got a marionette timeout")
369             self.result = False, ("EXTERNAL-TIMEOUT", None)
370         except (socket.timeout, IOError):
371             # This can happen on a crash
372             # Also, should check after the test if the firefox process is still running
373             # and otherwise ignore any other result and set it to crash
374             self.result = False, ("CRASH", None)
375         except Exception as e:
376             message = getattr(e, "message", "")
377             if message:
378                 message += "\n"
379             message += traceback.format_exc(e)
380             self.result = False, ("ERROR", e)
381
382         finally:
383             self.result_flag.set()
384
385
386 class MarionetteTestharnessExecutor(TestharnessExecutor):
387     def __init__(self, browser, server_config, timeout_multiplier=1,
388                  close_after_done=True, debug_info=None, capabilities=None,
389                  **kwargs):
390         """Marionette-based executor for testharness.js tests"""
391         TestharnessExecutor.__init__(self, browser, server_config,
392                                      timeout_multiplier=timeout_multiplier,
393                                      debug_info=debug_info)
394
395         self.protocol = MarionetteProtocol(self, browser, capabilities, timeout_multiplier)
396         self.script = open(os.path.join(here, "testharness_marionette.js")).read()
397         self.close_after_done = close_after_done
398         self.window_id = str(uuid.uuid4())
399
400         self.original_pref_values = {}
401
402         if marionette is None:
403             do_delayed_imports()
404
405     def is_alive(self):
406         return self.protocol.is_alive
407
408     def on_environment_change(self, new_environment):
409         self.protocol.on_environment_change(self.last_environment, new_environment)
410
411         if new_environment["protocol"] != self.last_environment["protocol"]:
412             self.protocol.load_runner(new_environment["protocol"])
413
414     def do_test(self, test):
415         timeout = (test.timeout * self.timeout_multiplier if self.debug_info is None
416                    else None)
417
418         success, data = ExecuteAsyncScriptRun(self.logger,
419                                               self.do_testharness,
420                                               self.protocol,
421                                               self.test_url(test),
422                                               timeout).run()
423         if success:
424             return self.convert_result(test, data)
425
426         return (test.result_cls(*data), [])
427
428     def do_testharness(self, marionette, url, timeout):
429         if self.close_after_done:
430             marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
431             self.protocol.close_old_windows(self.protocol)
432
433         if timeout is not None:
434             timeout_ms = str(timeout * 1000)
435         else:
436             timeout_ms = "null"
437
438         script = self.script % {"abs_url": url,
439                                 "url": strip_server(url),
440                                 "window_id": self.window_id,
441                                 "timeout_multiplier": self.timeout_multiplier,
442                                 "timeout": timeout_ms,
443                                 "explicit_timeout": timeout is None}
444
445         rv = marionette.execute_async_script(script, new_sandbox=False)
446         return rv
447
448
449 class MarionetteRefTestExecutor(RefTestExecutor):
450     def __init__(self, browser, server_config, timeout_multiplier=1,
451                  screenshot_cache=None, close_after_done=True,
452                  debug_info=None, reftest_internal=False,
453                  reftest_screenshot="unexpected",
454                  group_metadata=None, capabilities=None, **kwargs):
455         """Marionette-based executor for reftests"""
456         RefTestExecutor.__init__(self,
457                                  browser,
458                                  server_config,
459                                  screenshot_cache=screenshot_cache,
460                                  timeout_multiplier=timeout_multiplier,
461                                  debug_info=debug_info)
462         self.protocol = MarionetteProtocol(self, browser, capabilities,
463                                            timeout_multiplier)
464         self.implementation = (InternalRefTestImplementation
465                                if reftest_internal
466                                else RefTestImplementation)(self)
467         self.implementation_kwargs = ({"screenshot": reftest_screenshot} if
468                                       reftest_internal else {})
469
470         self.close_after_done = close_after_done
471         self.has_window = False
472         self.original_pref_values = {}
473         self.group_metadata = group_metadata
474
475         with open(os.path.join(here, "reftest.js")) as f:
476             self.script = f.read()
477         with open(os.path.join(here, "reftest-wait_marionette.js")) as f:
478             self.wait_script = f.read()
479
480     def setup(self, runner):
481         super(self.__class__, self).setup(runner)
482         self.implementation.setup(**self.implementation_kwargs)
483
484     def teardown(self):
485         try:
486             self.implementation.teardown()
487             handle = self.protocol.marionette.window_handles[0]
488             self.protocol.marionette.switch_to_window(handle)
489             super(self.__class__, self).teardown()
490         except Exception as e:
491             # Ignore errors during teardown
492             self.logger.warning(traceback.format_exc(e))
493
494     def is_alive(self):
495         return self.protocol.is_alive
496
497     def on_environment_change(self, new_environment):
498         self.protocol.on_environment_change(self.last_environment, new_environment)
499
500     def do_test(self, test):
501         if not isinstance(self.implementation, InternalRefTestImplementation):
502             if self.close_after_done and self.has_window:
503                 self.protocol.marionette.close()
504                 self.protocol.marionette.switch_to_window(
505                     self.protocol.marionette.window_handles[-1])
506                 self.has_window = False
507
508             if not self.has_window:
509                 self.protocol.marionette.execute_script(self.script)
510                 self.protocol.marionette.switch_to_window(self.protocol.marionette.window_handles[-1])
511                 self.has_window = True
512
513         result = self.implementation.run_test(test)
514         return self.convert_result(test, result)
515
516     def screenshot(self, test, viewport_size, dpi):
517         # https://github.com/w3c/wptrunner/issues/166
518         assert viewport_size is None
519         assert dpi is None
520
521         timeout = self.timeout_multiplier * test.timeout if self.debug_info is None else None
522
523         test_url = self.test_url(test)
524
525         return ExecuteAsyncScriptRun(self.logger,
526                                      self._screenshot,
527                                      self.protocol,
528                                      test_url,
529                                      timeout).run()
530
531     def _screenshot(self, marionette, url, timeout):
532         marionette.navigate(url)
533
534         marionette.execute_async_script(self.wait_script)
535
536         screenshot = marionette.screenshot(full=False)
537         # strip off the data:img/png, part of the url
538         if screenshot.startswith("data:image/png;base64,"):
539             screenshot = screenshot.split(",", 1)[1]
540
541         return screenshot
542
543
544 class InternalRefTestImplementation(object):
545     def __init__(self, executor):
546         self.timeout_multiplier = executor.timeout_multiplier
547         self.executor = executor
548
549     @property
550     def logger(self):
551         return self.executor.logger
552
553     def setup(self, screenshot="unexpected"):
554         data = {"screenshot": screenshot}
555         if self.executor.group_metadata is not None:
556             data["urlCount"] = {urlparse.urljoin(self.executor.server_url(key[0]), key[1]):value
557                                 for key, value in self.executor.group_metadata.get("url_count", {}).iteritems()
558                                 if value > 1}
559         self.executor.protocol.marionette.set_context(self.executor.protocol.marionette.CONTEXT_CHROME)
560         self.executor.protocol.marionette._send_message("reftest:setup", data)
561
562     def run_test(self, test):
563         viewport_size = test.viewport_size
564         dpi = test.dpi
565
566         references = self.get_references(test)
567         rv = self.executor.protocol.marionette._send_message("reftest:run",
568                                                              {"test": self.executor.test_url(test),
569                                                               "references": references,
570                                                               "expected": test.expected(),
571                                                               "timeout": test.timeout * 1000})["value"]
572         return rv
573
574     def get_references(self, node):
575         rv = []
576         for item, relation in node.references:
577             rv.append([self.executor.test_url(item), self.get_references(item), relation])
578         return rv
579
580     def teardown(self):
581         try:
582             self.executor.protocol.marionette._send_message("reftest:teardown", {})
583             self.executor.protocol.marionette.set_context(self.executor.protocol.marionette.CONTEXT_CONTENT)
584         except Exception as e:
585             # Ignore errors during teardown
586             self.logger.warning(traceback.format_exc(e))
587
588
589
590 class GeckoDriverProtocol(WebDriverProtocol):
591     server_cls = GeckoDriverServer
592
593
594 class MarionetteWdspecExecutor(WdspecExecutor):
595     protocol_cls = GeckoDriverProtocol