Unreviewed. Update W3C WebDriver imported tests.
[WebKit-https.git] / WebDriverTests / imported / w3c / tools / wptrunner / wptrunner / executors / executormarionette.py
1 import os
2 import socket
3 import threading
4 import traceback
5 import urlparse
6 import uuid
7
8 errors = None
9 marionette = None
10 pytestrunner = None
11
12 here = os.path.join(os.path.split(__file__)[0])
13
14 from .base import (ExecutorException,
15                    Protocol,
16                    RefTestExecutor,
17                    RefTestImplementation,
18                    TestExecutor,
19                    TestharnessExecutor,
20                    WdspecExecutor,
21                    WdspecRun,
22                    WebDriverProtocol,
23                    extra_timeout,
24                    testharness_result_converter,
25                    reftest_result_converter,
26                    strip_server)
27
28 from ..testrunner import Stop
29 from ..webdriver_server import GeckoDriverServer
30
31
32
33 def do_delayed_imports():
34     global errors, marionette
35
36     # Marionette client used to be called marionette, recently it changed
37     # to marionette_driver for unfathomable reasons
38     try:
39         import marionette
40         from marionette import errors
41     except ImportError:
42         from marionette_driver import marionette, errors
43
44
45 class MarionetteProtocol(Protocol):
46     def __init__(self, executor, browser, capabilities=None, timeout_multiplier=1):
47         do_delayed_imports()
48
49         Protocol.__init__(self, executor, browser)
50         self.marionette = None
51         self.marionette_port = browser.marionette_port
52         self.capabilities = capabilities
53         self.timeout_multiplier = timeout_multiplier
54         self.timeout = None
55         self.runner_handle = None
56
57     def setup(self, runner):
58         """Connect to browser via Marionette."""
59         Protocol.setup(self, runner)
60
61         self.logger.debug("Connecting to Marionette on port %i" % self.marionette_port)
62         startup_timeout = marionette.Marionette.DEFAULT_STARTUP_TIMEOUT * self.timeout_multiplier
63         self.marionette = marionette.Marionette(host='localhost',
64                                                 port=self.marionette_port,
65                                                 socket_timeout=None,
66                                                 startup_timeout=startup_timeout)
67         try:
68             self.logger.debug("Waiting for Marionette connection")
69             while True:
70                 try:
71                     self.marionette.raise_for_port()
72                     break
73                 except IOError:
74                     # When running in a debugger wait indefinitely for Firefox to start
75                     if self.executor.debug_info is None:
76                         raise
77
78             self.logger.debug("Starting Marionette session")
79             self.marionette.start_session()
80             self.logger.debug("Marionette session started")
81
82         except Exception as e:
83             self.logger.warning("Failed to start a Marionette session: %s" % e)
84             self.executor.runner.send_message("init_failed")
85
86         else:
87             try:
88                 self.after_connect()
89             except Exception:
90                 self.logger.warning("Post-connection steps failed")
91                 self.logger.error(traceback.format_exc())
92                 self.executor.runner.send_message("init_failed")
93             else:
94                 self.executor.runner.send_message("init_succeeded")
95
96     def teardown(self):
97         try:
98             self.marionette._request_in_app_shutdown()
99             self.marionette.delete_session(send_request=False)
100         except Exception:
101             # This is typically because the session never started
102             pass
103         if self.marionette is not None:
104             del self.marionette
105
106     @property
107     def is_alive(self):
108         """Check if the Marionette connection is still active."""
109         try:
110             self.marionette.current_window_handle
111         except Exception:
112             return False
113         return True
114
115     def after_connect(self):
116         self.load_runner(self.executor.last_environment["protocol"])
117
118     def set_timeout(self, timeout):
119         """Set the Marionette script timeout.
120
121         :param timeout: Script timeout in seconds
122
123         """
124         self.marionette.timeout.script = timeout
125         self.timeout = timeout
126
127     def load_runner(self, protocol):
128         # Check if we previously had a test window open, and if we did make sure it's closed
129         self.marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
130         url = urlparse.urljoin(self.executor.server_url(protocol), "/testharness_runner.html")
131         self.logger.debug("Loading %s" % url)
132         self.runner_handle = self.marionette.current_window_handle
133         try:
134             self.marionette.navigate(url)
135         except Exception as e:
136             self.logger.critical(
137                 "Loading initial page %s failed. Ensure that the "
138                 "there are no other programs bound to this port and "
139                 "that your firewall rules or network setup does not "
140                 "prevent access.\e%s" % (url, traceback.format_exc(e)))
141         self.marionette.execute_script(
142             "document.title = '%s'" % threading.current_thread().name.replace("'", '"'))
143
144     def close_old_windows(self, protocol):
145         handles = self.marionette.window_handles
146         runner_handle = None
147         try:
148             handles.remove(self.runner_handle)
149             runner_handle = self.runner_handle
150         except ValueError:
151             # The runner window probably changed id but we can restore it
152             # This isn't supposed to happen, but marionette ids are not yet stable
153             # We assume that the first handle returned corresponds to the runner,
154             # but it hopefully doesn't matter too much if that assumption is
155             # wrong since we reload the runner in that tab anyway.
156             runner_handle = handles.pop(0)
157
158         for handle in handles:
159             try:
160                 self.marionette.switch_to_window(handle)
161                 self.marionette.close()
162             except errors.NoSuchWindowException:
163                 # We might have raced with the previous test to close this
164                 # window, skip it.
165                 pass
166
167         self.marionette.switch_to_window(runner_handle)
168         if runner_handle != self.runner_handle:
169             self.load_runner(protocol)
170
171     def wait(self):
172         try:
173             socket_timeout = self.marionette.client.socket_timeout
174         except AttributeError:
175             # This can happen if there was a crash
176             return
177         if socket_timeout:
178             try:
179                 self.marionette.timeout.script = socket_timeout / 2
180             except (socket.error, IOError):
181                 self.logger.debug("Socket closed")
182                 return
183
184         self.marionette.switch_to_window(self.runner_handle)
185         while True:
186             try:
187                 self.marionette.execute_async_script("")
188             except errors.NoSuchWindowException:
189                 # The window closed
190                 break
191             except errors.ScriptTimeoutException:
192                 self.logger.debug("Script timed out")
193                 pass
194             except (socket.timeout, IOError):
195                 self.logger.debug("Socket closed")
196                 break
197             except Exception as e:
198                 self.logger.warning(traceback.format_exc(e))
199                 break
200
201     def on_environment_change(self, old_environment, new_environment):
202         #Unset all the old prefs
203         for name in old_environment.get("prefs", {}).iterkeys():
204             value = self.executor.original_pref_values[name]
205             if value is None:
206                 self.clear_user_pref(name)
207             else:
208                 self.set_pref(name, value)
209
210         for name, value in new_environment.get("prefs", {}).iteritems():
211             self.executor.original_pref_values[name] = self.get_pref(name)
212             self.set_pref(name, value)
213
214     def set_pref(self, name, value):
215         if value.lower() not in ("true", "false"):
216             try:
217                 int(value)
218             except ValueError:
219                 value = "'%s'" % value
220         else:
221             value = value.lower()
222
223         self.logger.info("Setting pref %s (%s)" % (name, value))
224
225         script = """
226             let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
227                                           .getService(Components.interfaces.nsIPrefBranch);
228             let pref = '%s';
229             let type = prefInterface.getPrefType(pref);
230             let value = %s;
231             switch(type) {
232                 case prefInterface.PREF_STRING:
233                     prefInterface.setCharPref(pref, value);
234                     break;
235                 case prefInterface.PREF_BOOL:
236                     prefInterface.setBoolPref(pref, value);
237                     break;
238                 case prefInterface.PREF_INT:
239                     prefInterface.setIntPref(pref, value);
240                     break;
241             }
242             """ % (name, value)
243         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
244             self.marionette.execute_script(script)
245
246     def clear_user_pref(self, name):
247         self.logger.info("Clearing pref %s" % (name))
248         script = """
249             let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
250                                           .getService(Components.interfaces.nsIPrefBranch);
251             let pref = '%s';
252             prefInterface.clearUserPref(pref);
253             """ % name
254         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
255             self.marionette.execute_script(script)
256
257     def get_pref(self, name):
258         script = """
259             let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
260                                           .getService(Components.interfaces.nsIPrefBranch);
261             let pref = '%s';
262             let type = prefInterface.getPrefType(pref);
263             switch(type) {
264                 case prefInterface.PREF_STRING:
265                     return prefInterface.getCharPref(pref);
266                 case prefInterface.PREF_BOOL:
267                     return prefInterface.getBoolPref(pref);
268                 case prefInterface.PREF_INT:
269                     return prefInterface.getIntPref(pref);
270                 case prefInterface.PREF_INVALID:
271                     return null;
272             }
273             """ % name
274         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
275             self.marionette.execute_script(script)
276
277     def clear_origin(self, url):
278         self.logger.info("Clearing origin %s" % (url))
279         script = """
280             let url = '%s';
281             let uri = Components.classes["@mozilla.org/network/io-service;1"]
282                                 .getService(Ci.nsIIOService)
283                                 .newURI(url);
284             let ssm = Components.classes["@mozilla.org/scriptsecuritymanager;1"]
285                                 .getService(Ci.nsIScriptSecurityManager);
286             let principal = ssm.createCodebasePrincipal(uri, {});
287             let qms = Components.classes["@mozilla.org/dom/quota-manager-service;1"]
288                                 .getService(Components.interfaces.nsIQuotaManagerService);
289             qms.clearStoragesForPrincipal(principal, "default", true);
290             """ % url
291         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
292             self.marionette.execute_script(script)
293
294
295 class ExecuteAsyncScriptRun(object):
296     def __init__(self, logger, func, protocol, url, timeout):
297         self.logger = logger
298         self.result = (None, None)
299         self.protocol = protocol
300         self.marionette = protocol.marionette
301         self.func = func
302         self.url = url
303         self.timeout = timeout
304         self.result_flag = threading.Event()
305
306     def run(self):
307         index = self.url.rfind("/storage/")
308         if index != -1:
309             # Clear storage
310             self.protocol.clear_origin(self.url)
311
312         timeout = self.timeout
313
314         try:
315             if timeout is not None:
316                 if timeout + extra_timeout != self.protocol.timeout:
317                     self.protocol.set_timeout(timeout + extra_timeout)
318             else:
319                 # We just want it to never time out, really, but marionette doesn't
320                 # make that possible. It also seems to time out immediately if the
321                 # timeout is set too high. This works at least.
322                 self.protocol.set_timeout(2**28 - 1)
323         except IOError:
324             self.logger.error("Lost marionette connection before starting test")
325             return Stop
326
327         executor = threading.Thread(target = self._run)
328         executor.start()
329
330         if timeout is not None:
331             wait_timeout = timeout + 2 * extra_timeout
332         else:
333             wait_timeout = None
334
335         flag = self.result_flag.wait(wait_timeout)
336
337         if self.result == (None, None):
338             self.logger.debug("Timed out waiting for a result")
339             self.result = False, ("EXTERNAL-TIMEOUT", None)
340         elif self.result[1] is None:
341             # We didn't get any data back from the test, so check if the
342             # browser is still responsive
343             if self.protocol.is_alive:
344                 self.result = False, ("ERROR", None)
345             else:
346                 self.result = False, ("CRASH", None)
347         return self.result
348
349     def _run(self):
350         try:
351             self.result = True, self.func(self.marionette, self.url, self.timeout)
352         except errors.ScriptTimeoutException:
353             self.logger.debug("Got a marionette timeout")
354             self.result = False, ("EXTERNAL-TIMEOUT", None)
355         except (socket.timeout, IOError):
356             # This can happen on a crash
357             # Also, should check after the test if the firefox process is still running
358             # and otherwise ignore any other result and set it to crash
359             self.result = False, ("CRASH", None)
360         except Exception as e:
361             message = getattr(e, "message", "")
362             if message:
363                 message += "\n"
364             message += traceback.format_exc(e)
365             self.result = False, ("ERROR", e)
366
367         finally:
368             self.result_flag.set()
369
370
371 class MarionetteTestharnessExecutor(TestharnessExecutor):
372     def __init__(self, browser, server_config, timeout_multiplier=1,
373                  close_after_done=True, debug_info=None, capabilities=None,
374                  **kwargs):
375         """Marionette-based executor for testharness.js tests"""
376         TestharnessExecutor.__init__(self, browser, server_config,
377                                      timeout_multiplier=timeout_multiplier,
378                                      debug_info=debug_info)
379
380         self.protocol = MarionetteProtocol(self, browser, capabilities, timeout_multiplier)
381         self.script = open(os.path.join(here, "testharness_marionette.js")).read()
382         self.close_after_done = close_after_done
383         self.window_id = str(uuid.uuid4())
384
385         self.original_pref_values = {}
386
387         if marionette is None:
388             do_delayed_imports()
389
390     def is_alive(self):
391         return self.protocol.is_alive
392
393     def on_environment_change(self, new_environment):
394         self.protocol.on_environment_change(self.last_environment, new_environment)
395
396         if new_environment["protocol"] != self.last_environment["protocol"]:
397             self.protocol.load_runner(new_environment["protocol"])
398
399     def do_test(self, test):
400         timeout = (test.timeout * self.timeout_multiplier if self.debug_info is None
401                    else None)
402
403         success, data = ExecuteAsyncScriptRun(self.logger,
404                                               self.do_testharness,
405                                               self.protocol,
406                                               self.test_url(test),
407                                               timeout).run()
408         if success:
409             return self.convert_result(test, data)
410
411         return (test.result_cls(*data), [])
412
413     def do_testharness(self, marionette, url, timeout):
414         if self.close_after_done:
415             marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
416             self.protocol.close_old_windows(self.protocol)
417
418         if timeout is not None:
419             timeout_ms = str(timeout * 1000)
420         else:
421             timeout_ms = "null"
422
423         script = self.script % {"abs_url": url,
424                                 "url": strip_server(url),
425                                 "window_id": self.window_id,
426                                 "timeout_multiplier": self.timeout_multiplier,
427                                 "timeout": timeout_ms,
428                                 "explicit_timeout": timeout is None}
429
430         rv = marionette.execute_async_script(script, new_sandbox=False)
431         return rv
432
433
434 class MarionetteRefTestExecutor(RefTestExecutor):
435     def __init__(self, browser, server_config, timeout_multiplier=1,
436                  screenshot_cache=None, close_after_done=True,
437                  debug_info=None, reftest_internal=False,
438                  reftest_screenshot="unexpected",
439                  group_metadata=None, capabilities=None, **kwargs):
440         """Marionette-based executor for reftests"""
441         RefTestExecutor.__init__(self,
442                                  browser,
443                                  server_config,
444                                  screenshot_cache=screenshot_cache,
445                                  timeout_multiplier=timeout_multiplier,
446                                  debug_info=debug_info)
447         self.protocol = MarionetteProtocol(self, browser, capabilities,
448                                            timeout_multiplier)
449         self.implementation = (InternalRefTestImplementation
450                                if reftest_internal
451                                else RefTestImplementation)(self)
452         self.implementation_kwargs = ({"screenshot": reftest_screenshot} if
453                                       reftest_internal else {})
454
455         self.close_after_done = close_after_done
456         self.has_window = False
457         self.original_pref_values = {}
458         self.group_metadata = group_metadata
459
460         with open(os.path.join(here, "reftest.js")) as f:
461             self.script = f.read()
462         with open(os.path.join(here, "reftest-wait_marionette.js")) as f:
463             self.wait_script = f.read()
464
465     def setup(self, runner):
466         super(self.__class__, self).setup(runner)
467         self.implementation.setup(**self.implementation_kwargs)
468
469     def teardown(self):
470         try:
471             self.implementation.teardown()
472             handle = self.protocol.marionette.window_handles[0]
473             self.protocol.marionette.switch_to_window(handle)
474             super(self.__class__, self).teardown()
475         except Exception as e:
476             # Ignore errors during teardown
477             self.logger.warning(traceback.format_exc(e))
478
479     def is_alive(self):
480         return self.protocol.is_alive
481
482     def on_environment_change(self, new_environment):
483         self.protocol.on_environment_change(self.last_environment, new_environment)
484
485     def do_test(self, test):
486         if not isinstance(self.implementation, InternalRefTestImplementation):
487             if self.close_after_done and self.has_window:
488                 self.protocol.marionette.close()
489                 self.protocol.marionette.switch_to_window(
490                     self.protocol.marionette.window_handles[-1])
491                 self.has_window = False
492
493             if not self.has_window:
494                 self.protocol.marionette.execute_script(self.script)
495                 self.protocol.marionette.switch_to_window(self.protocol.marionette.window_handles[-1])
496                 self.has_window = True
497
498         result = self.implementation.run_test(test)
499         return self.convert_result(test, result)
500
501     def screenshot(self, test, viewport_size, dpi):
502         # https://github.com/w3c/wptrunner/issues/166
503         assert viewport_size is None
504         assert dpi is None
505
506         timeout = self.timeout_multiplier * test.timeout if self.debug_info is None else None
507
508         test_url = self.test_url(test)
509
510         return ExecuteAsyncScriptRun(self.logger,
511                                      self._screenshot,
512                                      self.protocol,
513                                      test_url,
514                                      timeout).run()
515
516     def _screenshot(self, marionette, url, timeout):
517         marionette.navigate(url)
518
519         marionette.execute_async_script(self.wait_script)
520
521         screenshot = marionette.screenshot(full=False)
522         # strip off the data:img/png, part of the url
523         if screenshot.startswith("data:image/png;base64,"):
524             screenshot = screenshot.split(",", 1)[1]
525
526         return screenshot
527
528
529 class InternalRefTestImplementation(object):
530     def __init__(self, executor):
531         self.timeout_multiplier = executor.timeout_multiplier
532         self.executor = executor
533
534     @property
535     def logger(self):
536         return self.executor.logger
537
538     def setup(self, screenshot="unexpected"):
539         data = {"screenshot": screenshot}
540         if self.executor.group_metadata is not None:
541             data["urlCount"] = {urlparse.urljoin(self.executor.server_url(key[0]), key[1]):value
542                                 for key, value in self.executor.group_metadata.get("url_count", {}).iteritems()
543                                 if value > 1}
544         self.executor.protocol.marionette.set_context(self.executor.protocol.marionette.CONTEXT_CHROME)
545         self.executor.protocol.marionette._send_message("reftest:setup", data)
546
547     def run_test(self, test):
548         viewport_size = test.viewport_size
549         dpi = test.dpi
550
551         references = self.get_references(test)
552         rv = self.executor.protocol.marionette._send_message("reftest:run",
553                                                              {"test": self.executor.test_url(test),
554                                                               "references": references,
555                                                               "expected": test.expected(),
556                                                               "timeout": test.timeout * 1000})["value"]
557         return rv
558
559     def get_references(self, node):
560         rv = []
561         for item, relation in node.references:
562             rv.append([self.executor.test_url(item), self.get_references(item), relation])
563         return rv
564
565     def teardown(self):
566         try:
567             self.executor.protocol.marionette._send_message("reftest:teardown", {})
568             self.executor.protocol.marionette.set_context(self.executor.protocol.marionette.CONTEXT_CONTENT)
569         except Exception as e:
570             # Ignore errors during teardown
571             self.logger.warning(traceback.format_exc(e))
572
573
574
575 class GeckoDriverProtocol(WebDriverProtocol):
576     server_cls = GeckoDriverServer
577
578
579 class MarionetteWdspecExecutor(WdspecExecutor):
580     protocol_cls = GeckoDriverProtocol