fa1c6d5b976b296f1d4f187ea4e8c054c9cb5488
[WebKit-https.git] / WebDriverTests / imported / w3c / tools / wptrunner / wptrunner / executors / executormarionette.py
1 import os
2 import socket
3 import threading
4 import traceback
5 import urlparse
6 import uuid
7
8 errors = None
9 marionette = None
10 pytestrunner = None
11
12 here = os.path.join(os.path.split(__file__)[0])
13
14 from .base import (ExecutorException,
15                    Protocol,
16                    RefTestExecutor,
17                    RefTestImplementation,
18                    TestExecutor,
19                    TestharnessExecutor,
20                    WdspecExecutor,
21                    WdspecRun,
22                    WebDriverProtocol,
23                    extra_timeout,
24                    testharness_result_converter,
25                    reftest_result_converter,
26                    strip_server)
27
28 from ..testrunner import Stop
29 from ..webdriver_server import GeckoDriverServer
30
31
32
33 def do_delayed_imports():
34     global errors, marionette
35
36     # Marionette client used to be called marionette, recently it changed
37     # to marionette_driver for unfathomable reasons
38     try:
39         import marionette
40         from marionette import errors
41     except ImportError:
42         from marionette_driver import marionette, errors
43
44
45 class MarionetteProtocol(Protocol):
46     def __init__(self, executor, browser, timeout_multiplier=1):
47         do_delayed_imports()
48
49         Protocol.__init__(self, executor, browser)
50         self.marionette = None
51         self.marionette_port = browser.marionette_port
52         self.timeout_multiplier = timeout_multiplier
53         self.timeout = None
54         self.runner_handle = None
55
56     def setup(self, runner):
57         """Connect to browser via Marionette."""
58         Protocol.setup(self, runner)
59
60         self.logger.debug("Connecting to Marionette on port %i" % self.marionette_port)
61         startup_timeout = marionette.Marionette.DEFAULT_STARTUP_TIMEOUT * self.timeout_multiplier
62         self.marionette = marionette.Marionette(host='localhost',
63                                                 port=self.marionette_port,
64                                                 socket_timeout=None,
65                                                 startup_timeout=None)
66
67         # XXX Move this timeout somewhere
68         self.logger.debug("Waiting for Marionette connection")
69         while True:
70             success = self.marionette.wait_for_port(startup_timeout)
71             #When running in a debugger wait indefinitely for firefox to start
72             if success or self.executor.debug_info is None:
73                 break
74
75         session_started = False
76         if success:
77             try:
78                 self.logger.debug("Starting Marionette session")
79                 self.marionette.start_session()
80             except Exception as e:
81                 self.logger.warning("Starting marionette session failed: %s" % e)
82             else:
83                 self.logger.debug("Marionette session started")
84                 session_started = True
85
86         if not success or not session_started:
87             self.logger.warning("Failed to connect to Marionette")
88             self.executor.runner.send_message("init_failed")
89         else:
90             try:
91                 self.after_connect()
92             except Exception:
93                 self.logger.warning("Post-connection steps failed")
94                 self.logger.error(traceback.format_exc())
95                 self.executor.runner.send_message("init_failed")
96             else:
97                 self.executor.runner.send_message("init_succeeded")
98
99     def teardown(self):
100         try:
101             self.marionette._request_in_app_shutdown()
102             self.marionette.delete_session(send_request=False, reset_session_id=True)
103         except Exception:
104             # This is typically because the session never started
105             pass
106         if self.marionette is not None:
107             del self.marionette
108
109     @property
110     def is_alive(self):
111         """Check if the Marionette connection is still active."""
112         try:
113             self.marionette.current_window_handle
114         except Exception:
115             return False
116         return True
117
118     def after_connect(self):
119         self.load_runner(self.executor.last_environment["protocol"])
120
121     def set_timeout(self, timeout):
122         """Set the Marionette script timeout.
123
124         :param timeout: Script timeout in seconds
125
126         """
127         self.marionette.timeout.script = timeout
128         self.timeout = timeout
129
130     def load_runner(self, protocol):
131         # Check if we previously had a test window open, and if we did make sure it's closed
132         self.marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
133         url = urlparse.urljoin(self.executor.server_url(protocol), "/testharness_runner.html")
134         self.logger.debug("Loading %s" % url)
135         self.runner_handle = self.marionette.current_window_handle
136         try:
137             self.marionette.navigate(url)
138         except Exception as e:
139             self.logger.critical(
140                 "Loading initial page %s failed. Ensure that the "
141                 "there are no other programs bound to this port and "
142                 "that your firewall rules or network setup does not "
143                 "prevent access.\e%s" % (url, traceback.format_exc(e)))
144         self.marionette.execute_script(
145             "document.title = '%s'" % threading.current_thread().name.replace("'", '"'))
146
147     def close_old_windows(self, protocol):
148         handles = self.marionette.window_handles
149         runner_handle = None
150         try:
151             handles.remove(self.runner_handle)
152             runner_handle = self.runner_handle
153         except ValueError:
154             # The runner window probably changed id but we can restore it
155             # This isn't supposed to happen, but marionette ids are not yet stable
156             # We assume that the first handle returned corresponds to the runner,
157             # but it hopefully doesn't matter too much if that assumption is
158             # wrong since we reload the runner in that tab anyway.
159             runner_handle = handles.pop(0)
160
161         for handle in handles:
162             self.marionette.switch_to_window(handle)
163             self.marionette.close()
164
165         self.marionette.switch_to_window(runner_handle)
166         if runner_handle != self.runner_handle:
167             self.load_runner(protocol)
168
169     def wait(self):
170         socket_timeout = self.marionette.client.sock.gettimeout()
171         if socket_timeout:
172             self.marionette.timeout.script = socket_timeout / 2
173
174         self.marionette.switch_to_window(self.runner_handle)
175         while True:
176             try:
177                 self.marionette.execute_async_script("")
178             except errors.NoSuchWindowException:
179                 # The window closed
180                 break
181             except errors.ScriptTimeoutException:
182                 self.logger.debug("Script timed out")
183                 pass
184             except (socket.timeout, IOError):
185                 self.logger.debug("Socket closed")
186                 break
187             except Exception as e:
188                 self.logger.warning(traceback.format_exc(e))
189                 break
190
191     def on_environment_change(self, old_environment, new_environment):
192         #Unset all the old prefs
193         for name in old_environment.get("prefs", {}).iterkeys():
194             value = self.executor.original_pref_values[name]
195             if value is None:
196                 self.clear_user_pref(name)
197             else:
198                 self.set_pref(name, value)
199
200         for name, value in new_environment.get("prefs", {}).iteritems():
201             self.executor.original_pref_values[name] = self.get_pref(name)
202             self.set_pref(name, value)
203
204     def set_pref(self, name, value):
205         if value.lower() not in ("true", "false"):
206             try:
207                 int(value)
208             except ValueError:
209                 value = "'%s'" % value
210         else:
211             value = value.lower()
212
213         self.logger.info("Setting pref %s (%s)" % (name, value))
214
215         script = """
216             let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
217                                           .getService(Components.interfaces.nsIPrefBranch);
218             let pref = '%s';
219             let type = prefInterface.getPrefType(pref);
220             let value = %s;
221             switch(type) {
222                 case prefInterface.PREF_STRING:
223                     prefInterface.setCharPref(pref, value);
224                     break;
225                 case prefInterface.PREF_BOOL:
226                     prefInterface.setBoolPref(pref, value);
227                     break;
228                 case prefInterface.PREF_INT:
229                     prefInterface.setIntPref(pref, value);
230                     break;
231             }
232             """ % (name, value)
233         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
234             self.marionette.execute_script(script)
235
236     def clear_user_pref(self, name):
237         self.logger.info("Clearing pref %s" % (name))
238         script = """
239             let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
240                                           .getService(Components.interfaces.nsIPrefBranch);
241             let pref = '%s';
242             prefInterface.clearUserPref(pref);
243             """ % name
244         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
245             self.marionette.execute_script(script)
246
247     def get_pref(self, name):
248         script = """
249             let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
250                                           .getService(Components.interfaces.nsIPrefBranch);
251             let pref = '%s';
252             let type = prefInterface.getPrefType(pref);
253             switch(type) {
254                 case prefInterface.PREF_STRING:
255                     return prefInterface.getCharPref(pref);
256                 case prefInterface.PREF_BOOL:
257                     return prefInterface.getBoolPref(pref);
258                 case prefInterface.PREF_INT:
259                     return prefInterface.getIntPref(pref);
260                 case prefInterface.PREF_INVALID:
261                     return null;
262             }
263             """ % name
264         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
265             self.marionette.execute_script(script)
266
267     def clear_origin(self, url):
268         self.logger.info("Clearing origin %s" % (url))
269         script = """
270             let url = '%s';
271             let uri = Components.classes["@mozilla.org/network/io-service;1"]
272                                 .getService(Ci.nsIIOService)
273                                 .newURI(url);
274             let ssm = Components.classes["@mozilla.org/scriptsecuritymanager;1"]
275                                 .getService(Ci.nsIScriptSecurityManager);
276             let principal = ssm.createCodebasePrincipal(uri, {});
277             let qms = Components.classes["@mozilla.org/dom/quota-manager-service;1"]
278                                 .getService(Components.interfaces.nsIQuotaManagerService);
279             qms.clearStoragesForPrincipal(principal, "default", true);
280             """ % url
281         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
282             self.marionette.execute_script(script)
283
284
285 class ExecuteAsyncScriptRun(object):
286     def __init__(self, logger, func, protocol, url, timeout):
287         self.logger = logger
288         self.result = (None, None)
289         self.protocol = protocol
290         self.marionette = protocol.marionette
291         self.func = func
292         self.url = url
293         self.timeout = timeout
294         self.result_flag = threading.Event()
295
296     def run(self):
297         index = self.url.rfind("/storage/");
298         if index != -1:
299             # Clear storage
300             self.protocol.clear_origin(self.url)
301
302         timeout = self.timeout
303
304         try:
305             if timeout is not None:
306                 if timeout + extra_timeout != self.protocol.timeout:
307                     self.protocol.set_timeout(timeout + extra_timeout)
308             else:
309                 # We just want it to never time out, really, but marionette doesn't
310                 # make that possible. It also seems to time out immediately if the
311                 # timeout is set too high. This works at least.
312                 self.protocol.set_timeout(2**28 - 1)
313         except IOError:
314             self.logger.error("Lost marionette connection before starting test")
315             return Stop
316
317         executor = threading.Thread(target = self._run)
318         executor.start()
319
320         if timeout is not None:
321             wait_timeout = timeout + 2 * extra_timeout
322         else:
323             wait_timeout = None
324
325         flag = self.result_flag.wait(wait_timeout)
326
327         if self.result == (None, None):
328             self.logger.debug("Timed out waiting for a result")
329             self.result = False, ("EXTERNAL-TIMEOUT", None)
330         elif self.result[1] is None:
331             # We didn't get any data back from the test, so check if the
332             # browser is still responsive
333             if self.protocol.is_alive:
334                 self.result = False, ("ERROR", None)
335             else:
336                 self.result = False, ("CRASH", None)
337         return self.result
338
339     def _run(self):
340         try:
341             self.result = True, self.func(self.marionette, self.url, self.timeout)
342         except errors.ScriptTimeoutException:
343             self.logger.debug("Got a marionette timeout")
344             self.result = False, ("EXTERNAL-TIMEOUT", None)
345         except (socket.timeout, IOError):
346             # This can happen on a crash
347             # Also, should check after the test if the firefox process is still running
348             # and otherwise ignore any other result and set it to crash
349             self.result = False, ("CRASH", None)
350         except Exception as e:
351             message = getattr(e, "message", "")
352             if message:
353                 message += "\n"
354             message += traceback.format_exc(e)
355             self.result = False, ("ERROR", e)
356
357         finally:
358             self.result_flag.set()
359
360
361 class MarionetteTestharnessExecutor(TestharnessExecutor):
362     def __init__(self, browser, server_config, timeout_multiplier=1,
363                  close_after_done=True, debug_info=None, **kwargs):
364         """Marionette-based executor for testharness.js tests"""
365         TestharnessExecutor.__init__(self, browser, server_config,
366                                      timeout_multiplier=timeout_multiplier,
367                                      debug_info=debug_info)
368
369         self.protocol = MarionetteProtocol(self, browser, timeout_multiplier)
370         self.script = open(os.path.join(here, "testharness_marionette.js")).read()
371         self.close_after_done = close_after_done
372         self.window_id = str(uuid.uuid4())
373
374         self.original_pref_values = {}
375
376         if marionette is None:
377             do_delayed_imports()
378
379     def is_alive(self):
380         return self.protocol.is_alive
381
382     def on_environment_change(self, new_environment):
383         self.protocol.on_environment_change(self.last_environment, new_environment)
384
385         if new_environment["protocol"] != self.last_environment["protocol"]:
386             self.protocol.load_runner(new_environment["protocol"])
387
388     def do_test(self, test):
389         timeout = (test.timeout * self.timeout_multiplier if self.debug_info is None
390                    else None)
391
392         success, data = ExecuteAsyncScriptRun(self.logger,
393                                               self.do_testharness,
394                                               self.protocol,
395                                               self.test_url(test),
396                                               timeout).run()
397         if success:
398             return self.convert_result(test, data)
399
400         return (test.result_cls(*data), [])
401
402     def do_testharness(self, marionette, url, timeout):
403         if self.close_after_done:
404             marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
405             self.protocol.close_old_windows(self.protocol)
406
407         if timeout is not None:
408             timeout_ms = str(timeout * 1000)
409         else:
410             timeout_ms = "null"
411
412         script = self.script % {"abs_url": url,
413                                 "url": strip_server(url),
414                                 "window_id": self.window_id,
415                                 "timeout_multiplier": self.timeout_multiplier,
416                                 "timeout": timeout_ms,
417                                 "explicit_timeout": timeout is None}
418
419         rv = marionette.execute_async_script(script, new_sandbox=False)
420         return rv
421
422
423 class MarionetteRefTestExecutor(RefTestExecutor):
424     def __init__(self, browser, server_config, timeout_multiplier=1,
425                  screenshot_cache=None, close_after_done=True,
426                  debug_info=None, reftest_internal=False,
427                  reftest_screenshot="unexpected",
428                  group_metadata=None, **kwargs):
429         """Marionette-based executor for reftests"""
430         RefTestExecutor.__init__(self,
431                                  browser,
432                                  server_config,
433                                  screenshot_cache=screenshot_cache,
434                                  timeout_multiplier=timeout_multiplier,
435                                  debug_info=debug_info)
436         self.protocol = MarionetteProtocol(self, browser)
437         self.implementation = (InternalRefTestImplementation
438                                if reftest_internal
439                                else RefTestImplementation)(self)
440         self.implementation_kwargs = ({"screenshot": reftest_screenshot} if
441                                       reftest_internal else {})
442
443         self.close_after_done = close_after_done
444         self.has_window = False
445         self.original_pref_values = {}
446         self.group_metadata = group_metadata
447
448         with open(os.path.join(here, "reftest.js")) as f:
449             self.script = f.read()
450         with open(os.path.join(here, "reftest-wait_marionette.js")) as f:
451             self.wait_script = f.read()
452
453     def setup(self, runner):
454         super(self.__class__, self).setup(runner)
455         self.implementation.setup(**self.implementation_kwargs)
456
457     def teardown(self):
458         try:
459             self.implementation.teardown()
460             handle = self.protocol.marionette.window_handles[0]
461             self.protocol.marionette.switch_to_window(handle)
462             super(self.__class__, self).teardown()
463         except Exception as e:
464             # Ignore errors during teardown
465             self.logger.warning(traceback.format_exc(e))
466
467     def is_alive(self):
468         return self.protocol.is_alive
469
470     def on_environment_change(self, new_environment):
471         self.protocol.on_environment_change(self.last_environment, new_environment)
472
473     def do_test(self, test):
474         if not isinstance(self.implementation, InternalRefTestImplementation):
475             if self.close_after_done and self.has_window:
476                 self.protocol.marionette.close()
477                 self.protocol.marionette.switch_to_window(
478                     self.protocol.marionette.window_handles[-1])
479                 self.has_window = False
480
481             if not self.has_window:
482                 self.protocol.marionette.execute_script(self.script)
483                 self.protocol.marionette.switch_to_window(self.protocol.marionette.window_handles[-1])
484                 self.has_window = True
485
486         result = self.implementation.run_test(test)
487         return self.convert_result(test, result)
488
489     def screenshot(self, test, viewport_size, dpi):
490         # https://github.com/w3c/wptrunner/issues/166
491         assert viewport_size is None
492         assert dpi is None
493
494         timeout =  self.timeout_multiplier * test.timeout if self.debug_info is None else None
495
496         test_url = self.test_url(test)
497
498         return ExecuteAsyncScriptRun(self.logger,
499                                      self._screenshot,
500                                      self.protocol,
501                                      test_url,
502                                      timeout).run()
503
504     def _screenshot(self, marionette, url, timeout):
505         marionette.navigate(url)
506
507         marionette.execute_async_script(self.wait_script)
508
509         screenshot = marionette.screenshot(full=False)
510         # strip off the data:img/png, part of the url
511         if screenshot.startswith("data:image/png;base64,"):
512             screenshot = screenshot.split(",", 1)[1]
513
514         return screenshot
515
516
517 class InternalRefTestImplementation(object):
518     def __init__(self, executor):
519         self.timeout_multiplier = executor.timeout_multiplier
520         self.executor = executor
521
522     @property
523     def logger(self):
524         return self.executor.logger
525
526     def setup(self, screenshot="unexpected"):
527         data = {"screenshot": screenshot}
528         if self.executor.group_metadata is not None:
529             data["urlCount"] = {urlparse.urljoin(self.executor.server_url(key[0]), key[1]):value
530                                 for key, value in self.executor.group_metadata.get("url_count", {}).iteritems()
531                                 if value > 1}
532         self.executor.protocol.marionette.set_context(self.executor.protocol.marionette.CONTEXT_CHROME)
533         self.executor.protocol.marionette._send_message("reftest:setup", data)
534
535     def run_test(self, test):
536         viewport_size = test.viewport_size
537         dpi = test.dpi
538
539         references = self.get_references(test)
540         rv = self.executor.protocol.marionette._send_message("reftest:run",
541                                                              {"test": self.executor.test_url(test),
542                                                               "references": references,
543                                                               "expected": test.expected(),
544                                                               "timeout": test.timeout * 1000})["value"]
545         return rv
546
547     def get_references(self, node):
548         rv = []
549         for item, relation in node.references:
550             rv.append([self.executor.test_url(item), self.get_references(item), relation])
551         return rv
552
553     def teardown(self):
554         try:
555             self.executor.protocol.marionette._send_message("reftest:teardown", {})
556             self.executor.protocol.marionette.set_context(self.executor.protocol.marionette.CONTEXT_CONTENT)
557         except Exception as e:
558             # Ignore errors during teardown
559             self.logger.warning(traceback.traceback.format_exc(e))
560
561
562
563 class GeckoDriverProtocol(WebDriverProtocol):
564     server_cls = GeckoDriverServer
565
566
567 class MarionetteWdspecExecutor(WdspecExecutor):
568     protocol_cls = GeckoDriverProtocol