REGRESSION(r263625): [WPE][GTK] MiniBrowser output no longer includes stderr and...
[WebKit-https.git] / Tools / Scripts / webkitpy / common / system / executive.py
1 # Copyright (c) 2009, Google Inc. All rights reserved.
2 # Copyright (c) 2009 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import errno
31 import logging
32 import multiprocessing
33 import os
34 import signal
35 import subprocess
36 import sys
37 import time
38
39 from webkitpy.common.system.abstractexecutive import AbstractExecutive
40 from webkitpy.common.system.outputtee import Tee
41 from webkitpy.common import unicode_compatibility
42
43
44 _log = logging.getLogger(__name__)
45
46
47 class ScriptError(Exception):
48
49     def __init__(self,
50                  message=None,
51                  script_args=None,
52                  exit_code=None,
53                  output=None,
54                  cwd=None):
55         if not message:
56             message = 'Failed to run "%s"' % repr(script_args)
57             if exit_code:
58                 message += " exit_code: %d" % exit_code
59             if cwd:
60                 message += " cwd: %s" % cwd
61
62         Exception.__init__(self, message)
63         self.script_args = script_args  # 'args' is already used by Exception
64         self.exit_code = exit_code
65         self.output = output
66         self.cwd = cwd
67
68     def message_with_output(self, output_limit=500):
69         if self.output:
70             if output_limit and len(self.output) > output_limit:
71                 return u"%s\n\nLast %s characters of output:\n%s" % \
72                     (self, output_limit, self.output[-output_limit:])
73             return u"%s\n\n%s" % (self, self.output)
74         return unicode_compatibility.unicode(self)
75
76     def command_name(self):
77         command_path = self.script_args
78         if type(command_path) is list:
79             command_path = command_path[0]
80         return os.path.basename(command_path)
81
82
83 class Executive(AbstractExecutive):
84     PIPE = subprocess.PIPE
85     STDOUT = subprocess.STDOUT
86
87     class WrappedPopen(object):
88         def __init__(self, popen):
89             for attribute in dir(popen):
90                 if attribute.startswith('__'):
91                     continue
92                 setattr(self, attribute, getattr(popen, attribute))
93
94         def __enter__(self):
95             return self
96
97         def __exit__(self, *args):
98             self.wait()
99
100     def __init__(self):
101         self.pid_to_system_pid = {}
102         self._is_native_win = sys.platform.startswith('win')
103         self._is_cygwin = sys.platform == 'cygwin'
104
105     def _should_close_fds(self):
106         # We need to pass close_fds=True to work around Python bug #2320
107         # (otherwise we can hang when we kill DumpRenderTree when we are running
108         # multiple threads). See http://bugs.python.org/issue2320 .
109         # In Python 2.7.10, close_fds is also supported on Windows.
110         # However, "you cannot set close_fds to true and also redirect the standard
111         # handles by setting stdin, stdout or stderr.".
112         if self._is_native_win:
113             return False
114         else:
115             return True
116
117     def _run_command_with_teed_output(self, args, teed_output, **kwargs):
118         child_process = self.popen(args,
119                                    stdout=self.PIPE,
120                                    stderr=self.STDOUT,
121                                    close_fds=self._should_close_fds(),
122                                    **kwargs)
123
124         with child_process:
125             # Use our own custom wait loop because Popen ignores a tee'd
126             # stderr/stdout.
127             # FIXME: This could be improved not to flatten output to stdout.
128             while child_process.poll() is None:
129                 output_line = child_process.stdout.readline()
130                 teed_output.write(unicode_compatibility.decode_for(output_line, str))
131             return child_process.poll()
132
133     # FIXME: Remove this deprecated method and move callers to run_command.
134     # FIXME: This method is a hack to allow running command which both
135     # capture their output and print out to stdin.  Useful for things
136     # like "build-webkit" where we want to display to the user that we're building
137     # but still have the output to stuff into a log file.
138     def run_and_throw_if_fail(self, args, quiet=False, decode_output=True, **kwargs):
139         # Cache the child's output locally so it can be used for error reports.
140         child_out_file = unicode_compatibility.StringIO()
141         tee_stdout = sys.stdout
142         try:
143             if quiet:
144                 dev_null = open(os.devnull, "w")  # FIXME: Does this need an encoding?
145                 tee_stdout = dev_null
146             child_stdout = Tee(child_out_file, tee_stdout)
147             exit_code = self._run_command_with_teed_output(args, child_stdout, **kwargs)
148         finally:
149             if quiet:
150                 dev_null.close()
151
152         child_output = child_out_file.getvalue()
153         child_out_file.close()
154
155         if decode_output:
156             child_output = unicode_compatibility.decode_if_necessary(child_output, self._child_process_encoding())
157         else:
158             child_output = unicode_compatibility.encode_if_necessary(child_output, self._child_process_encoding())
159
160         if exit_code:
161             raise ScriptError(script_args=args,
162                               exit_code=exit_code,
163                               output=child_output)
164         return child_output
165
166     def cpu_count(self):
167         try:
168             cpus = int(os.environ.get('NUMBER_OF_PROCESSORS'))
169             if cpus > 0:
170                 return cpus
171         except (ValueError, TypeError):
172             pass
173         return multiprocessing.cpu_count()
174
175     def kill_process(self, pid):
176         # Killing a process with a pid of 0 or a negative pid is a valid command, but
177         # will kill all processes in this process' group (if 0) or all non-system processes
178         # (if -1) (kill(2)). Throw an exception if this is the behavior requested, this
179         # class is not designed to provide this functionality.
180         if pid is None or pid <= 0:
181             raise RuntimeError('Cannot kill process with invalid pid of {}'.format(pid))
182
183         """Attempts to kill the given pid.
184         Will fail silently if pid does not exist or insufficient permisssions."""
185         if self._is_native_win:
186             # We only use taskkill.exe on windows (not cygwin) because subprocess.pid
187             # is a CYGWIN pid and taskkill.exe expects a windows pid.
188             # Thankfully os.kill on CYGWIN handles either pid type.
189             task_kill_executable = os.path.join('C:', os.sep, 'WINDOWS', 'system32', 'taskkill.exe')
190             command = [task_kill_executable, "/f", "/t", "/pid", pid]
191             # taskkill will exit 128 if the process is not found.  We should log.
192             self.run_command(command, ignore_errors=True)
193             return
194
195         # According to http://docs.python.org/library/os.html
196         # os.kill isn't available on Windows. python 2.5.5 os.kill appears
197         # to work in cygwin, however it occasionally raises EAGAIN.
198         retries_left = 10 if self._is_cygwin else 2
199         current_signal = signal.SIGTERM
200         while retries_left > 0 and self.check_running_pid(pid):
201             try:
202                 retries_left -= 1
203                 os.kill(pid, current_signal)
204             except OSError as e:
205                 if current_signal == signal.SIGTERM:
206                     pass
207                 elif e.errno == errno.EAGAIN:
208                     if retries_left <= 0:
209                         _log.warn("Failed to kill pid %s.  Too many EAGAIN errors." % pid)
210                 elif e.errno == errno.ESRCH:  # The process does not exist.
211                     return
212                 elif e.errno == errno.EPIPE:  # The process has exited already on cygwin
213                     return
214                 elif e.errno == errno.ECHILD:
215                     # Can't wait on a non-child process, but the kill worked.
216                     return
217                 elif e.errno == errno.EACCES and self._is_cygwin:
218                     # Cygwin python sometimes can't kill native processes.
219                     return
220                 else:
221                     raise
222
223             # Give processes one chance to clean up quickly before exiting.
224             current_signal = signal.SIGKILL
225
226     def _win32_check_running_pid(self, pid):
227         # importing ctypes at the top-level seems to cause weird crashes at
228         # exit under cygwin on apple's win port. Only win32 needs cygwin, so
229         # we import it here instead. See https://bugs.webkit.org/show_bug.cgi?id=91682
230         import ctypes
231
232         class PROCESSENTRY32(ctypes.Structure):
233             _fields_ = [("dwSize", ctypes.c_ulong),
234                         ("cntUsage", ctypes.c_ulong),
235                         ("th32ProcessID", ctypes.c_ulong),
236                         ("th32DefaultHeapID", ctypes.POINTER(ctypes.c_ulong)),
237                         ("th32ModuleID", ctypes.c_ulong),
238                         ("cntThreads", ctypes.c_ulong),
239                         ("th32ParentProcessID", ctypes.c_ulong),
240                         ("pcPriClassBase", ctypes.c_ulong),
241                         ("dwFlags", ctypes.c_ulong),
242                         ("szExeFile", ctypes.c_char * 260)]
243
244         CreateToolhelp32Snapshot = ctypes.windll.kernel32.CreateToolhelp32Snapshot
245         Process32First = ctypes.windll.kernel32.Process32First
246         Process32Next = ctypes.windll.kernel32.Process32Next
247         CloseHandle = ctypes.windll.kernel32.CloseHandle
248         TH32CS_SNAPPROCESS = 0x00000002  # win32 magic number
249         hProcessSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0)
250         pe32 = PROCESSENTRY32()
251         pe32.dwSize = ctypes.sizeof(PROCESSENTRY32)
252         result = False
253         if not Process32First(hProcessSnap, ctypes.byref(pe32)):
254             _log.debug("Failed getting first process.")
255             CloseHandle(hProcessSnap)
256             return result
257         while True:
258             if pe32.th32ProcessID == pid:
259                 result = True
260                 break
261             if not Process32Next(hProcessSnap, ctypes.byref(pe32)):
262                 break
263         CloseHandle(hProcessSnap)
264         return result
265
266     def check_running_pid(self, pid):
267         # An undefined process or a negative process are never running.
268         if pid is None or pid <= 0:
269             return False
270
271         """Return True if pid is alive, otherwise return False."""
272         if self._is_native_win:
273             return self._win32_check_running_pid(pid)
274
275         try:
276             os.kill(pid, 0)
277             return True
278         except OSError:
279             return False
280
281     def running_pids(self, process_name_filter=None):
282         if self._is_native_win:
283             # FIXME: running_pids isn't implemented on native Windows yet...
284             return []
285
286         if not process_name_filter:
287             process_name_filter = lambda process_name: True
288
289         running_pids = []
290         if self._is_cygwin:
291             ps_process = self.run_command(['ps', '-e'], ignore_errors=True)
292             for line in ps_process.splitlines():
293                 tokens = line.strip().split()
294                 try:
295                     pid, ppid, pgid, winpid, tty, uid, stime, process_name = tokens
296                     if process_name_filter(process_name):
297                         running_pids.append(int(pid))
298                         self.pid_to_system_pid[int(pid)] = int(winpid)
299                 except ValueError as e:
300                     pass
301         else:
302             with self.popen(['ps', '-eo', 'pid,comm'], stdout=self.PIPE, stderr=self.PIPE) as ps_process:
303                 stdout, _ = ps_process.communicate()
304                 for line in stdout.splitlines():
305                     try:
306                         # In some cases the line can contain one or more
307                         # leading white-spaces, so strip it before split.
308                         pid, process_name = line.strip().split(b' ', 1)
309                         if process_name_filter(unicode_compatibility.decode_for(process_name, str)):
310                             running_pids.append(int(pid))
311                     except ValueError as e:
312                         pass
313
314         return sorted(running_pids)
315
316     def _windows_image_name(self, process_name):
317         name, extension = os.path.splitext(process_name)
318         if not extension:
319             # taskkill expects processes to end in .exe
320             # If necessary we could add a flag to disable appending .exe.
321             process_name = "%s.exe" % name
322         return process_name
323
324     def _windows_kill_command(self):
325         if self._is_native_win:
326             return os.path.join("C:", os.sep, "WINDOWS", "system32", "taskkill.exe")
327         else:
328             return 'taskkill.exe'
329
330     def interrupt(self, pid):
331         if self._is_native_win:
332             command = [self._windows_kill_command(), "/f", "/t", "/pid", str(pid)]
333             self.run_command(command, ignore_errors=True)
334         else:
335             try:
336                 os.kill(pid, signal.SIGINT)
337             except OSError:
338                 # Silently ignore when the pid doesn't exist.
339                 # It's impossible for callers to avoid race conditions with process shutdown.
340                 pass
341
342     def kill_all(self, process_name):
343         """Attempts to kill processes matching process_name.
344         Will fail silently if no process are found."""
345         if self._is_cygwin or self._is_native_win:
346             image_name = self._windows_image_name(process_name)
347             command = [self._windows_kill_command(), "/f", "/im", image_name]
348             # taskkill will exit 128 if the process is not found.  We should log.
349             self.run_command(command, ignore_errors=True)
350             return
351
352         # FIXME: This is inconsistent that kill_all uses TERM and kill_process
353         # uses KILL.  Windows is always using /f (which seems like -KILL).
354         # We should pick one mode, or add support for switching between them.
355         # Note: Mac OS X 10.6 requires -SIGNALNAME before -u USER
356         command = ["killall", "-TERM", "-u", os.getenv("USER"), process_name]
357         # killall returns 1 if no process can be found and 2 on command error.
358         # FIXME: We should pass a custom error_handler to allow only exit_code 1.
359         # We should log in exit_code == 1
360         self.run_command(command, ignore_errors=True)
361
362     def _compute_stdin(self, input):
363         """Returns (stdin, string_to_communicate)"""
364         # FIXME: We should be returning /dev/null for stdin
365         # or closing stdin after process creation to prevent
366         # child processes from getting input from the user.
367         if not input:
368             return (None, None)
369         if hasattr(input, "read"):  # Check if the input is a file.
370             return (input, None)  # Assume the file is in the right encoding.
371
372         # Popen in Python 2.5 and before does not automatically encode unicode objects.
373         # http://bugs.python.org/issue5290
374         # See https://bugs.webkit.org/show_bug.cgi?id=37528
375         # for an example of a regresion caused by passing a unicode string directly.
376         # FIXME: We may need to encode differently on different platforms.
377         input = unicode_compatibility.encode_if_necessary(input, self._child_process_encoding())
378         return (self.PIPE, input)
379
380     # FIXME: run_and_throw_if_fail should be merged into this method.
381     def run_command(self,
382                     args,
383                     cwd=None,
384                     env=None,
385                     input=None,
386                     stdout=subprocess.PIPE,
387                     error_handler=None,
388                     ignore_errors=False,
389                     return_exit_code=False,
390                     return_stderr=True,
391                     decode_output=True):
392         """Popen wrapper for convenience and to work around python bugs."""
393         assert(isinstance(args, list) or isinstance(args, tuple))
394         start_time = time.time()
395
396         stdin, string_to_communicate = self._compute_stdin(input)
397         stderr = self.STDOUT if return_stderr else None
398
399         process = self.popen(args,
400                              stdin=stdin,
401                              stdout=stdout,
402                              stderr=stderr,
403                              cwd=cwd,
404                              env=env,
405                              close_fds=self._should_close_fds())
406         with process:
407             if not string_to_communicate:
408                 output = process.communicate()[0]
409             else:
410                 output = process.communicate(unicode_compatibility.encode_if_necessary(string_to_communicate, 'utf-8'))[0]
411
412             # run_command automatically decodes to unicode() and converts CRLF to LF unless explicitly told not to.
413             if decode_output:
414                 output = unicode_compatibility.decode_if_necessary(output, self._child_process_encoding()).replace('\r\n', '\n')
415
416             # wait() is not threadsafe and can throw OSError due to:
417             # http://bugs.python.org/issue1731717
418             exit_code = process.wait()
419
420             _log.debug('"%s" took %.2fs' % (self.command_for_printing(args), time.time() - start_time))
421
422             if return_exit_code:
423                 return exit_code
424
425             if exit_code:
426                 script_error = ScriptError(script_args=args,
427                                            exit_code=exit_code,
428                                            output=output,
429                                            cwd=cwd)
430
431                 if ignore_errors:
432                     assert error_handler is None, "don't specify error_handler if ignore_errors is True"
433                     error_handler = Executive.ignore_error
434
435                 (error_handler or self.default_error_handler)(script_error)
436             return output
437
438     def _child_process_encoding(self):
439         # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW
440         # to launch subprocesses, so we have to encode arguments using the
441         # current code page.
442         if self._is_native_win and sys.version < '3':
443             return 'mbcs'
444         # All other platforms use UTF-8.
445         # FIXME: Using UTF-8 on Cygwin will confuse Windows-native commands
446         # which will expect arguments to be encoded using the current code
447         # page.
448         return 'utf-8'
449
450     def _should_encode_child_process_arguments(self):
451         # Cygwin's Python's os.execv doesn't support unicode command
452         # arguments, and neither does Cygwin's execv itself.
453         if self._is_cygwin:
454             return True
455
456         # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW
457         # to launch subprocesses, so we have to encode arguments using the
458         # current code page.
459         if self._is_native_win and sys.version < '3':
460             return True
461
462         return False
463
464     def _encode_argument_if_needed(self, argument):
465         if not self._should_encode_child_process_arguments():
466             return argument
467         return unicode_compatibility.encode_if_necessary(argument, self._child_process_encoding())
468
469     def _stringify_args(self, args):
470         # Popen will throw an exception if args are non-strings (like int())
471         string_args = map(unicode_compatibility.unicode, args)
472         # The Windows implementation of Popen cannot handle unicode strings. :(
473         return map(self._encode_argument_if_needed, string_args)
474
475     def _needs_interpreter_check(self, argument):
476         return not argument.endswith(('perl', 'python', 'ruby', 'taskkill.exe', 'git', 'svn'))
477
478     # The only required argument to popen is named "args", the rest are optional keyword arguments.
479     def popen(self, args, **kwargs):
480         if self._is_native_win:
481             _log.debug("Looking at {0}".format(args))
482             # Must include proper interpreter
483             if self._needs_interpreter_check(args[0]):
484                 try:
485                     with open(args[0], 'r') as f:
486                         line = f.readline()
487                         if "perl" in line:
488                             args.insert(0, "perl")
489                         elif "python" in line:
490                             args.insert(0, "python")
491                         elif "ruby" in line:
492                             args.insert(0, "ruby")
493                 except IOError:
494                     pass
495
496         # FIXME: We should always be stringifying the args, but callers who pass shell=True
497         # expect that the exact bytes passed will get passed to the shell (even if they're wrongly encoded).
498         # shell=True is wrong for many other reasons, and we should remove this
499         # hack as soon as we can fix all callers to not use shell=True.
500         if kwargs.get('shell') == True:
501             string_args = args
502         else:
503             string_args = self._stringify_args(args)
504
505         # Python 3 treats Popen as a context manager, we should allow this in Python 2
506         result = subprocess.Popen(string_args, **kwargs)
507         if not callable(getattr(result, "__enter__", None)) and not callable(getattr(result, "__exit__", None)):
508             return self.WrappedPopen(result)
509         return result
510
511     def run_in_parallel(self, command_lines_and_cwds, processes=None):
512         """Runs a list of (cmd_line list, cwd string) tuples in parallel and returns a list of (retcode, stdout, stderr) tuples."""
513         assert len(command_lines_and_cwds)
514
515         if self._is_cygwin or self._is_native_win:
516             return map(_run_command_thunk, command_lines_and_cwds)
517         pool = multiprocessing.Pool(processes=processes)
518         results = pool.map(_run_command_thunk, command_lines_and_cwds)
519         pool.close()
520         pool.join()
521         return results
522
523
524 def _run_command_thunk(cmd_line_and_cwd):
525     # Note that this needs to be a bare module (and hence Picklable) method to work with multiprocessing.Pool.
526     (cmd_line, cwd) = cmd_line_and_cwd
527     proc = subprocess.Popen(cmd_line, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
528     stdout, stderr = proc.communicate()
529     return (proc.returncode, stdout, stderr)