[Win] Pass close_fds = True in Python popen call.
[WebKit-https.git] / Tools / Scripts / webkitpy / common / system / executive.py
1 # Copyright (c) 2009, Google Inc. All rights reserved.
2 # Copyright (c) 2009 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import StringIO
31 import errno
32 import logging
33 import multiprocessing
34 import os
35 import signal
36 import subprocess
37 import sys
38 import time
39
40 from webkitpy.common.system.outputtee import Tee
41 from webkitpy.common.system.filesystem import FileSystem
42
43
44 _log = logging.getLogger(__name__)
45
46
47 class ScriptError(Exception):
48
49     def __init__(self,
50                  message=None,
51                  script_args=None,
52                  exit_code=None,
53                  output=None,
54                  cwd=None):
55         if not message:
56             message = 'Failed to run "%s"' % repr(script_args)
57             if exit_code:
58                 message += " exit_code: %d" % exit_code
59             if cwd:
60                 message += " cwd: %s" % cwd
61
62         Exception.__init__(self, message)
63         self.script_args = script_args  # 'args' is already used by Exception
64         self.exit_code = exit_code
65         self.output = output
66         self.cwd = cwd
67
68     def message_with_output(self, output_limit=500):
69         if self.output:
70             if output_limit and len(self.output) > output_limit:
71                 return u"%s\n\nLast %s characters of output:\n%s" % \
72                     (self, output_limit, self.output[-output_limit:])
73             return u"%s\n\n%s" % (self, self.output)
74         return unicode(self)
75
76     def command_name(self):
77         command_path = self.script_args
78         if type(command_path) is list:
79             command_path = command_path[0]
80         return os.path.basename(command_path)
81
82
83 class Executive(object):
84     PIPE = subprocess.PIPE
85     STDOUT = subprocess.STDOUT
86
87     def __init__(self):
88         self.pid_to_system_pid = {}
89
90     def _should_close_fds(self):
91         # We need to pass close_fds=True to work around Python bug #2320
92         # (otherwise we can hang when we kill DumpRenderTree when we are running
93         # multiple threads). See http://bugs.python.org/issue2320 .
94         # In Python 2.7.10, close_fds is also supported on Windows.
95         return True
96
97     def _run_command_with_teed_output(self, args, teed_output, **kwargs):
98         child_process = self.popen(args,
99                                    stdout=self.PIPE,
100                                    stderr=self.STDOUT,
101                                    close_fds=self._should_close_fds(),
102                                    **kwargs)
103
104         # Use our own custom wait loop because Popen ignores a tee'd
105         # stderr/stdout.
106         # FIXME: This could be improved not to flatten output to stdout.
107         while True:
108             output_line = child_process.stdout.readline()
109             if output_line == "" and child_process.poll() != None:
110                 # poll() is not threadsafe and can throw OSError due to:
111                 # http://bugs.python.org/issue1731717
112                 return child_process.poll()
113             # We assume that the child process wrote to us in utf-8,
114             # so no re-encoding is necessary before writing here.
115             teed_output.write(output_line)
116
117     # FIXME: Remove this deprecated method and move callers to run_command.
118     # FIXME: This method is a hack to allow running command which both
119     # capture their output and print out to stdin.  Useful for things
120     # like "build-webkit" where we want to display to the user that we're building
121     # but still have the output to stuff into a log file.
122     def run_and_throw_if_fail(self, args, quiet=False, decode_output=True, **kwargs):
123         # Cache the child's output locally so it can be used for error reports.
124         child_out_file = StringIO.StringIO()
125         tee_stdout = sys.stdout
126         if quiet:
127             dev_null = open(os.devnull, "w")  # FIXME: Does this need an encoding?
128             tee_stdout = dev_null
129         child_stdout = Tee(child_out_file, tee_stdout)
130         exit_code = self._run_command_with_teed_output(args, child_stdout, **kwargs)
131         if quiet:
132             dev_null.close()
133
134         child_output = child_out_file.getvalue()
135         child_out_file.close()
136
137         if decode_output:
138             child_output = child_output.decode(self._child_process_encoding())
139
140         if exit_code:
141             raise ScriptError(script_args=args,
142                               exit_code=exit_code,
143                               output=child_output)
144         return child_output
145
146     def cpu_count(self):
147         try:
148             cpus = int(os.environ.get('NUMBER_OF_PROCESSORS'))
149             if cpus > 0:
150                 return cpus
151         except (ValueError, TypeError):
152             pass
153         return multiprocessing.cpu_count()
154
155     @staticmethod
156     def interpreter_for_script(script_path, fs=None):
157         fs = fs or FileSystem()
158         lines = fs.read_text_file(script_path).splitlines()
159         if not len(lines):
160             return None
161         first_line = lines[0]
162         if not first_line.startswith('#!'):
163             return None
164         if first_line.find('python') > -1:
165             return sys.executable
166         if first_line.find('perl') > -1:
167             return 'perl'
168         if first_line.find('ruby') > -1:
169             return 'ruby'
170         return None
171
172     @staticmethod
173     def shell_command_for_script(script_path, fs=None):
174         fs = fs or FileSystem()
175         # Win32 does not support shebang. We need to detect the interpreter ourself.
176         if sys.platform.startswith('win'):
177             interpreter = Executive.interpreter_for_script(script_path, fs)
178             if interpreter:
179                 return [interpreter, script_path]
180         return [script_path]
181
182     def kill_process(self, pid):
183         """Attempts to kill the given pid.
184         Will fail silently if pid does not exist or insufficient permisssions."""
185         if sys.platform.startswith('win32'):
186             # We only use taskkill.exe on windows (not cygwin) because subprocess.pid
187             # is a CYGWIN pid and taskkill.exe expects a windows pid.
188             # Thankfully os.kill on CYGWIN handles either pid type.
189             task_kill_executable = os.path.join('C:', os.sep, 'WINDOWS', 'system32', 'taskkill.exe')
190             command = [task_kill_executable, "/f", "/t", "/pid", pid]
191             # taskkill will exit 128 if the process is not found.  We should log.
192             self.run_command(command, error_handler=self.ignore_error)
193             return
194
195         # According to http://docs.python.org/library/os.html
196         # os.kill isn't available on Windows. python 2.5.5 os.kill appears
197         # to work in cygwin, however it occasionally raises EAGAIN.
198         retries_left = 10 if sys.platform == "cygwin" else 1
199         while retries_left > 0:
200             try:
201                 retries_left -= 1
202                 # Give processes one change to clean up quickly before exiting.
203                 # Following up with a kill should have no effect if the process
204                 # already exited, and forcefully kill it if SIGTERM wasn't enough.
205                 os.kill(pid, signal.SIGTERM)
206                 os.kill(pid, signal.SIGKILL)
207             except OSError, e:
208                 if e.errno == errno.EAGAIN:
209                     if retries_left <= 0:
210                         _log.warn("Failed to kill pid %s.  Too many EAGAIN errors." % pid)
211                     continue
212                 if e.errno == errno.ESRCH:  # The process does not exist.
213                     return
214                 if e.errno == errno.EPIPE:  # The process has exited already on cygwin
215                     return
216                 if e.errno == errno.ECHILD:
217                     # Can't wait on a non-child process, but the kill worked.
218                     return
219                 if e.errno == errno.EACCES and sys.platform == 'cygwin':
220                     # Cygwin python sometimes can't kill native processes.
221                     return
222                 raise
223
224     def _win32_check_running_pid(self, pid):
225         # importing ctypes at the top-level seems to cause weird crashes at
226         # exit under cygwin on apple's win port. Only win32 needs cygwin, so
227         # we import it here instead. See https://bugs.webkit.org/show_bug.cgi?id=91682
228         import ctypes
229
230         class PROCESSENTRY32(ctypes.Structure):
231             _fields_ = [("dwSize", ctypes.c_ulong),
232                         ("cntUsage", ctypes.c_ulong),
233                         ("th32ProcessID", ctypes.c_ulong),
234                         ("th32DefaultHeapID", ctypes.POINTER(ctypes.c_ulong)),
235                         ("th32ModuleID", ctypes.c_ulong),
236                         ("cntThreads", ctypes.c_ulong),
237                         ("th32ParentProcessID", ctypes.c_ulong),
238                         ("pcPriClassBase", ctypes.c_ulong),
239                         ("dwFlags", ctypes.c_ulong),
240                         ("szExeFile", ctypes.c_char * 260)]
241
242         CreateToolhelp32Snapshot = ctypes.windll.kernel32.CreateToolhelp32Snapshot
243         Process32First = ctypes.windll.kernel32.Process32First
244         Process32Next = ctypes.windll.kernel32.Process32Next
245         CloseHandle = ctypes.windll.kernel32.CloseHandle
246         TH32CS_SNAPPROCESS = 0x00000002  # win32 magic number
247         hProcessSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0)
248         pe32 = PROCESSENTRY32()
249         pe32.dwSize = ctypes.sizeof(PROCESSENTRY32)
250         result = False
251         if not Process32First(hProcessSnap, ctypes.byref(pe32)):
252             _log.debug("Failed getting first process.")
253             CloseHandle(hProcessSnap)
254             return result
255         while True:
256             if pe32.th32ProcessID == pid:
257                 result = True
258                 break
259             if not Process32Next(hProcessSnap, ctypes.byref(pe32)):
260                 break
261         CloseHandle(hProcessSnap)
262         return result
263
264     def check_running_pid(self, pid):
265         """Return True if pid is alive, otherwise return False."""
266         if sys.platform.startswith('win'):
267             return self._win32_check_running_pid(pid)
268
269         try:
270             os.kill(pid, 0)
271             return True
272         except OSError:
273             return False
274
275     def running_pids(self, process_name_filter=None):
276         if sys.platform.startswith('win'):
277             # FIXME: running_pids isn't implemented on native Windows yet...
278             return []
279
280         if not process_name_filter:
281             process_name_filter = lambda process_name: True
282
283         running_pids = []
284         if sys.platform in ("cygwin"):
285             ps_process = self.run_command(['ps', '-e'], error_handler=Executive.ignore_error)
286             for line in ps_process.splitlines():
287                 tokens = line.strip().split()
288                 try:
289                     pid, ppid, pgid, winpid, tty, uid, stime, process_name = tokens
290                     if process_name_filter(process_name):
291                         running_pids.append(int(pid))
292                         self.pid_to_system_pid[int(pid)] = int(winpid)
293                 except ValueError, e:
294                     pass
295         else:
296             ps_process = self.popen(['ps', '-eo', 'pid,comm'], stdout=self.PIPE, stderr=self.PIPE)
297             stdout, _ = ps_process.communicate()
298             for line in stdout.splitlines():
299                 try:
300                     # In some cases the line can contain one or more
301                     # leading white-spaces, so strip it before split.
302                     pid, process_name = line.strip().split(' ', 1)
303                     if process_name_filter(process_name):
304                         running_pids.append(int(pid))
305                 except ValueError, e:
306                     pass
307
308         return sorted(running_pids)
309
310     def wait_newest(self, process_name_filter=None):
311         if not process_name_filter:
312             process_name_filter = lambda process_name: True
313
314         running_pids = self.running_pids(process_name_filter)
315         if not running_pids:
316             return
317         pid = running_pids[-1]
318
319         while self.check_running_pid(pid):
320             time.sleep(0.25)
321
322     def wait_limited(self, pid, limit_in_seconds=None, check_frequency_in_seconds=None):
323         seconds_left = limit_in_seconds or 10
324         sleep_length = check_frequency_in_seconds or 1
325         while seconds_left > 0 and self.check_running_pid(pid):
326             seconds_left -= sleep_length
327             time.sleep(sleep_length)
328
329     def _windows_image_name(self, process_name):
330         name, extension = os.path.splitext(process_name)
331         if not extension:
332             # taskkill expects processes to end in .exe
333             # If necessary we could add a flag to disable appending .exe.
334             process_name = "%s.exe" % name
335         return process_name
336
337     def interrupt(self, pid):
338         interrupt_signal = signal.SIGINT
339         # FIXME: The python docs seem to imply that platform == 'win32' may need to use signal.CTRL_C_EVENT
340         # http://docs.python.org/2/library/signal.html
341         try:
342             os.kill(pid, interrupt_signal)
343         except OSError:
344             # Silently ignore when the pid doesn't exist.
345             # It's impossible for callers to avoid race conditions with process shutdown.
346             pass
347
348     def kill_all(self, process_name):
349         """Attempts to kill processes matching process_name.
350         Will fail silently if no process are found."""
351         if sys.platform == 'cygwin' or sys.platform.startswith('win'):
352             image_name = self._windows_image_name(process_name)
353             killCommmand = 'taskkill.exe'
354             if sys.platform.startswith('win'):
355                 killCommand = os.path.join('C:', os.sep, 'WINDOWS', 'system32', 'taskkill.exe')
356             command = [killCommmand, "/f", "/im", image_name]
357             # taskkill will exit 128 if the process is not found.  We should log.
358             self.run_command(command, error_handler=self.ignore_error)
359             return
360
361         # FIXME: This is inconsistent that kill_all uses TERM and kill_process
362         # uses KILL.  Windows is always using /f (which seems like -KILL).
363         # We should pick one mode, or add support for switching between them.
364         # Note: Mac OS X 10.6 requires -SIGNALNAME before -u USER
365         command = ["killall", "-TERM", "-u", os.getenv("USER"), process_name]
366         # killall returns 1 if no process can be found and 2 on command error.
367         # FIXME: We should pass a custom error_handler to allow only exit_code 1.
368         # We should log in exit_code == 1
369         self.run_command(command, error_handler=self.ignore_error)
370
371     # Error handlers do not need to be static methods once all callers are
372     # updated to use an Executive object.
373
374     @staticmethod
375     def default_error_handler(error):
376         raise error
377
378     @staticmethod
379     def ignore_error(error):
380         pass
381
382     def _compute_stdin(self, input):
383         """Returns (stdin, string_to_communicate)"""
384         # FIXME: We should be returning /dev/null for stdin
385         # or closing stdin after process creation to prevent
386         # child processes from getting input from the user.
387         if not input:
388             return (None, None)
389         if hasattr(input, "read"):  # Check if the input is a file.
390             return (input, None)  # Assume the file is in the right encoding.
391
392         # Popen in Python 2.5 and before does not automatically encode unicode objects.
393         # http://bugs.python.org/issue5290
394         # See https://bugs.webkit.org/show_bug.cgi?id=37528
395         # for an example of a regresion caused by passing a unicode string directly.
396         # FIXME: We may need to encode differently on different platforms.
397         if isinstance(input, unicode):
398             input = input.encode(self._child_process_encoding())
399         return (self.PIPE, input)
400
401     def command_for_printing(self, args):
402         """Returns a print-ready string representing command args.
403         The string should be copy/paste ready for execution in a shell."""
404         args = self._stringify_args(args)
405         escaped_args = []
406         for arg in args:
407             if isinstance(arg, unicode):
408                 # Escape any non-ascii characters for easy copy/paste
409                 arg = arg.encode("unicode_escape")
410             # FIXME: Do we need to fix quotes here?
411             escaped_args.append(arg)
412         return " ".join(escaped_args)
413
414     # FIXME: run_and_throw_if_fail should be merged into this method.
415     def run_command(self,
416                     args,
417                     cwd=None,
418                     env=None,
419                     input=None,
420                     error_handler=None,
421                     return_exit_code=False,
422                     return_stderr=True,
423                     decode_output=True):
424         """Popen wrapper for convenience and to work around python bugs."""
425         assert(isinstance(args, list) or isinstance(args, tuple))
426         start_time = time.time()
427
428         stdin, string_to_communicate = self._compute_stdin(input)
429         stderr = self.STDOUT if return_stderr else None
430
431         process = self.popen(args,
432                              stdin=stdin,
433                              stdout=self.PIPE,
434                              stderr=stderr,
435                              cwd=cwd,
436                              env=env,
437                              close_fds=self._should_close_fds())
438         output = process.communicate(string_to_communicate)[0]
439
440         # run_command automatically decodes to unicode() unless explicitly told not to.
441         if decode_output:
442             output = output.decode(self._child_process_encoding())
443
444         # wait() is not threadsafe and can throw OSError due to:
445         # http://bugs.python.org/issue1731717
446         exit_code = process.wait()
447
448         _log.debug('"%s" took %.2fs' % (self.command_for_printing(args), time.time() - start_time))
449
450         if return_exit_code:
451             return exit_code
452
453         if exit_code:
454             script_error = ScriptError(script_args=args,
455                                        exit_code=exit_code,
456                                        output=output,
457                                        cwd=cwd)
458             (error_handler or self.default_error_handler)(script_error)
459         return output
460
461     def _child_process_encoding(self):
462         # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW
463         # to launch subprocesses, so we have to encode arguments using the
464         # current code page.
465         if sys.platform.startswith('win') and sys.version < '3':
466             return 'mbcs'
467         # All other platforms use UTF-8.
468         # FIXME: Using UTF-8 on Cygwin will confuse Windows-native commands
469         # which will expect arguments to be encoded using the current code
470         # page.
471         return 'utf-8'
472
473     def _should_encode_child_process_arguments(self):
474         # Cygwin's Python's os.execv doesn't support unicode command
475         # arguments, and neither does Cygwin's execv itself.
476         if sys.platform == 'cygwin':
477             return True
478
479         # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW
480         # to launch subprocesses, so we have to encode arguments using the
481         # current code page.
482         if sys.platform.startswith('win') and sys.version < '3':
483             return True
484
485         return False
486
487     def _encode_argument_if_needed(self, argument):
488         if not self._should_encode_child_process_arguments():
489             return argument
490         return argument.encode(self._child_process_encoding())
491
492     def _stringify_args(self, args):
493         # Popen will throw an exception if args are non-strings (like int())
494         string_args = map(unicode, args)
495         # The Windows implementation of Popen cannot handle unicode strings. :(
496         return map(self._encode_argument_if_needed, string_args)
497
498     def _needs_interpreter_check(self, argument):
499         return not argument.endswith(('perl', 'python', 'ruby', 'taskkill.exe', 'git', 'svn'))
500
501     # The only required argument to popen is named "args", the rest are optional keyword arguments.
502     def popen(self, args, **kwargs):
503         if sys.platform.startswith('win'):
504             _log.debug("Looking at {0}".format(args))
505             # Must include proper interpreter
506             if self._needs_interpreter_check(args[0]):
507                 try:
508                     with open(args[0], 'r') as f:
509                         line = f.readline()
510                         if "perl" in line:
511                             args.insert(0, "perl")
512                         elif "python" in line:
513                             args.insert(0, "python")
514                         elif "ruby" in line:
515                             args.insert(0, "ruby")
516                 except IOError:
517                     pass
518
519         # FIXME: We should always be stringifying the args, but callers who pass shell=True
520         # expect that the exact bytes passed will get passed to the shell (even if they're wrongly encoded).
521         # shell=True is wrong for many other reasons, and we should remove this
522         # hack as soon as we can fix all callers to not use shell=True.
523         if kwargs.get('shell') == True:
524             string_args = args
525         else:
526             string_args = self._stringify_args(args)
527         return subprocess.Popen(string_args, **kwargs)
528
529     def run_in_parallel(self, command_lines_and_cwds, processes=None):
530         """Runs a list of (cmd_line list, cwd string) tuples in parallel and returns a list of (retcode, stdout, stderr) tuples."""
531         assert len(command_lines_and_cwds)
532
533         if sys.platform == 'cygwin' or sys.platform.startswith('win'):
534             return map(_run_command_thunk, command_lines_and_cwds)
535         pool = multiprocessing.Pool(processes=processes)
536         results = pool.map(_run_command_thunk, command_lines_and_cwds)
537         pool.close()
538         pool.join()
539         return results
540
541
542 def _run_command_thunk(cmd_line_and_cwd):
543     # Note that this needs to be a bare module (and hence Picklable) method to work with multiprocessing.Pool.
544     (cmd_line, cwd) = cmd_line_and_cwd
545     proc = subprocess.Popen(cmd_line, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
546     stdout, stderr = proc.communicate()
547     return (proc.returncode, stdout, stderr)