82cf2d7476047e33e5887bc75956f0c4faba6e3d
[WebKit-https.git] / Tools / Scripts / webkitpy / common / system / executive.py
1 # Copyright (c) 2009, Google Inc. All rights reserved.
2 # Copyright (c) 2009 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import StringIO
31 import errno
32 import logging
33 import multiprocessing
34 import os
35 import signal
36 import subprocess
37 import sys
38 import time
39
40 from webkitpy.common.system.outputtee import Tee
41 from webkitpy.common.system.filesystem import FileSystem
42
43
44 _log = logging.getLogger(__name__)
45
46
47 class ScriptError(Exception):
48
49     def __init__(self,
50                  message=None,
51                  script_args=None,
52                  exit_code=None,
53                  output=None,
54                  cwd=None):
55         if not message:
56             message = 'Failed to run "%s"' % repr(script_args)
57             if exit_code:
58                 message += " exit_code: %d" % exit_code
59             if cwd:
60                 message += " cwd: %s" % cwd
61
62         Exception.__init__(self, message)
63         self.script_args = script_args  # 'args' is already used by Exception
64         self.exit_code = exit_code
65         self.output = output
66         self.cwd = cwd
67
68     def message_with_output(self, output_limit=500):
69         if self.output:
70             if output_limit and len(self.output) > output_limit:
71                 return u"%s\n\nLast %s characters of output:\n%s" % \
72                     (self, output_limit, self.output[-output_limit:])
73             return u"%s\n\n%s" % (self, self.output)
74         return unicode(self)
75
76     def command_name(self):
77         command_path = self.script_args
78         if type(command_path) is list:
79             command_path = command_path[0]
80         return os.path.basename(command_path)
81
82
83 class Executive(object):
84     PIPE = subprocess.PIPE
85     STDOUT = subprocess.STDOUT
86
87     def __init__(self):
88         self.pid_to_system_pid = {}
89
90     def _should_close_fds(self):
91         # We need to pass close_fds=True to work around Python bug #2320
92         # (otherwise we can hang when we kill DumpRenderTree when we are running
93         # multiple threads). See http://bugs.python.org/issue2320 .
94         # Note that close_fds isn't supported on Windows, but this bug only
95         # shows up on Mac and Linux.
96         return not (sys.platform == 'cygwin' or sys.platform.startswith('win'))
97
98     def _run_command_with_teed_output(self, args, teed_output, **kwargs):
99         child_process = self.popen(args,
100                                    stdout=self.PIPE,
101                                    stderr=self.STDOUT,
102                                    close_fds=self._should_close_fds(),
103                                    **kwargs)
104
105         # Use our own custom wait loop because Popen ignores a tee'd
106         # stderr/stdout.
107         # FIXME: This could be improved not to flatten output to stdout.
108         while True:
109             output_line = child_process.stdout.readline()
110             if output_line == "" and child_process.poll() != None:
111                 # poll() is not threadsafe and can throw OSError due to:
112                 # http://bugs.python.org/issue1731717
113                 return child_process.poll()
114             # We assume that the child process wrote to us in utf-8,
115             # so no re-encoding is necessary before writing here.
116             teed_output.write(output_line)
117
118     # FIXME: Remove this deprecated method and move callers to run_command.
119     # FIXME: This method is a hack to allow running command which both
120     # capture their output and print out to stdin.  Useful for things
121     # like "build-webkit" where we want to display to the user that we're building
122     # but still have the output to stuff into a log file.
123     def run_and_throw_if_fail(self, args, quiet=False, decode_output=True, **kwargs):
124         # Cache the child's output locally so it can be used for error reports.
125         child_out_file = StringIO.StringIO()
126         tee_stdout = sys.stdout
127         if quiet:
128             dev_null = open(os.devnull, "w")  # FIXME: Does this need an encoding?
129             tee_stdout = dev_null
130         child_stdout = Tee(child_out_file, tee_stdout)
131         exit_code = self._run_command_with_teed_output(args, child_stdout, **kwargs)
132         if quiet:
133             dev_null.close()
134
135         child_output = child_out_file.getvalue()
136         child_out_file.close()
137
138         if decode_output:
139             child_output = child_output.decode(self._child_process_encoding())
140
141         if exit_code:
142             raise ScriptError(script_args=args,
143                               exit_code=exit_code,
144                               output=child_output)
145         return child_output
146
147     def cpu_count(self):
148         try:
149             cpus = int(os.environ.get('NUMBER_OF_PROCESSORS'))
150             if cpus > 0:
151                 return cpus
152         except (ValueError, TypeError):
153             pass
154         return multiprocessing.cpu_count()
155
156     @staticmethod
157     def interpreter_for_script(script_path, fs=None):
158         fs = fs or FileSystem()
159         lines = fs.read_text_file(script_path).splitlines()
160         if not len(lines):
161             return None
162         first_line = lines[0]
163         if not first_line.startswith('#!'):
164             return None
165         if first_line.find('python') > -1:
166             return sys.executable
167         if first_line.find('perl') > -1:
168             return 'perl'
169         if first_line.find('ruby') > -1:
170             return 'ruby'
171         return None
172
173     @staticmethod
174     def shell_command_for_script(script_path, fs=None):
175         fs = fs or FileSystem()
176         # Win32 does not support shebang. We need to detect the interpreter ourself.
177         if sys.platform.startswith('win'):
178             interpreter = Executive.interpreter_for_script(script_path, fs)
179             if interpreter:
180                 return [interpreter, script_path]
181         return [script_path]
182
183     def kill_process(self, pid):
184         """Attempts to kill the given pid.
185         Will fail silently if pid does not exist or insufficient permisssions."""
186         if sys.platform.startswith('win32'):
187             # We only use taskkill.exe on windows (not cygwin) because subprocess.pid
188             # is a CYGWIN pid and taskkill.exe expects a windows pid.
189             # Thankfully os.kill on CYGWIN handles either pid type.
190             task_kill_executable = os.path.join('C:', os.sep, 'WINDOWS', 'system32', 'taskkill.exe')
191             command = [task_kill_executable, "/f", "/t", "/pid", pid]
192             # taskkill will exit 128 if the process is not found.  We should log.
193             self.run_command(command, error_handler=self.ignore_error)
194             return
195
196         # According to http://docs.python.org/library/os.html
197         # os.kill isn't available on Windows. python 2.5.5 os.kill appears
198         # to work in cygwin, however it occasionally raises EAGAIN.
199         retries_left = 10 if sys.platform == "cygwin" else 1
200         while retries_left > 0:
201             try:
202                 retries_left -= 1
203                 # Give processes one change to clean up quickly before exiting.
204                 # Following up with a kill should have no effect if the process
205                 # already exited, and forcefully kill it if SIGTERM wasn't enough.
206                 os.kill(pid, signal.SIGTERM)
207                 os.kill(pid, signal.SIGKILL)
208             except OSError, e:
209                 if e.errno == errno.EAGAIN:
210                     if retries_left <= 0:
211                         _log.warn("Failed to kill pid %s.  Too many EAGAIN errors." % pid)
212                     continue
213                 if e.errno == errno.ESRCH:  # The process does not exist.
214                     return
215                 if e.errno == errno.EPIPE:  # The process has exited already on cygwin
216                     return
217                 if e.errno == errno.ECHILD:
218                     # Can't wait on a non-child process, but the kill worked.
219                     return
220                 if e.errno == errno.EACCES and sys.platform == 'cygwin':
221                     # Cygwin python sometimes can't kill native processes.
222                     return
223                 raise
224
225     def _win32_check_running_pid(self, pid):
226         # importing ctypes at the top-level seems to cause weird crashes at
227         # exit under cygwin on apple's win port. Only win32 needs cygwin, so
228         # we import it here instead. See https://bugs.webkit.org/show_bug.cgi?id=91682
229         import ctypes
230
231         class PROCESSENTRY32(ctypes.Structure):
232             _fields_ = [("dwSize", ctypes.c_ulong),
233                         ("cntUsage", ctypes.c_ulong),
234                         ("th32ProcessID", ctypes.c_ulong),
235                         ("th32DefaultHeapID", ctypes.POINTER(ctypes.c_ulong)),
236                         ("th32ModuleID", ctypes.c_ulong),
237                         ("cntThreads", ctypes.c_ulong),
238                         ("th32ParentProcessID", ctypes.c_ulong),
239                         ("pcPriClassBase", ctypes.c_ulong),
240                         ("dwFlags", ctypes.c_ulong),
241                         ("szExeFile", ctypes.c_char * 260)]
242
243         CreateToolhelp32Snapshot = ctypes.windll.kernel32.CreateToolhelp32Snapshot
244         Process32First = ctypes.windll.kernel32.Process32First
245         Process32Next = ctypes.windll.kernel32.Process32Next
246         CloseHandle = ctypes.windll.kernel32.CloseHandle
247         TH32CS_SNAPPROCESS = 0x00000002  # win32 magic number
248         hProcessSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0)
249         pe32 = PROCESSENTRY32()
250         pe32.dwSize = ctypes.sizeof(PROCESSENTRY32)
251         result = False
252         if not Process32First(hProcessSnap, ctypes.byref(pe32)):
253             _log.debug("Failed getting first process.")
254             CloseHandle(hProcessSnap)
255             return result
256         while True:
257             if pe32.th32ProcessID == pid:
258                 result = True
259                 break
260             if not Process32Next(hProcessSnap, ctypes.byref(pe32)):
261                 break
262         CloseHandle(hProcessSnap)
263         return result
264
265     def check_running_pid(self, pid):
266         """Return True if pid is alive, otherwise return False."""
267         if sys.platform.startswith('win'):
268             return self._win32_check_running_pid(pid)
269
270         try:
271             os.kill(pid, 0)
272             return True
273         except OSError:
274             return False
275
276     def running_pids(self, process_name_filter=None):
277         if sys.platform.startswith('win'):
278             # FIXME: running_pids isn't implemented on native Windows yet...
279             return []
280
281         if not process_name_filter:
282             process_name_filter = lambda process_name: True
283
284         running_pids = []
285         if sys.platform in ("cygwin"):
286             ps_process = self.run_command(['ps', '-e'], error_handler=Executive.ignore_error)
287             for line in ps_process.splitlines():
288                 tokens = line.strip().split()
289                 try:
290                     pid, ppid, pgid, winpid, tty, uid, stime, process_name = tokens
291                     if process_name_filter(process_name):
292                         running_pids.append(int(pid))
293                         self.pid_to_system_pid[int(pid)] = int(winpid)
294                 except ValueError, e:
295                     pass
296         else:
297             ps_process = self.popen(['ps', '-eo', 'pid,comm'], stdout=self.PIPE, stderr=self.PIPE)
298             stdout, _ = ps_process.communicate()
299             for line in stdout.splitlines():
300                 try:
301                     # In some cases the line can contain one or more
302                     # leading white-spaces, so strip it before split.
303                     pid, process_name = line.strip().split(' ', 1)
304                     if process_name_filter(process_name):
305                         running_pids.append(int(pid))
306                 except ValueError, e:
307                     pass
308
309         return sorted(running_pids)
310
311     def wait_newest(self, process_name_filter=None):
312         if not process_name_filter:
313             process_name_filter = lambda process_name: True
314
315         running_pids = self.running_pids(process_name_filter)
316         if not running_pids:
317             return
318         pid = running_pids[-1]
319
320         while self.check_running_pid(pid):
321             time.sleep(0.25)
322
323     def wait_limited(self, pid, limit_in_seconds=None, check_frequency_in_seconds=None):
324         seconds_left = limit_in_seconds or 10
325         sleep_length = check_frequency_in_seconds or 1
326         while seconds_left > 0 and self.check_running_pid(pid):
327             seconds_left -= sleep_length
328             time.sleep(sleep_length)
329
330     def _windows_image_name(self, process_name):
331         name, extension = os.path.splitext(process_name)
332         if not extension:
333             # taskkill expects processes to end in .exe
334             # If necessary we could add a flag to disable appending .exe.
335             process_name = "%s.exe" % name
336         return process_name
337
338     def interrupt(self, pid):
339         interrupt_signal = signal.SIGINT
340         # FIXME: The python docs seem to imply that platform == 'win32' may need to use signal.CTRL_C_EVENT
341         # http://docs.python.org/2/library/signal.html
342         try:
343             os.kill(pid, interrupt_signal)
344         except OSError:
345             # Silently ignore when the pid doesn't exist.
346             # It's impossible for callers to avoid race conditions with process shutdown.
347             pass
348
349     def kill_all(self, process_name):
350         """Attempts to kill processes matching process_name.
351         Will fail silently if no process are found."""
352         if sys.platform == 'cygwin' or sys.platform.startswith('win'):
353             image_name = self._windows_image_name(process_name)
354             killCommmand = 'taskkill.exe'
355             if sys.platform.startswith('win'):
356                 killCommand = os.path.join('C:', os.sep, 'WINDOWS', 'system32', 'taskkill.exe')
357             command = [killCommmand, "/f", "/im", image_name]
358             # taskkill will exit 128 if the process is not found.  We should log.
359             self.run_command(command, error_handler=self.ignore_error)
360             return
361
362         # FIXME: This is inconsistent that kill_all uses TERM and kill_process
363         # uses KILL.  Windows is always using /f (which seems like -KILL).
364         # We should pick one mode, or add support for switching between them.
365         # Note: Mac OS X 10.6 requires -SIGNALNAME before -u USER
366         command = ["killall", "-TERM", "-u", os.getenv("USER"), process_name]
367         # killall returns 1 if no process can be found and 2 on command error.
368         # FIXME: We should pass a custom error_handler to allow only exit_code 1.
369         # We should log in exit_code == 1
370         self.run_command(command, error_handler=self.ignore_error)
371
372     # Error handlers do not need to be static methods once all callers are
373     # updated to use an Executive object.
374
375     @staticmethod
376     def default_error_handler(error):
377         raise error
378
379     @staticmethod
380     def ignore_error(error):
381         pass
382
383     def _compute_stdin(self, input):
384         """Returns (stdin, string_to_communicate)"""
385         # FIXME: We should be returning /dev/null for stdin
386         # or closing stdin after process creation to prevent
387         # child processes from getting input from the user.
388         if not input:
389             return (None, None)
390         if hasattr(input, "read"):  # Check if the input is a file.
391             return (input, None)  # Assume the file is in the right encoding.
392
393         # Popen in Python 2.5 and before does not automatically encode unicode objects.
394         # http://bugs.python.org/issue5290
395         # See https://bugs.webkit.org/show_bug.cgi?id=37528
396         # for an example of a regresion caused by passing a unicode string directly.
397         # FIXME: We may need to encode differently on different platforms.
398         if isinstance(input, unicode):
399             input = input.encode(self._child_process_encoding())
400         return (self.PIPE, input)
401
402     def command_for_printing(self, args):
403         """Returns a print-ready string representing command args.
404         The string should be copy/paste ready for execution in a shell."""
405         args = self._stringify_args(args)
406         escaped_args = []
407         for arg in args:
408             if isinstance(arg, unicode):
409                 # Escape any non-ascii characters for easy copy/paste
410                 arg = arg.encode("unicode_escape")
411             # FIXME: Do we need to fix quotes here?
412             escaped_args.append(arg)
413         return " ".join(escaped_args)
414
415     # FIXME: run_and_throw_if_fail should be merged into this method.
416     def run_command(self,
417                     args,
418                     cwd=None,
419                     env=None,
420                     input=None,
421                     error_handler=None,
422                     return_exit_code=False,
423                     return_stderr=True,
424                     decode_output=True):
425         """Popen wrapper for convenience and to work around python bugs."""
426         assert(isinstance(args, list) or isinstance(args, tuple))
427         start_time = time.time()
428
429         stdin, string_to_communicate = self._compute_stdin(input)
430         stderr = self.STDOUT if return_stderr else None
431
432         process = self.popen(args,
433                              stdin=stdin,
434                              stdout=self.PIPE,
435                              stderr=stderr,
436                              cwd=cwd,
437                              env=env,
438                              close_fds=self._should_close_fds())
439         output = process.communicate(string_to_communicate)[0]
440
441         # run_command automatically decodes to unicode() unless explicitly told not to.
442         if decode_output:
443             output = output.decode(self._child_process_encoding())
444
445         # wait() is not threadsafe and can throw OSError due to:
446         # http://bugs.python.org/issue1731717
447         exit_code = process.wait()
448
449         _log.debug('"%s" took %.2fs' % (self.command_for_printing(args), time.time() - start_time))
450
451         if return_exit_code:
452             return exit_code
453
454         if exit_code:
455             script_error = ScriptError(script_args=args,
456                                        exit_code=exit_code,
457                                        output=output,
458                                        cwd=cwd)
459             (error_handler or self.default_error_handler)(script_error)
460         return output
461
462     def _child_process_encoding(self):
463         # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW
464         # to launch subprocesses, so we have to encode arguments using the
465         # current code page.
466         if sys.platform.startswith('win') and sys.version < '3':
467             return 'mbcs'
468         # All other platforms use UTF-8.
469         # FIXME: Using UTF-8 on Cygwin will confuse Windows-native commands
470         # which will expect arguments to be encoded using the current code
471         # page.
472         return 'utf-8'
473
474     def _should_encode_child_process_arguments(self):
475         # Cygwin's Python's os.execv doesn't support unicode command
476         # arguments, and neither does Cygwin's execv itself.
477         if sys.platform == 'cygwin':
478             return True
479
480         # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW
481         # to launch subprocesses, so we have to encode arguments using the
482         # current code page.
483         if sys.platform.startswith('win') and sys.version < '3':
484             return True
485
486         return False
487
488     def _encode_argument_if_needed(self, argument):
489         if not self._should_encode_child_process_arguments():
490             return argument
491         return argument.encode(self._child_process_encoding())
492
493     def _stringify_args(self, args):
494         # Popen will throw an exception if args are non-strings (like int())
495         string_args = map(unicode, args)
496         # The Windows implementation of Popen cannot handle unicode strings. :(
497         return map(self._encode_argument_if_needed, string_args)
498
499     def _needs_interpreter_check(self, argument):
500         return not argument.endswith(('perl', 'python', 'ruby', 'taskkill.exe', 'git', 'svn'))
501
502     # The only required argument to popen is named "args", the rest are optional keyword arguments.
503     def popen(self, args, **kwargs):
504         if sys.platform.startswith('win'):
505             _log.debug("Looking at {0}".format(args))
506             # Must include proper interpreter
507             if self._needs_interpreter_check(args[0]):
508                 try:
509                     with open(args[0], 'r') as f:
510                         line = f.readline()
511                         if "perl" in line:
512                             args.insert(0, "perl")
513                         elif "python" in line:
514                             args.insert(0, "python")
515                         elif "ruby" in line:
516                             args.insert(0, "ruby")
517                 except IOError:
518                     pass
519
520         # FIXME: We should always be stringifying the args, but callers who pass shell=True
521         # expect that the exact bytes passed will get passed to the shell (even if they're wrongly encoded).
522         # shell=True is wrong for many other reasons, and we should remove this
523         # hack as soon as we can fix all callers to not use shell=True.
524         if kwargs.get('shell') == True:
525             string_args = args
526         else:
527             string_args = self._stringify_args(args)
528         return subprocess.Popen(string_args, **kwargs)
529
530     def run_in_parallel(self, command_lines_and_cwds, processes=None):
531         """Runs a list of (cmd_line list, cwd string) tuples in parallel and returns a list of (retcode, stdout, stderr) tuples."""
532         assert len(command_lines_and_cwds)
533
534         if sys.platform == 'cygwin' or sys.platform.startswith('win'):
535             return map(_run_command_thunk, command_lines_and_cwds)
536         pool = multiprocessing.Pool(processes=processes)
537         results = pool.map(_run_command_thunk, command_lines_and_cwds)
538         pool.close()
539         pool.join()
540         return results
541
542
543 def _run_command_thunk(cmd_line_and_cwd):
544     # Note that this needs to be a bare module (and hence Picklable) method to work with multiprocessing.Pool.
545     (cmd_line, cwd) = cmd_line_and_cwd
546     proc = subprocess.Popen(cmd_line, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
547     stdout, stderr = proc.communicate()
548     return (proc.returncode, stdout, stderr)