1 # Copyright (c) 2009, Google Inc. All rights reserved.
2 # Copyright (c) 2009 Apple Inc. All rights reserved.
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
8 # * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 # * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
14 # * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 # Python module for interacting with an SCM system (like SVN or Git)
36 # Import WebKit-specific modules.
37 from modules.logging import error, log
39 def detect_scm_system(path):
40 if SVN.in_working_directory(path):
43 if Git.in_working_directory(path):
48 def first_non_empty_line_after_index(lines, index=0):
49 first_non_empty_line = index
50 for line in lines[index:]:
51 if re.match("^\s*$", line):
52 first_non_empty_line += 1
55 return first_non_empty_line
59 def __init__(self, message):
60 self.message_lines = message[first_non_empty_line_after_index(message, 0):]
62 def body(self, lstrip=False):
63 lines = self.message_lines[first_non_empty_line_after_index(self.message_lines, 1):]
65 lines = [line.lstrip() for line in lines]
66 return "\n".join(lines) + "\n"
68 def description(self, lstrip=False, strip_url=False):
69 line = self.message_lines[0]
73 line = re.sub("^(\s*)<.+> ", "\1", line)
77 return "\n".join(self.message_lines) + "\n"
80 class ScriptError(Exception):
81 def __init__(self, message=None, script_args=None, exit_code=None, output=None, cwd=None):
83 message = 'Failed to run "%s"' % script_args
85 message += " exit_code: %d" % exit_code
87 message += " cwd: %s" % cwd
89 Exception.__init__(self, message)
90 self.script_args = script_args # 'args' is already used by Exception
91 self.exit_code = exit_code
95 def message_with_output(self, output_limit=500):
97 if len(self.output) > output_limit:
98 return "%s\nLast %s characters of output:\n%s" % (self, output_limit, self.output[-output_limit:])
99 return "%s\n%s" % (self, self.output)
103 class CheckoutNeedsUpdate(ScriptError):
104 def __init__(self, script_args, exit_code, output, cwd):
105 ScriptError.__init__(self, script_args=script_args, exit_code=exit_code, output=output, cwd=cwd)
108 def default_error_handler(error):
111 def commit_error_handler(error):
112 if re.search("resource out of date", error.output):
113 raise CheckoutNeedsUpdate(script_args=error.script_args, exit_code=error.exit_code, output=error.output, cwd=error.cwd)
114 default_error_handler(error)
116 def ignore_error(error):
120 def __init__(self, cwd, dryrun=False):
122 self.checkout_root = self.find_checkout_root(self.cwd)
126 def run_command(args, cwd=None, input=None, error_handler=default_error_handler, return_exit_code=False, return_stderr=True):
127 if hasattr(input, 'read'): # Check if the input is a file.
129 string_to_communicate = None
131 stdin = subprocess.PIPE if input else None
132 string_to_communicate = input
134 stderr = subprocess.STDOUT
137 process = subprocess.Popen(args, stdin=stdin, stdout=subprocess.PIPE, stderr=stderr, cwd=cwd)
138 output = process.communicate(string_to_communicate)[0]
139 exit_code = process.wait()
141 script_error = ScriptError(script_args=args, exit_code=exit_code, output=output, cwd=cwd)
142 error_handler(script_error)
147 def scripts_directory(self):
148 return os.path.join(self.checkout_root, "WebKitTools", "Scripts")
150 def script_path(self, script_name):
151 return os.path.join(self.scripts_directory(), script_name)
153 def ensure_clean_working_directory(self, force_clean):
154 if not force_clean and not self.working_directory_is_clean():
155 print self.run_command(self.status_command(), error_handler=ignore_error)
156 raise ScriptError(message="Working directory has modifications, pass --force-clean or --no-clean to continue.")
158 log("Cleaning working directory")
159 self.clean_working_directory()
161 def ensure_no_local_commits(self, force):
162 if not self.supports_local_commits():
164 commits = self.local_commits()
168 error("Working directory has local commits, pass --force-clean to continue.")
169 self.discard_local_commits()
171 def apply_patch(self, patch, force=False):
172 # It's possible that the patch was not made from the root directory.
173 # We should detect and handle that case.
174 curl_process = subprocess.Popen(['curl', '--location', '--silent', '--show-error', patch['url']], stdout=subprocess.PIPE)
175 args = [self.script_path('svn-apply')]
176 if patch.get('reviewer'):
177 args += ['--reviewer', patch['reviewer']]
179 args.append('--force')
181 self.run_command(args, input=curl_process.stdout)
183 def run_status_and_extract_filenames(self, status_command, status_regexp):
185 for line in self.run_command(status_command).splitlines():
186 match = re.search(status_regexp, line)
189 # status = match.group('status')
190 filename = match.group('filename')
191 filenames.append(filename)
194 def strip_r_from_svn_revision(self, svn_revision):
195 match = re.match("^r(?P<svn_revision>\d+)", svn_revision)
197 return match.group('svn_revision')
200 def svn_revision_from_commit_text(self, commit_text):
201 match = re.search(self.commit_success_regexp(), commit_text, re.MULTILINE)
202 return match.group('svn_revision')
204 # ChangeLog-specific code doesn't really belong in scm.py, but this function is very useful.
205 def modified_changelogs(self):
207 paths = self.changed_files()
209 if os.path.basename(path) == "ChangeLog":
210 changelog_paths.append(path)
211 return changelog_paths
214 def in_working_directory(path):
215 raise NotImplementedError, "subclasses must implement"
218 def find_checkout_root(path):
219 raise NotImplementedError, "subclasses must implement"
222 def commit_success_regexp():
223 raise NotImplementedError, "subclasses must implement"
225 def working_directory_is_clean(self):
226 raise NotImplementedError, "subclasses must implement"
228 def clean_working_directory(self):
229 raise NotImplementedError, "subclasses must implement"
231 def update_webkit(self):
232 raise NotImplementedError, "subclasses must implement"
234 def status_command(self):
235 raise NotImplementedError, "subclasses must implement"
237 def changed_files(self):
238 raise NotImplementedError, "subclasses must implement"
240 def display_name(self):
241 raise NotImplementedError, "subclasses must implement"
243 def create_patch(self):
244 raise NotImplementedError, "subclasses must implement"
246 def diff_for_revision(self, revision):
247 raise NotImplementedError, "subclasses must implement"
249 def apply_reverse_diff(self, revision):
250 raise NotImplementedError, "subclasses must implement"
252 def revert_files(self, file_paths):
253 raise NotImplementedError, "subclasses must implement"
255 def commit_with_message(self, message):
256 raise NotImplementedError, "subclasses must implement"
258 def svn_commit_log(self, svn_revision):
259 raise NotImplementedError, "subclasses must implement"
261 def last_svn_commit_log(self):
262 raise NotImplementedError, "subclasses must implement"
264 # Subclasses must indicate if they support local commits,
265 # but the SCM baseclass will only call local_commits methods when this is true.
267 def supports_local_commits():
268 raise NotImplementedError, "subclasses must implement"
270 def create_patch_from_local_commit(self, commit_id):
271 error("Your source control manager does not support creating a patch from a local commit.")
273 def create_patch_since_local_commit(self, commit_id):
274 error("Your source control manager does not support creating a patch from a local commit.")
276 def commit_locally_with_message(self, message):
277 error("Your source control manager does not support local commits.")
279 def discard_local_commits(self):
282 def local_commits(self):
287 def __init__(self, cwd, dryrun=False):
288 SCM.__init__(self, cwd, dryrun)
289 self.cached_version = None
292 def in_working_directory(path):
293 return os.path.isdir(os.path.join(path, '.svn'))
296 def find_uuid(cls, path):
297 if not cls.in_working_directory(path):
299 return cls.value_from_svn_info(path, 'Repository UUID')
302 def value_from_svn_info(cls, path, field_name):
303 svn_info_args = ['svn', 'info', path]
304 info_output = cls.run_command(svn_info_args).rstrip()
305 match = re.search("^%s: (?P<value>.+)$" % field_name, info_output, re.MULTILINE)
307 raise ScriptError(script_args=svn_info_args, message='svn info did not contain a %s.' % field_name)
308 return match.group('value')
311 def find_checkout_root(path):
312 uuid = SVN.find_uuid(path)
313 # If |path| is not in a working directory, we're supposed to return |path|.
316 # Search up the directory hierarchy until we find a different UUID.
319 if uuid != SVN.find_uuid(path):
322 (path, last_component) = os.path.split(path)
323 if last_path == path:
327 def commit_success_regexp():
328 return "^Committed revision (?P<svn_revision>\d+)\.$"
330 def svn_version(self):
331 if not self.cached_version:
332 self.cached_version = self.run_command(['svn', '--version', '--quiet'])
334 return self.cached_version
336 def working_directory_is_clean(self):
337 return self.run_command(['svn', 'diff']) == ""
339 def clean_working_directory(self):
340 self.run_command(['svn', 'revert', '-R', '.'])
342 def update_webkit(self):
343 self.run_command(self.script_path("update-webkit"))
345 def status_command(self):
346 return ['svn', 'status']
348 def changed_files(self):
349 if self.svn_version() > "1.6":
350 status_regexp = "^(?P<status>[ACDMR]).{6} (?P<filename>.+)$"
352 status_regexp = "^(?P<status>[ACDMR]).{5} (?P<filename>.+)$"
353 return self.run_status_and_extract_filenames(self.status_command(), status_regexp)
356 def supports_local_commits():
359 def display_name(self):
362 def create_patch(self):
363 return self.run_command(self.script_path("svn-create-patch"), cwd=self.checkout_root, return_stderr=False)
365 def diff_for_revision(self, revision):
366 return self.run_command(['svn', 'diff', '-c', str(revision)])
368 def _repository_url(self):
369 return self.value_from_svn_info(self.checkout_root, 'URL')
371 def apply_reverse_diff(self, revision):
372 # '-c -revision' applies the inverse diff of 'revision'
373 svn_merge_args = ['svn', 'merge', '--non-interactive', '-c', '-%s' % revision, self._repository_url()]
374 log("WARNING: svn merge has been known to take more than 10 minutes to complete. It is recommended you use git for rollouts.")
375 log("Running '%s'" % " ".join(svn_merge_args))
376 self.run_command(svn_merge_args)
378 def revert_files(self, file_paths):
379 self.run_command(['svn', 'revert'] + file_paths)
381 def commit_with_message(self, message):
383 # Return a string which looks like a commit so that things which parse this output will succeed.
384 return "Dry run, no commit.\nCommitted revision 0."
385 return self.run_command(['svn', 'commit', '-m', message], error_handler=commit_error_handler)
387 def svn_commit_log(self, svn_revision):
388 svn_revision = self.strip_r_from_svn_revision(str(svn_revision))
389 return self.run_command(['svn', 'log', '--non-interactive', '--revision', svn_revision]);
391 def last_svn_commit_log(self):
392 # BASE is the checkout revision, HEAD is the remote repository revision
393 # http://svnbook.red-bean.com/en/1.0/ch03s03.html
394 return self.svn_commit_log('BASE')
396 # All git-specific logic should go here.
398 def __init__(self, cwd, dryrun=False):
399 SCM.__init__(self, cwd, dryrun)
402 def in_working_directory(cls, path):
403 return cls.run_command(['git', 'rev-parse', '--is-inside-work-tree'], cwd=path, error_handler=ignore_error).rstrip() == "true"
406 def find_checkout_root(cls, path):
407 # "git rev-parse --show-cdup" would be another way to get to the root
408 (checkout_root, dot_git) = os.path.split(cls.run_command(['git', 'rev-parse', '--git-dir'], cwd=path))
409 # If we were using 2.6 # checkout_root = os.path.relpath(checkout_root, path)
410 if not os.path.isabs(checkout_root): # Sometimes git returns relative paths
411 checkout_root = os.path.join(path, checkout_root)
415 def commit_success_regexp():
416 return "^Committed r(?P<svn_revision>\d+)$"
419 def discard_local_commits(self):
420 self.run_command(['git', 'reset', '--hard', 'trunk'])
422 def local_commits(self):
423 return self.run_command(['git', 'log', '--pretty=oneline', 'HEAD...trunk']).splitlines()
425 def rebase_in_progress(self):
426 return os.path.exists(os.path.join(self.checkout_root, '.git/rebase-apply'))
428 def working_directory_is_clean(self):
429 return self.run_command(['git', 'diff-index', 'HEAD']) == ""
431 def clean_working_directory(self):
432 # Could run git clean here too, but that wouldn't match working_directory_is_clean
433 self.run_command(['git', 'reset', '--hard', 'HEAD'])
434 # Aborting rebase even though this does not match working_directory_is_clean
435 if self.rebase_in_progress():
436 self.run_command(['git', 'rebase', '--abort'])
438 def update_webkit(self):
439 # FIXME: Call update-webkit once https://bugs.webkit.org/show_bug.cgi?id=27162 is fixed.
440 log("Updating working directory")
441 self.run_command(['git', 'svn', 'rebase'])
443 def status_command(self):
444 return ['git', 'status']
446 def changed_files(self):
447 status_command = ['git', 'diff', '-r', '--name-status', '-C', '-M', 'HEAD']
448 status_regexp = '^(?P<status>[ADM])\t(?P<filename>.+)$'
449 return self.run_status_and_extract_filenames(status_command, status_regexp)
452 def supports_local_commits():
455 def display_name(self):
458 def create_patch(self):
459 return self.run_command(['git', 'diff', '--binary', 'HEAD'])
462 def git_commit_from_svn_revision(cls, revision):
463 # git svn find-rev always exits 0, even when the revision is not found.
464 return cls.run_command(['git', 'svn', 'find-rev', 'r%s' % revision]).rstrip()
466 def diff_for_revision(self, revision):
467 git_commit = self.git_commit_from_svn_revision(revision)
468 return self.create_patch_from_local_commit(git_commit)
470 def apply_reverse_diff(self, revision):
471 # Assume the revision is an svn revision.
472 git_commit = self.git_commit_from_svn_revision(revision)
474 raise ScriptError(message='Failed to find git commit for revision %s, git svn log output: "%s"' % (revision, git_commit))
476 # I think this will always fail due to ChangeLogs.
477 # FIXME: We need to detec specific failure conditions and handle them.
478 self.run_command(['git', 'revert', '--no-commit', git_commit], error_handler=ignore_error)
480 # Fix any ChangeLogs if necessary.
481 changelog_paths = self.modified_changelogs()
482 if len(changelog_paths):
483 self.run_command([self.script_path('resolve-ChangeLogs')] + changelog_paths)
485 def revert_files(self, file_paths):
486 self.run_command(['git', 'checkout', 'HEAD'] + file_paths)
488 def commit_with_message(self, message):
489 self.commit_locally_with_message(message)
490 return self.push_local_commits_to_server()
492 def svn_commit_log(self, svn_revision):
493 svn_revision = self.strip_r_from_svn_revision(svn_revision)
494 return self.run_command(['git', 'svn', 'log', '-r', svn_revision])
496 def last_svn_commit_log(self):
497 return self.run_command(['git', 'svn', 'log', '--limit=1'])
499 # Git-specific methods:
501 def create_patch_from_local_commit(self, commit_id):
502 return self.run_command(['git', 'diff', '--binary', commit_id + "^.." + commit_id])
504 def create_patch_since_local_commit(self, commit_id):
505 return self.run_command(['git', 'diff', '--binary', commit_id])
507 def commit_locally_with_message(self, message):
508 self.run_command(['git', 'commit', '--all', '-F', '-'], input=message)
510 def push_local_commits_to_server(self):
512 # Return a string which looks like a commit so that things which parse this output will succeed.
513 return "Dry run, no remote commit.\nCommitted r0"
514 return self.run_command(['git', 'svn', 'dcommit'], error_handler=commit_error_handler)
516 # This function supports the following argument formats:
517 # no args : rev-list trunk..HEAD
518 # A..B : rev-list A..B
520 # A B : [A, B] (different from git diff, which would use "rev-list A..B")
521 def commit_ids_from_commitish_arguments(self, args):
523 # FIXME: trunk is not always the remote branch name, need a way to detect the name.
524 args.append('trunk..HEAD')
527 for commitish in args:
528 if '...' in commitish:
529 raise ScriptError(message="'...' is not supported (found in '%s'). Did you mean '..'?" % commitish)
530 elif '..' in commitish:
531 commit_ids += reversed(self.run_command(['git', 'rev-list', commitish]).splitlines())
533 # Turn single commits or branch or tag names into commit ids.
534 commit_ids += self.run_command(['git', 'rev-parse', '--revs-only', commitish]).splitlines()
537 def commit_message_for_local_commit(self, commit_id):
538 commit_lines = self.run_command(['git', 'cat-file', 'commit', commit_id]).splitlines()
540 # Skip the git headers.
541 first_line_after_headers = 0
542 for line in commit_lines:
543 first_line_after_headers += 1
546 return CommitMessage(commit_lines[first_line_after_headers:])
548 def files_changed_summary_for_commit(self, commit_id):
549 return self.run_command(['git', 'diff-tree', '--shortstat', '--no-commit-id', commit_id])