1 # Copyright (c) 2009, Google Inc. All rights reserved.
2 # Copyright (c) 2009 Apple Inc. All rights reserved.
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
8 # * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 # * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
14 # * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 # WebKit's Python module for interacting with Bugzilla
38 from datetime import datetime # used in timestamp()
40 # Import WebKit-specific modules.
41 from modules.logging import error, log
42 from modules.committers import CommitterList
44 # WebKit includes a built copy of BeautifulSoup in Scripts/modules
45 # so this import should always succeed.
46 from .BeautifulSoup import BeautifulSoup
49 from mechanize import Browser
50 except ImportError, e:
52 mechanize is required.
55 sudo easy_install mechanize
58 http://wwwsearch.sourceforge.net/mechanize/
62 def credentials_from_git():
63 return [read_config("username"), read_config("password")]
65 def credentials_from_keychain(username=None):
67 return [username, None]
69 command = "/usr/bin/security %s -g -s %s" % ("find-internet-password", Bugzilla.bug_server_host)
71 command += " -a %s" % username
73 log('Reading Keychain for %s account and password. Click "Allow" to continue...' % Bugzilla.bug_server_host)
74 keychain_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
75 value = keychain_process.communicate()[0]
76 exit_code = keychain_process.wait()
79 return [username, None]
81 match = re.search('^\s*"acct"<blob>="(?P<username>.+)"', value, re.MULTILINE)
83 username = match.group('username')
86 match = re.search('^password: "(?P<password>.+)"', value, re.MULTILINE)
88 password = match.group('password')
90 return [username, password]
93 return platform.mac_ver()[0]
95 def parse_bug_id(message):
96 match = re.search("http\://webkit\.org/b/(?P<bug_id>\d+)", message)
98 return match.group('bug_id')
99 match = re.search(Bugzilla.bug_server_regex + "show_bug\.cgi\?id=(?P<bug_id>\d+)", message)
101 return match.group('bug_id')
104 # FIXME: This should not depend on git for config storage
105 def read_config(key):
106 # Need a way to read from svn too
107 config_process = subprocess.Popen("git config --get bugzilla.%s" % key, stdout=subprocess.PIPE, shell=True)
108 value = config_process.communicate()[0]
109 return_code = config_process.wait()
113 return value.rstrip('\n')
115 def read_credentials():
116 (username, password) = credentials_from_git()
118 if not username or not password:
119 (username, password) = credentials_from_keychain(username)
122 username = raw_input("Bugzilla login: ")
124 password = getpass.getpass("Bugzilla password for %s: " % username)
126 return [username, password]
129 return datetime.now().strftime("%Y%m%d%H%M%S")
132 class BugzillaError(Exception):
137 def __init__(self, dryrun=False, committers=CommitterList()):
139 self.authenticated = False
141 self.browser = Browser()
142 # Ignore bugs.webkit.org/robots.txt until we fix it to allow this script
143 self.browser.set_handle_robots(False)
144 self.committers = committers
146 # Defaults (until we support better option parsing):
147 bug_server_host = "bugs.webkit.org"
148 bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
149 bug_server_url = "https://%s/" % bug_server_host
151 def bug_url_for_bug_id(self, bug_id, xml=False):
152 content_type = "&ctype=xml" if xml else ""
153 return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type)
155 def short_bug_url_for_bug_id(self, bug_id):
156 return "http://webkit.org/b/%s" % bug_id
158 def attachment_url_for_id(self, attachment_id, action="view"):
160 if action and action != "view":
161 action_param = "&action=%s" % action
162 return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param)
164 def _parse_attachment_flag(self, element, flag_name, attachment, result_key):
165 flag = element.find('flag', attrs={'name' : flag_name})
167 attachment[flag_name] = flag['status']
168 if flag['status'] == '+':
169 attachment[result_key] = flag['setter']
171 def _parse_attachment_element(self, element, bug_id):
173 attachment['bug_id'] = bug_id
174 attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
175 attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
176 attachment['id'] = str(element.find('attachid').string)
177 attachment['url'] = self.attachment_url_for_id(attachment['id'])
178 attachment['name'] = unicode(element.find('desc').string)
179 attachment['type'] = str(element.find('type').string)
180 self._parse_attachment_flag(element, 'review', attachment, 'reviewer_email')
181 self._parse_attachment_flag(element, 'commit-queue', attachment, 'committer_email')
184 def fetch_attachments_from_bug(self, bug_id):
185 bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
186 log("Fetching: %s" % bug_url)
188 page = urllib2.urlopen(bug_url)
189 soup = BeautifulSoup(page)
192 for element in soup.findAll('attachment'):
193 attachment = self._parse_attachment_element(element, bug_id)
194 attachments.append(attachment)
197 def _parse_bug_id_from_attachment_page(self, page):
198 up_link = BeautifulSoup(page).find('link', rel='Up') # The "Up" relation happens to point to the bug.
200 return None # This attachment does not exist (or you don't have permissions to view it).
201 match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href'])
202 return int(match.group('bug_id'))
204 def bug_id_for_attachment_id(self, attachment_id):
205 attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
206 log("Fetching: %s" % attachment_url)
207 page = urllib2.urlopen(attachment_url)
208 return self._parse_bug_id_from_attachment_page(page)
210 # This should really return an Attachment object
211 # which can lazily fetch any missing data.
212 def fetch_attachment(self, attachment_id):
213 # We could grab all the attachment details off of the attachment edit page
214 # but we already have working code to do so off of the bugs page, so re-use that.
215 bug_id = self.bug_id_for_attachment_id(attachment_id)
218 attachments = self.fetch_attachments_from_bug(bug_id)
219 for attachment in attachments:
220 if attachment['id'] == attachment_id:
221 self._validate_committer_and_reviewer(attachment)
223 return None # This should never be hit.
225 def fetch_title_from_bug(self, bug_id):
226 bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
227 page = urllib2.urlopen(bug_url)
228 soup = BeautifulSoup(page)
229 return soup.find('short_desc').string
231 def fetch_patches_from_bug(self, bug_id):
233 for attachment in self.fetch_attachments_from_bug(bug_id):
234 if attachment['is_patch'] and not attachment['is_obsolete']:
235 patches.append(attachment)
238 # _view_source_link belongs in some sort of webkit_config.py module.
239 def _view_source_link(self, local_path):
240 return "http://trac.webkit.org/browser/trunk/%s" % local_path
242 def _validate_setter_email(self, patch, result_key, lookup_function, rejection_function, reject_invalid_patches):
243 setter_email = patch.get(result_key + '_email')
247 committer = lookup_function(setter_email)
249 patch[result_key] = committer.full_name
250 return patch[result_key]
252 if reject_invalid_patches:
253 committer_list = "WebKitTools/Scripts/modules/committers.py"
254 failure_message = "%s does not have %s permissions according to %s." % (setter_email, result_key, self._view_source_link(committer_list))
255 rejection_function(patch['id'], failure_message)
257 log("Warning, attachment %s on bug %s has invalid %s (%s)" % (patch['id'], patch['bug_id'], result_key, setter_email))
260 def _validate_reviewer(self, patch, reject_invalid_patches):
261 return self._validate_setter_email(patch, 'reviewer', self.committers.reviewer_by_email, self.reject_patch_from_review_queue, reject_invalid_patches)
263 def _validate_committer(self, patch, reject_invalid_patches):
264 return self._validate_setter_email(patch, 'committer', self.committers.committer_by_email, self.reject_patch_from_commit_queue, reject_invalid_patches)
266 # FIXME: This is a hack until we have a real Attachment object.
267 # _validate_committer and _validate_reviewer fill in the 'reviewer' and 'committer'
268 # keys which other parts of the code expect to be filled in.
269 def _validate_committer_and_reviewer(self, patch):
270 self._validate_reviewer(patch, reject_invalid_patches=False)
271 self._validate_committer(patch, reject_invalid_patches=False)
273 def fetch_unreviewed_patches_from_bug(self, bug_id):
274 unreviewed_patches = []
275 for attachment in self.fetch_attachments_from_bug(bug_id):
276 if attachment.get('review') == '?' and not attachment['is_obsolete']:
277 unreviewed_patches.append(attachment)
278 return unreviewed_patches
280 def fetch_reviewed_patches_from_bug(self, bug_id, reject_invalid_patches=False):
281 reviewed_patches = []
282 for attachment in self.fetch_attachments_from_bug(bug_id):
283 if self._validate_reviewer(attachment, reject_invalid_patches) and not attachment['is_obsolete']:
284 reviewed_patches.append(attachment)
285 return reviewed_patches
287 def fetch_commit_queue_patches_from_bug(self, bug_id, reject_invalid_patches=False):
288 commit_queue_patches = []
289 for attachment in self.fetch_reviewed_patches_from_bug(bug_id, reject_invalid_patches):
290 if self._validate_committer(attachment, reject_invalid_patches) and not attachment['is_obsolete']:
291 commit_queue_patches.append(attachment)
292 return commit_queue_patches
294 def _fetch_bug_ids_advanced_query(self, query):
295 page = urllib2.urlopen(query)
296 soup = BeautifulSoup(page)
299 # Grab the cells in the first column (which happens to be the bug ids)
300 for bug_link_cell in soup('td', "first-child"): # tds with the class "first-child"
301 bug_link = bug_link_cell.find("a")
302 bug_ids.append(bug_link.string) # the contents happen to be the bug id
306 def fetch_bug_ids_from_commit_queue(self):
307 commit_queue_url = self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=commit-queue%2B"
308 return self._fetch_bug_ids_advanced_query(commit_queue_url)
310 def fetch_bug_ids_from_review_queue(self):
311 review_queue_url = self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review?"
312 return self._fetch_bug_ids_advanced_query(review_queue_url)
314 def fetch_patches_from_commit_queue(self, reject_invalid_patches=False):
316 for bug_id in self.fetch_bug_ids_from_commit_queue():
317 patches = self.fetch_commit_queue_patches_from_bug(bug_id, reject_invalid_patches)
318 patches_to_land += patches
319 return patches_to_land
321 def fetch_patches_from_review_queue(self, limit):
322 patches_to_review = []
323 for bug_id in self.fetch_bug_ids_from_review_queue():
324 if len(patches_to_review) >= limit:
326 patches = self.fetch_unreviewed_patches_from_bug(bug_id)
327 patches_to_review += patches
328 return patches_to_review
330 def authenticate(self):
331 if self.authenticated:
335 log("Skipping log in for dry run...")
336 self.authenticated = True
339 (username, password) = read_credentials()
341 log("Logging in as %s..." % username)
342 self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1")
343 self.browser.select_form(name="login")
344 self.browser['Bugzilla_login'] = username
345 self.browser['Bugzilla_password'] = password
346 response = self.browser.submit()
348 match = re.search("<title>(.+?)</title>", response.read())
349 # If the resulting page has a title, and it contains the word "invalid" assume it's the login failure page.
350 if match and re.search("Invalid", match.group(1), re.IGNORECASE):
351 # FIXME: We could add the ability to try again on failure.
352 raise BugzillaError("Bugzilla login failed: %s" % match.group(1))
354 self.authenticated = True
356 def _fill_attachment_form(self, description, patch_file_object, comment_text=None, mark_for_review=False, mark_for_commit_queue=False, bug_id=None):
357 self.browser['description'] = description
358 self.browser['ispatch'] = ("1",)
359 self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
360 self.browser['flag_type-3'] = ('?',) if mark_for_commit_queue else ('X',)
362 patch_name = "bug-%s-%s.patch" % (bug_id, timestamp())
364 patch_name ="%s.patch" % timestamp()
365 self.browser.add_file(patch_file_object, "text/plain", patch_name, 'data')
367 def add_patch_to_bug(self, bug_id, patch_file_object, description, comment_text=None, mark_for_review=False, mark_for_commit_queue=False):
370 log('Adding patch "%s" to bug %s' % (description, bug_id))
375 self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % (self.bug_server_url, bug_id))
376 self.browser.select_form(name="entryform")
377 self._fill_attachment_form(description, patch_file_object, mark_for_review=mark_for_review, mark_for_commit_queue=mark_for_commit_queue, bug_id=bug_id)
380 self.browser['comment'] = comment_text
381 self.browser.submit()
383 def prompt_for_component(self, components):
384 log("Please pick a component:")
386 for name in components:
388 log("%2d. %s" % (i, name))
389 result = int(raw_input("Enter a number: ")) - 1
390 return components[result]
392 def _check_create_bug_response(self, response_html):
393 match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>", response_html)
395 return match.group('bug_id')
397 match = re.search('<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">', response_html, re.DOTALL)
398 error_message = "FAIL"
400 text_lines = BeautifulSoup(match.group('error_message')).findAll(text=True)
401 error_message = "\n" + '\n'.join([" " + line.strip() for line in text_lines if line.strip()])
402 raise BugzillaError("Bug not created: %s" % error_message)
404 def create_bug_with_patch(self, bug_title, bug_description, component, patch_file_object, patch_description, cc, mark_for_review=False, mark_for_commit_queue=False):
407 log('Creating bug with patch description "%s"' % patch_description)
412 self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
413 self.browser.select_form(name="Create")
414 component_items = self.browser.find_control('component').items
415 component_names = map(lambda item: item.name, component_items)
416 if not component or component not in component_names:
417 component = self.prompt_for_component(component_names)
418 self.browser['component'] = [component]
420 self.browser['cc'] = cc
421 self.browser['short_desc'] = bug_title
424 self.browser['comment'] = bug_description
426 self._fill_attachment_form(patch_description, patch_file_object, mark_for_review=mark_for_review, mark_for_commit_queue=mark_for_commit_queue)
427 response = self.browser.submit()
429 bug_id = self._check_create_bug_response(response.read())
430 log("Bug %s created." % bug_id)
431 log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id))
434 def _find_select_element_for_flag(self, flag_name):
435 # FIXME: This will break if we ever re-order attachment flags
436 if flag_name == "review":
437 return self.browser.find_control(type='select', nr=0)
438 if flag_name == "commit-queue":
439 return self.browser.find_control(type='select', nr=1)
440 raise Exception("Don't know how to find flag named \"%s\"" % flag_name)
442 def clear_attachment_flags(self, attachment_id, additional_comment_text=None):
445 comment_text = "Clearing flags on attachment: %s" % attachment_id
446 if additional_comment_text:
447 comment_text += "\n\n%s" % additional_comment_text
453 self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
454 self.browser.select_form(nr=1)
455 self.browser.set_value(comment_text, name='comment', nr=0)
456 self._find_select_element_for_flag('review').value = ("X",)
457 self._find_select_element_for_flag('commit-queue').value = ("X",)
458 self.browser.submit()
460 # FIXME: We need a way to test this on a live bugzilla instance.
461 def _set_flag_on_attachment(self, attachment_id, flag_name, flag_value, comment_text, additional_comment_text):
464 if additional_comment_text:
465 comment_text += "\n\n%s" % additional_comment_text
471 self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
472 self.browser.select_form(nr=1)
473 self.browser.set_value(comment_text, name='comment', nr=0)
474 self._find_select_element_for_flag(flag_name).value = (flag_value,)
475 self.browser.submit()
477 def reject_patch_from_commit_queue(self, attachment_id, additional_comment_text=None):
478 comment_text = "Rejecting patch %s from commit-queue." % attachment_id
479 self._set_flag_on_attachment(attachment_id, 'commit-queue', '-', comment_text, additional_comment_text)
481 def reject_patch_from_review_queue(self, attachment_id, additional_comment_text=None):
482 comment_text = "Rejecting patch %s from review queue." % attachment_id
483 self._set_flag_on_attachment(attachment_id, 'review', '-', comment_text, additional_comment_text)
485 def obsolete_attachment(self, attachment_id, comment_text = None):
488 log("Obsoleting attachment: %s" % attachment_id)
493 self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
494 self.browser.select_form(nr=1)
495 self.browser.find_control('isobsolete').items[0].selected = True
496 # Also clear any review flag (to remove it from review/commit queues)
497 self._find_select_element_for_flag('review').value = ("X",)
498 self._find_select_element_for_flag('commit-queue').value = ("X",)
501 # Bugzilla has two textareas named 'comment', one is somehow hidden. We want the first.
502 self.browser.set_value(comment_text, name='comment', nr=0)
503 self.browser.submit()
505 def post_comment_to_bug(self, bug_id, comment_text):
508 log("Adding comment to bug %s" % bug_id)
513 self.browser.open(self.bug_url_for_bug_id(bug_id))
514 self.browser.select_form(name="changeform")
515 self.browser['comment'] = comment_text
516 self.browser.submit()
518 def close_bug_as_fixed(self, bug_id, comment_text=None):
521 log("Closing bug %s as fixed" % bug_id)
526 self.browser.open(self.bug_url_for_bug_id(bug_id))
527 self.browser.select_form(name="changeform")
530 self.browser['comment'] = comment_text
531 self.browser['bug_status'] = ['RESOLVED']
532 self.browser['resolution'] = ['FIXED']
533 self.browser.submit()
535 def reopen_bug(self, bug_id, comment_text):
538 log("Re-opening bug %s" % bug_id)
539 log(comment_text) # Bugzilla requires a comment when re-opening a bug, so we know it will never be None.
543 self.browser.open(self.bug_url_for_bug_id(bug_id))
544 self.browser.select_form(name="changeform")
545 self.browser['bug_status'] = ['REOPENED']
546 self.browser['comment'] = comment_text
547 self.browser.submit()