2009-11-19 Eric Seidel <eric@webkit.org>
[WebKit-https.git] / WebKitTools / Scripts / modules / bugzilla.py
1 # Copyright (c) 2009, Google Inc. All rights reserved.
2 # Copyright (c) 2009 Apple Inc. All rights reserved.
3
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 #
30 # WebKit's Python module for interacting with Bugzilla
31
32 import getpass
33 import platform
34 import re
35 import subprocess
36 import urllib2
37
38 from datetime import datetime # used in timestamp()
39
40 # Import WebKit-specific modules.
41 from modules.logging import error, log
42 from modules.committers import CommitterList
43
44 # WebKit includes a built copy of BeautifulSoup in Scripts/modules
45 # so this import should always succeed.
46 from .BeautifulSoup import BeautifulSoup
47
48 try:
49     from mechanize import Browser
50 except ImportError, e:
51     print """
52 mechanize is required.
53
54 To install:
55 sudo easy_install mechanize
56
57 Or from the web:
58 http://wwwsearch.sourceforge.net/mechanize/
59 """
60     exit(1)
61
62 def credentials_from_git():
63     return [read_config("username"), read_config("password")]
64
65 def credentials_from_keychain(username=None):
66     if not is_mac_os_x():
67         return [username, None]
68
69     command = "/usr/bin/security %s -g -s %s" % ("find-internet-password", Bugzilla.bug_server_host)
70     if username:
71         command += " -a %s" % username
72
73     log('Reading Keychain for %s account and password.  Click "Allow" to continue...' % Bugzilla.bug_server_host)
74     keychain_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
75     value = keychain_process.communicate()[0]
76     exit_code = keychain_process.wait()
77
78     if exit_code:
79         return [username, None]
80
81     match = re.search('^\s*"acct"<blob>="(?P<username>.+)"', value, re.MULTILINE)
82     if match:
83         username = match.group('username')
84
85     password = None
86     match = re.search('^password: "(?P<password>.+)"', value, re.MULTILINE)
87     if match:
88         password = match.group('password')
89
90     return [username, password]
91
92 def is_mac_os_x():
93     return platform.mac_ver()[0]
94
95 def parse_bug_id(message):
96     match = re.search("http\://webkit\.org/b/(?P<bug_id>\d+)", message)
97     if match:
98         return match.group('bug_id')
99     match = re.search(Bugzilla.bug_server_regex + "show_bug\.cgi\?id=(?P<bug_id>\d+)", message)
100     if match:
101         return match.group('bug_id')
102     return None
103
104 # FIXME: This should not depend on git for config storage
105 def read_config(key):
106     # Need a way to read from svn too
107     config_process = subprocess.Popen("git config --get bugzilla.%s" % key, stdout=subprocess.PIPE, shell=True)
108     value = config_process.communicate()[0]
109     return_code = config_process.wait()
110
111     if return_code:
112         return None
113     return value.rstrip('\n')
114
115 def read_credentials():
116     (username, password) = credentials_from_git()
117
118     if not username or not password:
119         (username, password) = credentials_from_keychain(username)
120
121     if not username:
122         username = raw_input("Bugzilla login: ")
123     if not password:
124         password = getpass.getpass("Bugzilla password for %s: " % username)
125
126     return [username, password]
127
128 def timestamp():
129     return datetime.now().strftime("%Y%m%d%H%M%S")
130
131
132 class BugzillaError(Exception):
133     pass
134
135
136 class Bugzilla:
137     def __init__(self, dryrun=False, committers=CommitterList()):
138         self.dryrun = dryrun
139         self.authenticated = False
140
141         self.browser = Browser()
142         # Ignore bugs.webkit.org/robots.txt until we fix it to allow this script
143         self.browser.set_handle_robots(False)
144         self.committers = committers
145
146     # Defaults (until we support better option parsing):
147     bug_server_host = "bugs.webkit.org"
148     bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
149     bug_server_url = "https://%s/" % bug_server_host
150
151     def bug_url_for_bug_id(self, bug_id, xml=False):
152         content_type = "&ctype=xml" if xml else ""
153         return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type)
154
155     def short_bug_url_for_bug_id(self, bug_id):
156         return "http://webkit.org/b/%s" % bug_id
157
158     def attachment_url_for_id(self, attachment_id, action="view"):
159         action_param = ""
160         if action and action != "view":
161             action_param = "&action=%s" % action
162         return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param)
163
164     def _parse_attachment_flag(self, element, flag_name, attachment, result_key):
165         flag = element.find('flag', attrs={'name' : flag_name})
166         if flag:
167             attachment[flag_name] = flag['status']
168             if flag['status'] == '+':
169                 attachment[result_key] = flag['setter']
170
171     def _parse_attachment_element(self, element, bug_id):
172         attachment = {}
173         attachment['bug_id'] = bug_id
174         attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
175         attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
176         attachment['id'] = str(element.find('attachid').string)
177         attachment['url'] = self.attachment_url_for_id(attachment['id'])
178         attachment['name'] = unicode(element.find('desc').string)
179         attachment['type'] = str(element.find('type').string)
180         self._parse_attachment_flag(element, 'review', attachment, 'reviewer_email')
181         self._parse_attachment_flag(element, 'commit-queue', attachment, 'committer_email')
182         return attachment
183
184     def fetch_attachments_from_bug(self, bug_id):
185         bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
186         log("Fetching: %s" % bug_url)
187
188         page = urllib2.urlopen(bug_url)
189         soup = BeautifulSoup(page)
190
191         attachments = []
192         for element in soup.findAll('attachment'):
193             attachment = self._parse_attachment_element(element, bug_id)
194             attachments.append(attachment)
195         return attachments
196
197     def _parse_bug_id_from_attachment_page(self, page):
198         up_link = BeautifulSoup(page).find('link', rel='Up') # The "Up" relation happens to point to the bug.
199         if not up_link:
200             return None # This attachment does not exist (or you don't have permissions to view it).
201         match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href'])
202         return int(match.group('bug_id'))
203
204     def bug_id_for_attachment_id(self, attachment_id):
205         attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
206         log("Fetching: %s" % attachment_url)
207         page = urllib2.urlopen(attachment_url)
208         return self._parse_bug_id_from_attachment_page(page)
209
210     # This should really return an Attachment object
211     # which can lazily fetch any missing data.
212     def fetch_attachment(self, attachment_id):
213         # We could grab all the attachment details off of the attachment edit page
214         # but we already have working code to do so off of the bugs page, so re-use that.
215         bug_id = self.bug_id_for_attachment_id(attachment_id)
216         if not bug_id:
217             return None
218         attachments = self.fetch_attachments_from_bug(bug_id)
219         for attachment in attachments:
220             if attachment['id'] == attachment_id:
221                 self._validate_committer_and_reviewer(attachment)
222                 return attachment
223         return None # This should never be hit.
224
225     def fetch_title_from_bug(self, bug_id):
226         bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
227         page = urllib2.urlopen(bug_url)
228         soup = BeautifulSoup(page)
229         return soup.find('short_desc').string
230
231     def fetch_patches_from_bug(self, bug_id):
232         patches = []
233         for attachment in self.fetch_attachments_from_bug(bug_id):
234             if attachment['is_patch'] and not attachment['is_obsolete']:
235                 patches.append(attachment)
236         return patches
237
238     # _view_source_link belongs in some sort of webkit_config.py module.
239     def _view_source_link(self, local_path):
240         return "http://trac.webkit.org/browser/trunk/%s" % local_path
241
242     def _flag_permission_rejection_message(self, setter_email, flag_name):
243         committer_list = "WebKitTools/Scripts/modules/committers.py"
244         contribution_guidlines_url = "http://webkit.org/coding/contributing.html"
245         rejection_message = "%s does not have %s permissions according to %s." % (setter_email, flag_name, self._view_source_link(committer_list))
246         rejection_message += "\n\n- If you have %s rights please correct the error in %s by adding yourself to the file (no review needed) and then set the %s flag again." % (flag_name, committer_list, flag_name)
247         rejection_message += "\n\n- If you do not have %s rights please read %s for instructions on how to use bugzilla flags." % (flag_name, contribution_guidlines_url)
248         return rejection_message
249
250     def _validate_setter_email(self, patch, result_key, lookup_function, rejection_function, reject_invalid_patches):
251         setter_email = patch.get(result_key + '_email')
252         if not setter_email:
253             return None
254
255         committer = lookup_function(setter_email)
256         if committer:
257             patch[result_key] = committer.full_name
258             return patch[result_key]
259
260         if reject_invalid_patches:
261             rejection_function(patch['id'], self._flag_permission_rejection_message(setter_email, result_key))
262         else:
263             log("Warning, attachment %s on bug %s has invalid %s (%s)" % (patch['id'], patch['bug_id'], result_key, setter_email))
264         return None
265
266     def _validate_reviewer(self, patch, reject_invalid_patches):
267         return self._validate_setter_email(patch, 'reviewer', self.committers.reviewer_by_email, self.reject_patch_from_review_queue, reject_invalid_patches)
268
269     def _validate_committer(self, patch, reject_invalid_patches):
270         return self._validate_setter_email(patch, 'committer', self.committers.committer_by_email, self.reject_patch_from_commit_queue, reject_invalid_patches)
271
272     # FIXME: This is a hack until we have a real Attachment object.
273     # _validate_committer and _validate_reviewer fill in the 'reviewer' and 'committer'
274     # keys which other parts of the code expect to be filled in.
275     def _validate_committer_and_reviewer(self, patch):
276         self._validate_reviewer(patch, reject_invalid_patches=False)
277         self._validate_committer(patch, reject_invalid_patches=False)
278
279     def fetch_unreviewed_patches_from_bug(self, bug_id):
280         unreviewed_patches = []
281         for attachment in self.fetch_attachments_from_bug(bug_id):
282             if attachment.get('review') == '?' and not attachment['is_obsolete']:
283                 unreviewed_patches.append(attachment)
284         return unreviewed_patches
285
286     def fetch_reviewed_patches_from_bug(self, bug_id, reject_invalid_patches=False):
287         reviewed_patches = []
288         for attachment in self.fetch_attachments_from_bug(bug_id):
289             if self._validate_reviewer(attachment, reject_invalid_patches) and not attachment['is_obsolete']:
290                 reviewed_patches.append(attachment)
291         return reviewed_patches
292
293     def fetch_commit_queue_patches_from_bug(self, bug_id, reject_invalid_patches=False):
294         commit_queue_patches = []
295         for attachment in self.fetch_reviewed_patches_from_bug(bug_id, reject_invalid_patches):
296             if self._validate_committer(attachment, reject_invalid_patches) and not attachment['is_obsolete']:
297                 commit_queue_patches.append(attachment)
298         return commit_queue_patches
299
300     def _fetch_bug_ids_advanced_query(self, query):
301         page = urllib2.urlopen(query)
302         soup = BeautifulSoup(page)
303
304         bug_ids = []
305         # Grab the cells in the first column (which happens to be the bug ids)
306         for bug_link_cell in soup('td', "first-child"): # tds with the class "first-child"
307             bug_link = bug_link_cell.find("a")
308             bug_ids.append(bug_link.string) # the contents happen to be the bug id
309
310         return bug_ids
311
312     def fetch_bug_ids_from_commit_queue(self):
313         commit_queue_url = self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=commit-queue%2B"
314         return self._fetch_bug_ids_advanced_query(commit_queue_url)
315
316     def fetch_bug_ids_from_review_queue(self):
317         review_queue_url = self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review?"
318         return self._fetch_bug_ids_advanced_query(review_queue_url)
319
320     def fetch_patches_from_commit_queue(self, reject_invalid_patches=False):
321         patches_to_land = []
322         for bug_id in self.fetch_bug_ids_from_commit_queue():
323             patches = self.fetch_commit_queue_patches_from_bug(bug_id, reject_invalid_patches)
324             patches_to_land += patches
325         return patches_to_land
326
327     def fetch_patches_from_review_queue(self, limit):
328         patches_to_review = []
329         for bug_id in self.fetch_bug_ids_from_review_queue():
330             if len(patches_to_review) >= limit:
331                 break
332             patches = self.fetch_unreviewed_patches_from_bug(bug_id)
333             patches_to_review += patches
334         return patches_to_review
335
336     def authenticate(self):
337         if self.authenticated:
338             return
339
340         if self.dryrun:
341             log("Skipping log in for dry run...")
342             self.authenticated = True
343             return
344
345         (username, password) = read_credentials()
346
347         log("Logging in as %s..." % username)
348         self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1")
349         self.browser.select_form(name="login")
350         self.browser['Bugzilla_login'] = username
351         self.browser['Bugzilla_password'] = password
352         response = self.browser.submit()
353
354         match = re.search("<title>(.+?)</title>", response.read())
355         # If the resulting page has a title, and it contains the word "invalid" assume it's the login failure page.
356         if match and re.search("Invalid", match.group(1), re.IGNORECASE):
357             # FIXME: We could add the ability to try again on failure.
358             raise BugzillaError("Bugzilla login failed: %s" % match.group(1))
359
360         self.authenticated = True
361
362     def _fill_attachment_form(self, description, patch_file_object, comment_text=None, mark_for_review=False, mark_for_commit_queue=False, bug_id=None):
363         self.browser['description'] = description
364         self.browser['ispatch'] = ("1",)
365         self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
366         self.browser['flag_type-3'] = ('?',) if mark_for_commit_queue else ('X',)
367         if bug_id:
368             patch_name = "bug-%s-%s.patch" % (bug_id, timestamp())
369         else:
370             patch_name ="%s.patch" % timestamp()
371         self.browser.add_file(patch_file_object, "text/plain", patch_name, 'data')
372
373     def add_patch_to_bug(self, bug_id, patch_file_object, description, comment_text=None, mark_for_review=False, mark_for_commit_queue=False):
374         self.authenticate()
375         
376         log('Adding patch "%s" to bug %s' % (description, bug_id))
377         if self.dryrun:
378             log(comment_text)
379             return
380         
381         self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % (self.bug_server_url, bug_id))
382         self.browser.select_form(name="entryform")
383         self._fill_attachment_form(description, patch_file_object, mark_for_review=mark_for_review, mark_for_commit_queue=mark_for_commit_queue, bug_id=bug_id)
384         if comment_text:
385             log(comment_text)
386             self.browser['comment'] = comment_text
387         self.browser.submit()
388
389     def prompt_for_component(self, components):
390         log("Please pick a component:")
391         i = 0
392         for name in components:
393             i += 1
394             log("%2d. %s" % (i, name))
395         result = int(raw_input("Enter a number: ")) - 1
396         return components[result]
397
398     def _check_create_bug_response(self, response_html):
399         match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>", response_html)
400         if match:
401             return match.group('bug_id')
402
403         match = re.search('<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">', response_html, re.DOTALL)
404         error_message = "FAIL"
405         if match:
406             text_lines = BeautifulSoup(match.group('error_message')).findAll(text=True)
407             error_message = "\n" + '\n'.join(["  " + line.strip() for line in text_lines if line.strip()])
408         raise BugzillaError("Bug not created: %s" % error_message)
409
410     def create_bug_with_patch(self, bug_title, bug_description, component, patch_file_object, patch_description, cc, mark_for_review=False, mark_for_commit_queue=False):
411         self.authenticate()
412
413         log('Creating bug with patch description "%s"' % patch_description)
414         if self.dryrun:
415             log(bug_description)
416             return
417
418         self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
419         self.browser.select_form(name="Create")
420         component_items = self.browser.find_control('component').items
421         component_names = map(lambda item: item.name, component_items)
422         if not component or component not in component_names:
423             component = self.prompt_for_component(component_names)
424         self.browser['component'] = [component]
425         if cc:
426             self.browser['cc'] = cc
427         self.browser['short_desc'] = bug_title
428         if bug_description:
429             log(bug_description)
430             self.browser['comment'] = bug_description
431
432         self._fill_attachment_form(patch_description, patch_file_object, mark_for_review=mark_for_review, mark_for_commit_queue=mark_for_commit_queue)
433         response = self.browser.submit()
434
435         bug_id = self._check_create_bug_response(response.read())
436         log("Bug %s created." % bug_id)
437         log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id))
438         return bug_id
439
440     def _find_select_element_for_flag(self, flag_name):
441         # FIXME: This will break if we ever re-order attachment flags
442         if flag_name == "review":
443             return self.browser.find_control(type='select', nr=0)
444         if flag_name == "commit-queue":
445             return self.browser.find_control(type='select', nr=1)
446         raise Exception("Don't know how to find flag named \"%s\"" % flag_name)
447
448     def clear_attachment_flags(self, attachment_id, additional_comment_text=None):
449         self.authenticate()
450
451         comment_text = "Clearing flags on attachment: %s" % attachment_id
452         if additional_comment_text:
453             comment_text += "\n\n%s" % additional_comment_text
454         log(comment_text)
455
456         if self.dryrun:
457             return
458
459         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
460         self.browser.select_form(nr=1)
461         self.browser.set_value(comment_text, name='comment', nr=0)
462         self._find_select_element_for_flag('review').value = ("X",)
463         self._find_select_element_for_flag('commit-queue').value = ("X",)
464         self.browser.submit()
465
466     # FIXME: We need a way to test this on a live bugzilla instance.
467     def _set_flag_on_attachment(self, attachment_id, flag_name, flag_value, comment_text, additional_comment_text):
468         self.authenticate()
469
470         if additional_comment_text:
471             comment_text += "\n\n%s" % additional_comment_text
472         log(comment_text)
473
474         if self.dryrun:
475             return
476
477         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
478         self.browser.select_form(nr=1)
479         self.browser.set_value(comment_text, name='comment', nr=0)
480         self._find_select_element_for_flag(flag_name).value = (flag_value,)
481         self.browser.submit()
482
483     def reject_patch_from_commit_queue(self, attachment_id, additional_comment_text=None):
484         comment_text = "Rejecting patch %s from commit-queue." % attachment_id
485         self._set_flag_on_attachment(attachment_id, 'commit-queue', '-', comment_text, additional_comment_text)
486
487     def reject_patch_from_review_queue(self, attachment_id, additional_comment_text=None):
488         comment_text = "Rejecting patch %s from review queue." % attachment_id
489         self._set_flag_on_attachment(attachment_id, 'review', '-', comment_text, additional_comment_text)
490
491     def obsolete_attachment(self, attachment_id, comment_text = None):
492         self.authenticate()
493
494         log("Obsoleting attachment: %s" % attachment_id)
495         if self.dryrun:
496             log(comment_text)
497             return
498
499         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
500         self.browser.select_form(nr=1)
501         self.browser.find_control('isobsolete').items[0].selected = True
502         # Also clear any review flag (to remove it from review/commit queues)
503         self._find_select_element_for_flag('review').value = ("X",)
504         self._find_select_element_for_flag('commit-queue').value = ("X",)
505         if comment_text:
506             log(comment_text)
507             # Bugzilla has two textareas named 'comment', one is somehow hidden.  We want the first.
508             self.browser.set_value(comment_text, name='comment', nr=0)
509         self.browser.submit()
510     
511     def post_comment_to_bug(self, bug_id, comment_text):
512         self.authenticate()
513
514         log("Adding comment to bug %s" % bug_id)
515         if self.dryrun:
516             log(comment_text)
517             return
518
519         self.browser.open(self.bug_url_for_bug_id(bug_id))
520         self.browser.select_form(name="changeform")
521         self.browser['comment'] = comment_text
522         self.browser.submit()
523
524     def close_bug_as_fixed(self, bug_id, comment_text=None):
525         self.authenticate()
526
527         log("Closing bug %s as fixed" % bug_id)
528         if self.dryrun:
529             log(comment_text)
530             return
531
532         self.browser.open(self.bug_url_for_bug_id(bug_id))
533         self.browser.select_form(name="changeform")
534         if comment_text:
535             log(comment_text)
536             self.browser['comment'] = comment_text
537         self.browser['bug_status'] = ['RESOLVED']
538         self.browser['resolution'] = ['FIXED']
539         self.browser.submit()
540
541     def reopen_bug(self, bug_id, comment_text):
542         self.authenticate()
543
544         log("Re-opening bug %s" % bug_id)
545         log(comment_text) # Bugzilla requires a comment when re-opening a bug, so we know it will never be None.
546         if self.dryrun:
547             return
548
549         self.browser.open(self.bug_url_for_bug_id(bug_id))
550         self.browser.select_form(name="changeform")
551         self.browser['bug_status'] = ['REOPENED']
552         self.browser['comment'] = comment_text
553         self.browser.submit()