3a9ca680816a9d2605438e3fb22fd3640295636e
[WebKit-https.git] / WebKitTools / Scripts / modules / bugzilla.py
1 # Copyright (c) 2009, Google Inc. All rights reserved.
2 # Copyright (c) 2009 Apple Inc. All rights reserved.
3
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 #
30 # WebKit's Python module for interacting with Bugzilla
31
32 import getpass
33 import platform
34 import re
35 import subprocess
36 import urllib2
37
38 from datetime import datetime # used in timestamp()
39
40 # Import WebKit-specific modules.
41 from modules.logging import error, log
42 from modules.committers import CommitterList
43
44 # WebKit includes a built copy of BeautifulSoup in Scripts/modules
45 # so this import should always succeed.
46 from .BeautifulSoup import BeautifulSoup
47
48 try:
49     from mechanize import Browser
50 except ImportError, e:
51     print """
52 mechanize is required.
53
54 To install:
55 sudo easy_install mechanize
56
57 Or from the web:
58 http://wwwsearch.sourceforge.net/mechanize/
59 """
60     exit(1)
61
62 def credentials_from_git():
63     return [read_config("username"), read_config("password")]
64
65 def credentials_from_keychain(username=None):
66     if not is_mac_os_x():
67         return [username, None]
68
69     command = "/usr/bin/security %s -g -s %s" % ("find-internet-password", Bugzilla.bug_server_host)
70     if username:
71         command += " -a %s" % username
72
73     log('Reading Keychain for %s account and password.  Click "Allow" to continue...' % Bugzilla.bug_server_host)
74     keychain_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
75     value = keychain_process.communicate()[0]
76     exit_code = keychain_process.wait()
77
78     if exit_code:
79         return [username, None]
80
81     match = re.search('^\s*"acct"<blob>="(?P<username>.+)"', value, re.MULTILINE)
82     if match:
83         username = match.group('username')
84
85     password = None
86     match = re.search('^password: "(?P<password>.+)"', value, re.MULTILINE)
87     if match:
88         password = match.group('password')
89
90     return [username, password]
91
92 def is_mac_os_x():
93     return platform.mac_ver()[0]
94
95 def parse_bug_id(message):
96     match = re.search("http\://webkit\.org/b/(?P<bug_id>\d+)", message)
97     if match:
98         return match.group('bug_id')
99     match = re.search(Bugzilla.bug_server_regex + "show_bug\.cgi\?id=(?P<bug_id>\d+)", message)
100     if match:
101         return match.group('bug_id')
102     return None
103
104 # FIXME: This should not depend on git for config storage
105 def read_config(key):
106     # Need a way to read from svn too
107     config_process = subprocess.Popen("git config --get bugzilla.%s" % key, stdout=subprocess.PIPE, shell=True)
108     value = config_process.communicate()[0]
109     return_code = config_process.wait()
110
111     if return_code:
112         return None
113     return value.rstrip('\n')
114
115 def read_credentials():
116     (username, password) = credentials_from_git()
117
118     if not username or not password:
119         (username, password) = credentials_from_keychain(username)
120
121     if not username:
122         username = raw_input("Bugzilla login: ")
123     if not password:
124         password = getpass.getpass("Bugzilla password for %s: " % username)
125
126     return [username, password]
127
128 def timestamp():
129     return datetime.now().strftime("%Y%m%d%H%M%S")
130
131
132 class BugzillaError(Exception):
133     pass
134
135
136 class Bugzilla:
137     def __init__(self, dryrun=False, committers=CommitterList()):
138         self.dryrun = dryrun
139         self.authenticated = False
140
141         self.browser = Browser()
142         # Ignore bugs.webkit.org/robots.txt until we fix it to allow this script
143         self.browser.set_handle_robots(False)
144         self.committers = committers
145
146     # Defaults (until we support better option parsing):
147     bug_server_host = "bugs.webkit.org"
148     bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
149     bug_server_url = "https://%s/" % bug_server_host
150
151     def bug_url_for_bug_id(self, bug_id, xml=False):
152         content_type = "&ctype=xml" if xml else ""
153         return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type)
154
155     def short_bug_url_for_bug_id(self, bug_id):
156         return "http://webkit.org/b/%s" % bug_id
157
158     def attachment_url_for_id(self, attachment_id, action="view"):
159         action_param = ""
160         if action and action != "view":
161             action_param = "&action=%s" % action
162         return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param)
163
164     def _parse_attachment_flag(self, element, flag_name, attachment, result_key):
165         flag = element.find('flag', attrs={'name' : flag_name})
166         if flag:
167             attachment[flag_name] = flag['status']
168             if flag['status'] == '+':
169                 attachment[result_key] = flag['setter']
170
171     def _parse_attachment_element(self, element, bug_id):
172         attachment = {}
173         attachment['bug_id'] = bug_id
174         attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
175         attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
176         attachment['id'] = str(element.find('attachid').string)
177         attachment['url'] = self.attachment_url_for_id(attachment['id'])
178         attachment['name'] = unicode(element.find('desc').string)
179         attachment['type'] = str(element.find('type').string)
180         self._parse_attachment_flag(element, 'review', attachment, 'reviewer_email')
181         self._parse_attachment_flag(element, 'commit-queue', attachment, 'committer_email')
182         return attachment
183
184     def fetch_attachments_from_bug(self, bug_id):
185         bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
186         log("Fetching: %s" % bug_url)
187
188         page = urllib2.urlopen(bug_url)
189         soup = BeautifulSoup(page)
190
191         attachments = []
192         for element in soup.findAll('attachment'):
193             attachment = self._parse_attachment_element(element, bug_id)
194             attachments.append(attachment)
195         return attachments
196
197     def _parse_bug_id_from_attachment_page(self, page):
198         up_link = BeautifulSoup(page).find('link', rel='Up') # The "Up" relation happens to point to the bug.
199         if not up_link:
200             return None # This attachment does not exist (or you don't have permissions to view it).
201         match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href'])
202         return int(match.group('bug_id'))
203
204     def bug_id_for_attachment_id(self, attachment_id):
205         attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
206         log("Fetching: %s" % attachment_url)
207         page = urllib2.urlopen(attachment_url)
208         return self._parse_bug_id_from_attachment_page(page)
209
210     # This should really return an Attachment object
211     # which can lazily fetch any missing data.
212     def fetch_attachment(self, attachment_id):
213         # We could grab all the attachment details off of the attachment edit page
214         # but we already have working code to do so off of the bugs page, so re-use that.
215         bug_id = self.bug_id_for_attachment_id(attachment_id)
216         if not bug_id:
217             return None
218         attachments = self.fetch_attachments_from_bug(bug_id)
219         for attachment in attachments:
220             if attachment['id'] == attachment_id:
221                 self._validate_committer_and_reviewer(attachment)
222                 return attachment
223         return None # This should never be hit.
224
225     def fetch_title_from_bug(self, bug_id):
226         bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
227         page = urllib2.urlopen(bug_url)
228         soup = BeautifulSoup(page)
229         return soup.find('short_desc').string
230
231     def fetch_patches_from_bug(self, bug_id):
232         patches = []
233         for attachment in self.fetch_attachments_from_bug(bug_id):
234             if attachment['is_patch'] and not attachment['is_obsolete']:
235                 patches.append(attachment)
236         return patches
237
238     # _view_source_link belongs in some sort of webkit_config.py module.
239     def _view_source_link(self, local_path):
240         return "http://trac.webkit.org/browser/trunk/%s" % local_path
241
242     def _validate_setter_email(self, patch, result_key, lookup_function, rejection_function, reject_invalid_patches):
243         setter_email = patch.get(result_key + '_email')
244         if not setter_email:
245             return None
246
247         committer = lookup_function(setter_email)
248         if committer:
249             patch[result_key] = committer.full_name
250             return patch[result_key]
251
252         if reject_invalid_patches:
253             committer_list = "WebKitTools/Scripts/modules/committers.py"
254             failure_message = "%s does not have %s permissions according to %s." % (setter_email, result_key, self._view_source_link(committer_list))
255             rejection_function(patch['id'], failure_message)
256         else:
257             log("Warning, attachment %s on bug %s has invalid %s (%s)" % (patch['id'], patch['bug_id'], result_key, setter_email))
258         return None
259
260     def _validate_reviewer(self, patch, reject_invalid_patches):
261         return self._validate_setter_email(patch, 'reviewer', self.committers.reviewer_by_email, self.reject_patch_from_review_queue, reject_invalid_patches)
262
263     def _validate_committer(self, patch, reject_invalid_patches):
264         return self._validate_setter_email(patch, 'committer', self.committers.committer_by_email, self.reject_patch_from_commit_queue, reject_invalid_patches)
265
266     # FIXME: This is a hack until we have a real Attachment object.
267     # _validate_committer and _validate_reviewer fill in the 'reviewer' and 'committer'
268     # keys which other parts of the code expect to be filled in.
269     def _validate_committer_and_reviewer(self, patch):
270         self._validate_reviewer(patch, reject_invalid_patches=False)
271         self._validate_committer(patch, reject_invalid_patches=False)
272
273     def fetch_unreviewed_patches_from_bug(self, bug_id):
274         unreviewed_patches = []
275         for attachment in self.fetch_attachments_from_bug(bug_id):
276             if attachment.get('review') == '?' and not attachment['is_obsolete']:
277                 unreviewed_patches.append(attachment)
278         return unreviewed_patches
279
280     def fetch_reviewed_patches_from_bug(self, bug_id, reject_invalid_patches=False):
281         reviewed_patches = []
282         for attachment in self.fetch_attachments_from_bug(bug_id):
283             if self._validate_reviewer(attachment, reject_invalid_patches) and not attachment['is_obsolete']:
284                 reviewed_patches.append(attachment)
285         return reviewed_patches
286
287     def fetch_commit_queue_patches_from_bug(self, bug_id, reject_invalid_patches=False):
288         commit_queue_patches = []
289         for attachment in self.fetch_reviewed_patches_from_bug(bug_id, reject_invalid_patches):
290             if self._validate_committer(attachment, reject_invalid_patches) and not attachment['is_obsolete']:
291                 commit_queue_patches.append(attachment)
292         return commit_queue_patches
293
294     def _fetch_bug_ids_advanced_query(self, query):
295         page = urllib2.urlopen(query)
296         soup = BeautifulSoup(page)
297
298         bug_ids = []
299         # Grab the cells in the first column (which happens to be the bug ids)
300         for bug_link_cell in soup('td', "first-child"): # tds with the class "first-child"
301             bug_link = bug_link_cell.find("a")
302             bug_ids.append(bug_link.string) # the contents happen to be the bug id
303
304         return bug_ids
305
306     def fetch_bug_ids_from_commit_queue(self):
307         commit_queue_url = self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=commit-queue%2B"
308         return self._fetch_bug_ids_advanced_query(commit_queue_url)
309
310     def fetch_bug_ids_from_review_queue(self):
311         review_queue_url = self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review?"
312         return self._fetch_bug_ids_advanced_query(review_queue_url)
313
314     def fetch_patches_from_commit_queue(self, reject_invalid_patches=False):
315         patches_to_land = []
316         for bug_id in self.fetch_bug_ids_from_commit_queue():
317             patches = self.fetch_commit_queue_patches_from_bug(bug_id, reject_invalid_patches)
318             patches_to_land += patches
319         return patches_to_land
320
321     def fetch_patches_from_review_queue(self, limit):
322         patches_to_review = []
323         for bug_id in self.fetch_bug_ids_from_review_queue():
324             if len(patches_to_review) >= limit:
325                 break
326             patches = self.fetch_unreviewed_patches_from_bug(bug_id)
327             patches_to_review += patches
328         return patches_to_review
329
330     def authenticate(self):
331         if self.authenticated:
332             return
333
334         if self.dryrun:
335             log("Skipping log in for dry run...")
336             self.authenticated = True
337             return
338
339         (username, password) = read_credentials()
340
341         log("Logging in as %s..." % username)
342         self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1")
343         self.browser.select_form(name="login")
344         self.browser['Bugzilla_login'] = username
345         self.browser['Bugzilla_password'] = password
346         response = self.browser.submit()
347
348         match = re.search("<title>(.+?)</title>", response.read())
349         # If the resulting page has a title, and it contains the word "invalid" assume it's the login failure page.
350         if match and re.search("Invalid", match.group(1), re.IGNORECASE):
351             # FIXME: We could add the ability to try again on failure.
352             raise BugzillaError("Bugzilla login failed: %s" % match.group(1))
353
354         self.authenticated = True
355
356     def _fill_attachment_form(self, description, patch_file_object, comment_text=None, mark_for_review=False, mark_for_commit_queue=False, bug_id=None):
357         self.browser['description'] = description
358         self.browser['ispatch'] = ("1",)
359         self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
360         self.browser['flag_type-3'] = ('?',) if mark_for_commit_queue else ('X',)
361         if bug_id:
362             patch_name = "bug-%s-%s.patch" % (bug_id, timestamp())
363         else:
364             patch_name ="%s.patch" % timestamp()
365         self.browser.add_file(patch_file_object, "text/plain", patch_name, 'data')
366
367     def add_patch_to_bug(self, bug_id, patch_file_object, description, comment_text=None, mark_for_review=False, mark_for_commit_queue=False):
368         self.authenticate()
369         
370         log('Adding patch "%s" to bug %s' % (description, bug_id))
371         if self.dryrun:
372             log(comment_text)
373             return
374         
375         self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % (self.bug_server_url, bug_id))
376         self.browser.select_form(name="entryform")
377         self._fill_attachment_form(description, patch_file_object, mark_for_review=mark_for_review, mark_for_commit_queue=mark_for_commit_queue, bug_id=bug_id)
378         if comment_text:
379             log(comment_text)
380             self.browser['comment'] = comment_text
381         self.browser.submit()
382
383     def prompt_for_component(self, components):
384         log("Please pick a component:")
385         i = 0
386         for name in components:
387             i += 1
388             log("%2d. %s" % (i, name))
389         result = int(raw_input("Enter a number: ")) - 1
390         return components[result]
391
392     def _check_create_bug_response(self, response_html):
393         match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>", response_html)
394         if match:
395             return match.group('bug_id')
396
397         match = re.search('<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">', response_html, re.DOTALL)
398         error_message = "FAIL"
399         if match:
400             text_lines = BeautifulSoup(match.group('error_message')).findAll(text=True)
401             error_message = "\n" + '\n'.join(["  " + line.strip() for line in text_lines if line.strip()])
402         raise BugzillaError("Bug not created: %s" % error_message)
403
404     def create_bug_with_patch(self, bug_title, bug_description, component, patch_file_object, patch_description, cc, mark_for_review=False, mark_for_commit_queue=False):
405         self.authenticate()
406
407         log('Creating bug with patch description "%s"' % patch_description)
408         if self.dryrun:
409             log(bug_description)
410             return
411
412         self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
413         self.browser.select_form(name="Create")
414         component_items = self.browser.find_control('component').items
415         component_names = map(lambda item: item.name, component_items)
416         if not component or component not in component_names:
417             component = self.prompt_for_component(component_names)
418         self.browser['component'] = [component]
419         if cc:
420             self.browser['cc'] = cc
421         self.browser['short_desc'] = bug_title
422         if bug_description:
423             log(bug_description)
424             self.browser['comment'] = bug_description
425
426         self._fill_attachment_form(patch_description, patch_file_object, mark_for_review=mark_for_review, mark_for_commit_queue=mark_for_commit_queue)
427         response = self.browser.submit()
428
429         bug_id = self._check_create_bug_response(response.read())
430         log("Bug %s created." % bug_id)
431         log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id))
432         return bug_id
433
434     def _find_select_element_for_flag(self, flag_name):
435         # FIXME: This will break if we ever re-order attachment flags
436         if flag_name == "review":
437             return self.browser.find_control(type='select', nr=0)
438         if flag_name == "commit-queue":
439             return self.browser.find_control(type='select', nr=1)
440         raise Exception("Don't know how to find flag named \"%s\"" % flag_name)
441
442     def clear_attachment_flags(self, attachment_id, additional_comment_text=None):
443         self.authenticate()
444
445         comment_text = "Clearing flags on attachment: %s" % attachment_id
446         if additional_comment_text:
447             comment_text += "\n\n%s" % additional_comment_text
448         log(comment_text)
449
450         if self.dryrun:
451             return
452
453         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
454         self.browser.select_form(nr=1)
455         self.browser.set_value(comment_text, name='comment', nr=0)
456         self._find_select_element_for_flag('review').value = ("X",)
457         self._find_select_element_for_flag('commit-queue').value = ("X",)
458         self.browser.submit()
459
460     # FIXME: We need a way to test this on a live bugzilla instance.
461     def _set_flag_on_attachment(self, attachment_id, flag_name, flag_value, comment_text, additional_comment_text):
462         self.authenticate()
463
464         if additional_comment_text:
465             comment_text += "\n\n%s" % additional_comment_text
466         log(comment_text)
467
468         if self.dryrun:
469             return
470
471         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
472         self.browser.select_form(nr=1)
473         self.browser.set_value(comment_text, name='comment', nr=0)
474         self._find_select_element_for_flag(flag_name).value = (flag_value,)
475         self.browser.submit()
476
477     def reject_patch_from_commit_queue(self, attachment_id, additional_comment_text=None):
478         comment_text = "Rejecting patch %s from commit-queue." % attachment_id
479         self._set_flag_on_attachment(attachment_id, 'commit-queue', '-', comment_text, additional_comment_text)
480
481     def reject_patch_from_review_queue(self, attachment_id, additional_comment_text=None):
482         comment_text = "Rejecting patch %s from review queue." % attachment_id
483         self._set_flag_on_attachment(attachment_id, 'review', '-', comment_text, additional_comment_text)
484
485     def obsolete_attachment(self, attachment_id, comment_text = None):
486         self.authenticate()
487
488         log("Obsoleting attachment: %s" % attachment_id)
489         if self.dryrun:
490             log(comment_text)
491             return
492
493         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
494         self.browser.select_form(nr=1)
495         self.browser.find_control('isobsolete').items[0].selected = True
496         # Also clear any review flag (to remove it from review/commit queues)
497         self._find_select_element_for_flag('review').value = ("X",)
498         self._find_select_element_for_flag('commit-queue').value = ("X",)
499         if comment_text:
500             log(comment_text)
501             # Bugzilla has two textareas named 'comment', one is somehow hidden.  We want the first.
502             self.browser.set_value(comment_text, name='comment', nr=0)
503         self.browser.submit()
504     
505     def post_comment_to_bug(self, bug_id, comment_text):
506         self.authenticate()
507
508         log("Adding comment to bug %s" % bug_id)
509         if self.dryrun:
510             log(comment_text)
511             return
512
513         self.browser.open(self.bug_url_for_bug_id(bug_id))
514         self.browser.select_form(name="changeform")
515         self.browser['comment'] = comment_text
516         self.browser.submit()
517
518     def close_bug_as_fixed(self, bug_id, comment_text=None):
519         self.authenticate()
520
521         log("Closing bug %s as fixed" % bug_id)
522         if self.dryrun:
523             log(comment_text)
524             return
525
526         self.browser.open(self.bug_url_for_bug_id(bug_id))
527         self.browser.select_form(name="changeform")
528         if comment_text:
529             log(comment_text)
530             self.browser['comment'] = comment_text
531         self.browser['bug_status'] = ['RESOLVED']
532         self.browser['resolution'] = ['FIXED']
533         self.browser.submit()
534
535     def reopen_bug(self, bug_id, comment_text):
536         self.authenticate()
537
538         log("Re-opening bug %s" % bug_id)
539         log(comment_text) # Bugzilla requires a comment when re-opening a bug, so we know it will never be None.
540         if self.dryrun:
541             return
542
543         self.browser.open(self.bug_url_for_bug_id(bug_id))
544         self.browser.select_form(name="changeform")
545         self.browser['bug_status'] = ['REOPENED']
546         self.browser['comment'] = comment_text
547         self.browser.submit()