651e1b374346885f0c8c2a8437b94bcbe8928197
[WebKit-https.git] / Tools / Scripts / webkitpy / common / net / bugzilla / bugzilla.py
1 # Copyright (c) 2011 Google Inc. All rights reserved.
2 # Copyright (c) 2009 Apple Inc. All rights reserved.
3 # Copyright (c) 2010 Research In Motion Limited. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 #
31 # WebKit's Python module for interacting with Bugzilla
32
33 import mimetypes
34 import re
35 import StringIO
36 import socket
37 import urllib
38
39 from datetime import datetime # used in timestamp()
40
41 from .attachment import Attachment
42 from .bug import Bug
43
44 from webkitpy.common.system.deprecated_logging import log
45 from webkitpy.common.config import committers
46 import webkitpy.common.config.urls as config_urls
47 from webkitpy.common.net.credentials import Credentials
48 from webkitpy.common.system.user import User
49 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, SoupStrainer
50
51
52 class EditUsersParser(object):
53     def __init__(self):
54         self._group_name_to_group_string_cache = {}
55
56     def _login_and_uid_from_row(self, row):
57         first_cell = row.find("td")
58         # The first row is just headers, we skip it.
59         if not first_cell:
60             return None
61         # When there were no results, we have a fake "<none>" entry in the table.
62         if first_cell.find(text="<none>"):
63             return None
64         # Otherwise the <td> contains a single <a> which contains the login name or a single <i> with the string "<none>".
65         anchor_tag = first_cell.find("a")
66         login = unicode(anchor_tag.string).strip()
67         user_id = int(re.search(r"userid=(\d+)", str(anchor_tag['href'])).group(1))
68         return (login, user_id)
69
70     def login_userid_pairs_from_edit_user_results(self, results_page):
71         soup = BeautifulSoup(results_page, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
72         results_table = soup.find(id="admin_table")
73         login_userid_pairs = [self._login_and_uid_from_row(row) for row in results_table('tr')]
74         # Filter out None from the logins.
75         return filter(lambda pair: bool(pair), login_userid_pairs)
76
77     def _group_name_and_string_from_row(self, row):
78         label_element = row.find('label')
79         group_string = unicode(label_element['for'])
80         group_name = unicode(label_element.find('strong').string).rstrip(':')
81         return (group_name, group_string)
82
83     def user_dict_from_edit_user_page(self, page):
84         soup = BeautifulSoup(page, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
85         user_table = soup.find("table", {'class': 'main'})
86         user_dict = {}
87         for row in user_table('tr'):
88             label_element = row.find('label')
89             if not label_element:
90                 continue  # This must not be a row we know how to parse.
91             if row.find('table'):
92                 continue  # Skip the <tr> holding the groups table.
93
94             key = label_element['for']
95             if "group" in key:
96                 key = "groups"
97                 value = user_dict.get('groups', set())
98                 # We must be parsing a "tr" inside the inner group table.
99                 (group_name, _) = self._group_name_and_string_from_row(row)
100                 if row.find('input', {'type': 'checkbox', 'checked': 'checked'}):
101                     value.add(group_name)
102             else:
103                 value = unicode(row.find('td').string).strip()
104             user_dict[key] = value
105         return user_dict
106
107     def _group_rows_from_edit_user_page(self, edit_user_page):
108         soup = BeautifulSoup(edit_user_page, convertEntities=BeautifulSoup.HTML_ENTITIES)
109         return soup('td', {'class': 'groupname'})
110
111     def group_string_from_name(self, edit_user_page, group_name):
112         # Bugzilla uses "group_NUMBER" strings, which may be different per install
113         # so we just look them up once and cache them.
114         if not self._group_name_to_group_string_cache:
115             rows = self._group_rows_from_edit_user_page(edit_user_page)
116             name_string_pairs = map(self._group_name_and_string_from_row, rows)
117             self._group_name_to_group_string_cache = dict(name_string_pairs)
118         return self._group_name_to_group_string_cache[group_name]
119
120
121 def timestamp():
122     return datetime.now().strftime("%Y%m%d%H%M%S")
123
124
125 # A container for all of the logic for making and parsing bugzilla queries.
126 class BugzillaQueries(object):
127
128     def __init__(self, bugzilla):
129         self._bugzilla = bugzilla
130
131     def _is_xml_bugs_form(self, form):
132         # ClientForm.HTMLForm.find_control throws if the control is not found,
133         # so we do a manual search instead:
134         return "xml" in [control.id for control in form.controls]
135
136     # This is kinda a hack.  There is probably a better way to get this information from bugzilla.
137     def _parse_result_count(self, results_page):
138         result_count_text = BeautifulSoup(results_page).find(attrs={'class': 'bz_result_count'}).string
139         result_count_parts = result_count_text.strip().split(" ")
140         if result_count_parts[0] == "Zarro":
141             return 0
142         if result_count_parts[0] == "One":
143             return 1
144         return int(result_count_parts[0])
145
146     # Note: _load_query, _fetch_bug and _fetch_bugs_from_advanced_query
147     # are the only methods which access self._bugzilla.
148
149     def _load_query(self, query):
150         self._bugzilla.authenticate()
151         full_url = "%s%s" % (config_urls.bug_server_url, query)
152         return self._bugzilla.browser.open(full_url)
153
154     def _fetch_bugs_from_advanced_query(self, query):
155         results_page = self._load_query(query)
156         # Some simple searches can return a single result.
157         results_url = results_page.geturl()
158         if results_url.find("/show_bug.cgi?id=") != -1:
159             bug_id = int(results_url.split("=")[-1])
160             return [self._fetch_bug(bug_id)]
161         if not self._parse_result_count(results_page):
162             return []
163         # Bugzilla results pages have an "XML" submit button at the bottom
164         # which can be used to get an XML page containing all of the <bug> elements.
165         # This is slighty lame that this assumes that _load_query used
166         # self._bugzilla.browser and that it's in an acceptable state.
167         self._bugzilla.browser.select_form(predicate=self._is_xml_bugs_form)
168         bugs_xml = self._bugzilla.browser.submit()
169         return self._bugzilla._parse_bugs_from_xml(bugs_xml)
170
171     def _fetch_bug(self, bug_id):
172         return self._bugzilla.fetch_bug(bug_id)
173
174     def _fetch_bug_ids_advanced_query(self, query):
175         soup = BeautifulSoup(self._load_query(query))
176         # The contents of the <a> inside the cells in the first column happen
177         # to be the bug id.
178         return [int(bug_link_cell.find("a").string)
179                 for bug_link_cell in soup('td', "first-child")]
180
181     def _parse_attachment_ids_request_query(self, page):
182         digits = re.compile("\d+")
183         attachment_href = re.compile("attachment.cgi\?id=\d+&action=review")
184         attachment_links = SoupStrainer("a", href=attachment_href)
185         return [int(digits.search(tag["href"]).group(0))
186                 for tag in BeautifulSoup(page, parseOnlyThese=attachment_links)]
187
188     def _fetch_attachment_ids_request_query(self, query):
189         return self._parse_attachment_ids_request_query(self._load_query(query))
190
191     def _parse_quips(self, page):
192         soup = BeautifulSoup(page, convertEntities=BeautifulSoup.HTML_ENTITIES)
193         quips = soup.find(text=re.compile(r"Existing quips:")).findNext("ul").findAll("li")
194         return [unicode(quip_entry.string) for quip_entry in quips]
195
196     def fetch_quips(self):
197         return self._parse_quips(self._load_query("/quips.cgi?action=show"))
198
199     # List of all r+'d bugs.
200     def fetch_bug_ids_from_pending_commit_list(self):
201         needs_commit_query_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review%2B"
202         return self._fetch_bug_ids_advanced_query(needs_commit_query_url)
203
204     def fetch_bugs_matching_quicksearch(self, search_string):
205         # We may want to use a more explicit query than "quicksearch".
206         # If quicksearch changes we should probably change to use
207         # a normal buglist.cgi?query_format=advanced query.
208         quicksearch_url = "buglist.cgi?quicksearch=%s" % urllib.quote(search_string)
209         return self._fetch_bugs_from_advanced_query(quicksearch_url)
210
211     # Currently this returns all bugs across all components.
212     # In the future we may wish to extend this API to construct more restricted searches.
213     def fetch_bugs_matching_search(self, search_string):
214         query = "buglist.cgi?query_format=advanced"
215         if search_string:
216             query += "&short_desc_type=allwordssubstr&short_desc=%s" % urllib.quote(search_string)
217         return self._fetch_bugs_from_advanced_query(query)
218
219     def fetch_patches_from_pending_commit_list(self):
220         return sum([self._fetch_bug(bug_id).reviewed_patches()
221             for bug_id in self.fetch_bug_ids_from_pending_commit_list()], [])
222
223     def fetch_bugs_from_review_queue(self, cc_email=None):
224         query = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review?"
225
226         if cc_email:
227             query += "&emailcc1=1&emailtype1=substring&email1=%s" % urllib.quote(cc_email)
228
229         return self._fetch_bugs_from_advanced_query(query)
230
231     def fetch_bug_ids_from_commit_queue(self):
232         commit_queue_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=commit-queue%2B&order=Last+Changed"
233         return self._fetch_bug_ids_advanced_query(commit_queue_url)
234
235     def fetch_patches_from_commit_queue(self):
236         # This function will only return patches which have valid committers
237         # set.  It won't reject patches with invalid committers/reviewers.
238         return sum([self._fetch_bug(bug_id).commit_queued_patches()
239                     for bug_id in self.fetch_bug_ids_from_commit_queue()], [])
240
241     def fetch_bug_ids_from_review_queue(self):
242         review_queue_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review?"
243         return self._fetch_bug_ids_advanced_query(review_queue_url)
244
245     # This method will make several requests to bugzilla.
246     def fetch_patches_from_review_queue(self, limit=None):
247         # [:None] returns the whole array.
248         return sum([self._fetch_bug(bug_id).unreviewed_patches()
249             for bug_id in self.fetch_bug_ids_from_review_queue()[:limit]], [])
250
251     # NOTE: This is the only client of _fetch_attachment_ids_request_query
252     # This method only makes one request to bugzilla.
253     def fetch_attachment_ids_from_review_queue(self):
254         review_queue_url = "request.cgi?action=queue&type=review&group=type"
255         return self._fetch_attachment_ids_request_query(review_queue_url)
256
257     # This only works if your account has edituser privileges.
258     # We could easily parse https://bugs.webkit.org/userprefs.cgi?tab=permissions to
259     # check permissions, but bugzilla will just return an error if we don't have them.
260     def fetch_login_userid_pairs_matching_substring(self, search_string):
261         review_queue_url = "editusers.cgi?action=list&matchvalue=login_name&matchstr=%s&matchtype=substr" % urllib.quote(search_string)
262         results_page = self._load_query(review_queue_url)
263         # We could pull the EditUsersParser off Bugzilla if needed.
264         return EditUsersParser().login_userid_pairs_from_edit_user_results(results_page)
265
266     # FIXME: We should consider adding a BugzillaUser class.
267     def fetch_logins_matching_substring(self, search_string):
268         pairs = self.fetch_login_userid_pairs_matching_substring(search_string)
269         return map(lambda pair: pair[0], pairs)
270
271
272 class Bugzilla(object):
273     def __init__(self, committers=committers.CommitterList()):
274         self.authenticated = False
275         self.queries = BugzillaQueries(self)
276         self.committers = committers
277         self.cached_quips = []
278         self.edit_user_parser = EditUsersParser()
279         self._browser = None
280
281     def _get_browser(self):
282         if not self._browser:
283             self.setdefaulttimeout(600)
284             from webkitpy.thirdparty.autoinstalled.mechanize import Browser
285             self._browser = Browser()
286             # Ignore bugs.webkit.org/robots.txt until we fix it to allow this script.
287             self._browser.set_handle_robots(False)
288         return self._browser
289
290     def _set_browser(self, value):
291         self._browser = value
292
293     browser = property(_get_browser, _set_browser)
294
295     def setdefaulttimeout(self, value):
296         socket.setdefaulttimeout(value)
297
298     def fetch_user(self, user_id):
299         self.authenticate()
300         edit_user_page = self.browser.open(self.edit_user_url_for_id(user_id))
301         return self.edit_user_parser.user_dict_from_edit_user_page(edit_user_page)
302
303     def add_user_to_groups(self, user_id, group_names):
304         self.authenticate()
305         user_edit_page = self.browser.open(self.edit_user_url_for_id(user_id))
306         self.browser.select_form(nr=1)
307         for group_name in group_names:
308             group_string = self.edit_user_parser.group_string_from_name(user_edit_page, group_name)
309             self.browser.find_control(group_string).items[0].selected = True
310         self.browser.submit()
311
312     def quips(self):
313         # We only fetch and parse the list of quips once per instantiation
314         # so that we do not burden bugs.webkit.org.
315         if not self.cached_quips:
316             self.cached_quips = self.queries.fetch_quips()
317         return self.cached_quips
318
319     def bug_url_for_bug_id(self, bug_id, xml=False):
320         if not bug_id:
321             return None
322         content_type = "&ctype=xml&excludefield=attachmentdata" if xml else ""
323         return "%sshow_bug.cgi?id=%s%s" % (config_urls.bug_server_url, bug_id, content_type)
324
325     def short_bug_url_for_bug_id(self, bug_id):
326         if not bug_id:
327             return None
328         return "http://webkit.org/b/%s" % bug_id
329
330     def add_attachment_url(self, bug_id):
331         return "%sattachment.cgi?action=enter&bugid=%s" % (config_urls.bug_server_url, bug_id)
332
333     def attachment_url_for_id(self, attachment_id, action="view"):
334         if not attachment_id:
335             return None
336         action_param = ""
337         if action and action != "view":
338             action_param = "&action=%s" % action
339         return "%sattachment.cgi?id=%s%s" % (config_urls.bug_server_url,
340                                              attachment_id,
341                                              action_param)
342
343     def edit_user_url_for_id(self, user_id):
344         return "%seditusers.cgi?action=edit&userid=%s" % (config_urls.bug_server_url, user_id)
345
346     def _parse_attachment_flag(self,
347                                element,
348                                flag_name,
349                                attachment,
350                                result_key):
351         flag = element.find('flag', attrs={'name': flag_name})
352         if flag:
353             attachment[flag_name] = flag['status']
354             if flag['status'] == '+':
355                 attachment[result_key] = flag['setter']
356         # Sadly show_bug.cgi?ctype=xml does not expose the flag modification date.
357
358     def _string_contents(self, soup):
359         # WebKit's bugzilla instance uses UTF-8.
360         # BeautifulStoneSoup always returns Unicode strings, however
361         # the .string method returns a (unicode) NavigableString.
362         # NavigableString can confuse other parts of the code, so we
363         # convert from NavigableString to a real unicode() object using unicode().
364         return unicode(soup.string)
365
366     # Example: 2010-01-20 14:31 PST
367     # FIXME: Some bugzilla dates seem to have seconds in them?
368     # Python does not support timezones out of the box.
369     # Assume that bugzilla always uses PST (which is true for bugs.webkit.org)
370     _bugzilla_date_format = "%Y-%m-%d %H:%M:%S"
371
372     @classmethod
373     def _parse_date(cls, date_string):
374         (date, time, time_zone) = date_string.split(" ")
375         if time.count(':') == 1:
376             # Add seconds into the time.
377             time += ':0'
378         # Ignore the timezone because python doesn't understand timezones out of the box.
379         date_string = "%s %s" % (date, time)
380         return datetime.strptime(date_string, cls._bugzilla_date_format)
381
382     def _date_contents(self, soup):
383         return self._parse_date(self._string_contents(soup))
384
385     def _parse_attachment_element(self, element, bug_id):
386         attachment = {}
387         attachment['bug_id'] = bug_id
388         attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
389         attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
390         attachment['id'] = int(element.find('attachid').string)
391         # FIXME: No need to parse out the url here.
392         attachment['url'] = self.attachment_url_for_id(attachment['id'])
393         attachment["attach_date"] = self._date_contents(element.find("date"))
394         attachment['name'] = self._string_contents(element.find('desc'))
395         attachment['attacher_email'] = self._string_contents(element.find('attacher'))
396         attachment['type'] = self._string_contents(element.find('type'))
397         self._parse_attachment_flag(
398                 element, 'review', attachment, 'reviewer_email')
399         self._parse_attachment_flag(
400                 element, 'commit-queue', attachment, 'committer_email')
401         return attachment
402
403     def _parse_log_descr_element(self, element):
404         comment = {}
405         comment['comment_email'] = self._string_contents(element.find('who'))
406         comment['comment_date'] = self._date_contents(element.find('bug_when'))
407         comment['text'] = self._string_contents(element.find('thetext'))
408         return comment
409
410     def _parse_bugs_from_xml(self, page):
411         soup = BeautifulSoup(page)
412         # Without the unicode() call, BeautifulSoup occasionally complains of being
413         # passed None for no apparent reason.
414         return [Bug(self._parse_bug_dictionary_from_xml(unicode(bug_xml)), self) for bug_xml in soup('bug')]
415
416     def _parse_bug_dictionary_from_xml(self, page):
417         soup = BeautifulStoneSoup(page, convertEntities=BeautifulStoneSoup.XML_ENTITIES)
418         bug = {}
419         bug["id"] = int(soup.find("bug_id").string)
420         bug["title"] = self._string_contents(soup.find("short_desc"))
421         bug["bug_status"] = self._string_contents(soup.find("bug_status"))
422         dup_id = soup.find("dup_id")
423         if dup_id:
424             bug["dup_id"] = self._string_contents(dup_id)
425         bug["reporter_email"] = self._string_contents(soup.find("reporter"))
426         bug["assigned_to_email"] = self._string_contents(soup.find("assigned_to"))
427         bug["cc_emails"] = [self._string_contents(element) for element in soup.findAll('cc')]
428         bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')]
429         bug["comments"] = [self._parse_log_descr_element(element) for element in soup.findAll('long_desc')]
430
431         return bug
432
433     # Makes testing fetch_*_from_bug() possible until we have a better
434     # BugzillaNetwork abstration.
435
436     def _fetch_bug_page(self, bug_id):
437         bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
438         log("Fetching: %s" % bug_url)
439         return self.browser.open(bug_url)
440
441     def fetch_bug_dictionary(self, bug_id):
442         try:
443             return self._parse_bug_dictionary_from_xml(self._fetch_bug_page(bug_id))
444         except KeyboardInterrupt:
445             raise
446         except:
447             self.authenticate()
448             return self._parse_bug_dictionary_from_xml(self._fetch_bug_page(bug_id))
449
450     # FIXME: A BugzillaCache object should provide all these fetch_ methods.
451
452     def fetch_bug(self, bug_id):
453         return Bug(self.fetch_bug_dictionary(bug_id), self)
454
455     def fetch_attachment_contents(self, attachment_id):
456         attachment_url = self.attachment_url_for_id(attachment_id)
457         # We need to authenticate to download patches from security bugs.
458         self.authenticate()
459         return self.browser.open(attachment_url).read()
460
461     def _parse_bug_id_from_attachment_page(self, page):
462         # The "Up" relation happens to point to the bug.
463         up_link = BeautifulSoup(page).find('link', rel='Up')
464         if not up_link:
465             # This attachment does not exist (or you don't have permissions to
466             # view it).
467             return None
468         match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href'])
469         return int(match.group('bug_id'))
470
471     def bug_id_for_attachment_id(self, attachment_id):
472         self.authenticate()
473
474         attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
475         log("Fetching: %s" % attachment_url)
476         page = self.browser.open(attachment_url)
477         return self._parse_bug_id_from_attachment_page(page)
478
479     # FIXME: This should just return Attachment(id), which should be able to
480     # lazily fetch needed data.
481
482     def fetch_attachment(self, attachment_id):
483         # We could grab all the attachment details off of the attachment edit
484         # page but we already have working code to do so off of the bugs page,
485         # so re-use that.
486         bug_id = self.bug_id_for_attachment_id(attachment_id)
487         if not bug_id:
488             return None
489         attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True)
490         for attachment in attachments:
491             if attachment.id() == int(attachment_id):
492                 return attachment
493         return None # This should never be hit.
494
495     def authenticate(self):
496         if self.authenticated:
497             return
498
499         credentials = Credentials(config_urls.bug_server_host, git_prefix="bugzilla")
500
501         attempts = 0
502         while not self.authenticated:
503             attempts += 1
504             username, password = credentials.read_credentials()
505
506             log("Logging in as %s..." % username)
507             self.browser.open(config_urls.bug_server_url +
508                               "index.cgi?GoAheadAndLogIn=1")
509             self.browser.select_form(name="login")
510             self.browser['Bugzilla_login'] = username
511             self.browser['Bugzilla_password'] = password
512             self.browser.find_control("Bugzilla_restrictlogin").items[0].selected = False
513             response = self.browser.submit()
514
515             match = re.search("<title>(.+?)</title>", response.read())
516             # If the resulting page has a title, and it contains the word
517             # "invalid" assume it's the login failure page.
518             if match and re.search("Invalid", match.group(1), re.IGNORECASE):
519                 errorMessage = "Bugzilla login failed: %s" % match.group(1)
520                 # raise an exception only if this was the last attempt
521                 if attempts < 5:
522                     log(errorMessage)
523                 else:
524                     raise Exception(errorMessage)
525             else:
526                 self.authenticated = True
527                 self.username = username
528
529     # FIXME: Use enum instead of two booleans
530     def _commit_queue_flag(self, mark_for_landing, mark_for_commit_queue):
531         if mark_for_landing:
532             user = self.committers.account_by_email(self.username)
533             mark_for_commit_queue = True
534             if not user:
535                 log("Your Bugzilla login is not listed in committers.py. Uploading with cq? instead of cq+")
536                 mark_for_landing = False
537             elif not user.can_commit:
538                 log("You're not a committer yet or haven't updated committers.py yet. Uploading with cq? instead of cq+")
539                 mark_for_landing = False
540
541         if mark_for_landing:
542             return '+'
543         if mark_for_commit_queue:
544             return '?'
545         return 'X'
546
547     # FIXME: mark_for_commit_queue and mark_for_landing should be joined into a single commit_flag argument.
548     def _fill_attachment_form(self,
549                               description,
550                               file_object,
551                               mark_for_review=False,
552                               mark_for_commit_queue=False,
553                               mark_for_landing=False,
554                               is_patch=False,
555                               filename=None,
556                               mimetype=None):
557         self.browser['description'] = description
558         if is_patch:
559             self.browser['ispatch'] = ("1",)
560         # FIXME: Should this use self._find_select_element_for_flag?
561         self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
562         self.browser['flag_type-3'] = (self._commit_queue_flag(mark_for_landing, mark_for_commit_queue),)
563
564         filename = filename or "%s.patch" % timestamp()
565         if not mimetype:
566             mimetypes.add_type('text/plain', '.patch')  # Make sure mimetypes knows about .patch
567             mimetype, _ = mimetypes.guess_type(filename)
568         if not mimetype:
569             mimetype = "text/plain"  # Bugzilla might auto-guess for us and we might not need this?
570         self.browser.add_file(file_object, mimetype, filename, 'data')
571
572     def _file_object_for_upload(self, file_or_string):
573         if hasattr(file_or_string, 'read'):
574             return file_or_string
575         # Only if file_or_string is not already encoded do we want to encode it.
576         if isinstance(file_or_string, unicode):
577             file_or_string = file_or_string.encode('utf-8')
578         return StringIO.StringIO(file_or_string)
579
580     # timestamp argument is just for unittests.
581     def _filename_for_upload(self, file_object, bug_id, extension="txt", timestamp=timestamp):
582         if hasattr(file_object, "name"):
583             return file_object.name
584         return "bug-%s-%s.%s" % (bug_id, timestamp(), extension)
585
586     def add_attachment_to_bug(self, bug_id, file_or_string, description, filename=None, comment_text=None, mimetype=None):
587         self.authenticate()
588         log('Adding attachment "%s" to %s' % (description, self.bug_url_for_bug_id(bug_id)))
589         self.browser.open(self.add_attachment_url(bug_id))
590         self.browser.select_form(name="entryform")
591         file_object = self._file_object_for_upload(file_or_string)
592         filename = filename or self._filename_for_upload(file_object, bug_id)
593         self._fill_attachment_form(description, file_object, filename=filename, mimetype=mimetype)
594         if comment_text:
595             log(comment_text)
596             self.browser['comment'] = comment_text
597         self.browser.submit()
598
599     # FIXME: The arguments to this function should be simplified and then
600     # this should be merged into add_attachment_to_bug
601     def add_patch_to_bug(self,
602                          bug_id,
603                          file_or_string,
604                          description,
605                          comment_text=None,
606                          mark_for_review=False,
607                          mark_for_commit_queue=False,
608                          mark_for_landing=False):
609         self.authenticate()
610         log('Adding patch "%s" to %s' % (description, self.bug_url_for_bug_id(bug_id)))
611
612         self.browser.open(self.add_attachment_url(bug_id))
613         self.browser.select_form(name="entryform")
614         file_object = self._file_object_for_upload(file_or_string)
615         filename = self._filename_for_upload(file_object, bug_id, extension="patch")
616         self._fill_attachment_form(description,
617                                    file_object,
618                                    mark_for_review=mark_for_review,
619                                    mark_for_commit_queue=mark_for_commit_queue,
620                                    mark_for_landing=mark_for_landing,
621                                    is_patch=True,
622                                    filename=filename)
623         if comment_text:
624             log(comment_text)
625             self.browser['comment'] = comment_text
626         self.browser.submit()
627
628     # FIXME: There has to be a more concise way to write this method.
629     def _check_create_bug_response(self, response_html):
630         match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>",
631                           response_html)
632         if match:
633             return match.group('bug_id')
634
635         match = re.search(
636             '<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">',
637             response_html,
638             re.DOTALL)
639         error_message = "FAIL"
640         if match:
641             text_lines = BeautifulSoup(
642                     match.group('error_message')).findAll(text=True)
643             error_message = "\n" + '\n'.join(
644                     ["  " + line.strip()
645                      for line in text_lines if line.strip()])
646         raise Exception("Bug not created: %s" % error_message)
647
648     def create_bug(self,
649                    bug_title,
650                    bug_description,
651                    component=None,
652                    diff=None,
653                    patch_description=None,
654                    cc=None,
655                    blocked=None,
656                    assignee=None,
657                    mark_for_review=False,
658                    mark_for_commit_queue=False):
659         self.authenticate()
660
661         log('Creating bug with title "%s"' % bug_title)
662         self.browser.open(config_urls.bug_server_url + "enter_bug.cgi?product=WebKit")
663         self.browser.select_form(name="Create")
664         component_items = self.browser.find_control('component').items
665         component_names = map(lambda item: item.name, component_items)
666         if not component:
667             component = "New Bugs"
668         if component not in component_names:
669             component = User.prompt_with_list("Please pick a component:", component_names)
670         self.browser["component"] = [component]
671         if cc:
672             self.browser["cc"] = cc
673         if blocked:
674             self.browser["blocked"] = unicode(blocked)
675         if not assignee:
676             assignee = self.username
677         if assignee and not self.browser.find_control("assigned_to").disabled:
678             self.browser["assigned_to"] = assignee
679         self.browser["short_desc"] = bug_title
680         self.browser["comment"] = bug_description
681
682         if diff:
683             # _fill_attachment_form expects a file-like object
684             # Patch files are already binary, so no encoding needed.
685             assert(isinstance(diff, str))
686             patch_file_object = StringIO.StringIO(diff)
687             self._fill_attachment_form(
688                     patch_description,
689                     patch_file_object,
690                     mark_for_review=mark_for_review,
691                     mark_for_commit_queue=mark_for_commit_queue,
692                     is_patch=True)
693
694         response = self.browser.submit()
695
696         bug_id = self._check_create_bug_response(response.read())
697         log("Bug %s created." % bug_id)
698         log("%sshow_bug.cgi?id=%s" % (config_urls.bug_server_url, bug_id))
699         return bug_id
700
701     def _find_select_element_for_flag(self, flag_name):
702         # FIXME: This will break if we ever re-order attachment flags
703         if flag_name == "review":
704             return self.browser.find_control(type='select', nr=0)
705         elif flag_name == "commit-queue":
706             return self.browser.find_control(type='select', nr=1)
707         raise Exception("Don't know how to find flag named \"%s\"" % flag_name)
708
709     def clear_attachment_flags(self,
710                                attachment_id,
711                                additional_comment_text=None):
712         self.authenticate()
713
714         comment_text = "Clearing flags on attachment: %s" % attachment_id
715         if additional_comment_text:
716             comment_text += "\n\n%s" % additional_comment_text
717         log(comment_text)
718
719         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
720         self.browser.select_form(nr=1)
721         self.browser.set_value(comment_text, name='comment', nr=0)
722         self._find_select_element_for_flag('review').value = ("X",)
723         self._find_select_element_for_flag('commit-queue').value = ("X",)
724         self.browser.submit()
725
726     def set_flag_on_attachment(self,
727                                attachment_id,
728                                flag_name,
729                                flag_value,
730                                comment_text=None,
731                                additional_comment_text=None):
732         # FIXME: We need a way to test this function on a live bugzilla
733         # instance.
734
735         self.authenticate()
736
737         # FIXME: additional_comment_text seems useless and should be merged into comment-text.
738         if additional_comment_text:
739             comment_text += "\n\n%s" % additional_comment_text
740         log(comment_text)
741
742         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
743         self.browser.select_form(nr=1)
744
745         if comment_text:
746             self.browser.set_value(comment_text, name='comment', nr=0)
747
748         self._find_select_element_for_flag(flag_name).value = (flag_value,)
749         self.browser.submit()
750
751     # FIXME: All of these bug editing methods have a ridiculous amount of
752     # copy/paste code.
753
754     def obsolete_attachment(self, attachment_id, comment_text=None):
755         self.authenticate()
756
757         log("Obsoleting attachment: %s" % attachment_id)
758         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
759         self.browser.select_form(nr=1)
760         self.browser.find_control('isobsolete').items[0].selected = True
761         # Also clear any review flag (to remove it from review/commit queues)
762         self._find_select_element_for_flag('review').value = ("X",)
763         self._find_select_element_for_flag('commit-queue').value = ("X",)
764         if comment_text:
765             log(comment_text)
766             # Bugzilla has two textareas named 'comment', one is somehow
767             # hidden.  We want the first.
768             self.browser.set_value(comment_text, name='comment', nr=0)
769         self.browser.submit()
770
771     def add_cc_to_bug(self, bug_id, email_address_list):
772         self.authenticate()
773
774         log("Adding %s to the CC list for bug %s" % (email_address_list, bug_id))
775         self.browser.open(self.bug_url_for_bug_id(bug_id))
776         self.browser.select_form(name="changeform")
777         self.browser["newcc"] = ", ".join(email_address_list)
778         self.browser.submit()
779
780     def post_comment_to_bug(self, bug_id, comment_text, cc=None):
781         self.authenticate()
782
783         log("Adding comment to bug %s" % bug_id)
784         self.browser.open(self.bug_url_for_bug_id(bug_id))
785         self.browser.select_form(name="changeform")
786         self.browser["comment"] = comment_text
787         if cc:
788             self.browser["newcc"] = ", ".join(cc)
789         self.browser.submit()
790
791     def close_bug_as_fixed(self, bug_id, comment_text=None):
792         self.authenticate()
793
794         log("Closing bug %s as fixed" % bug_id)
795         self.browser.open(self.bug_url_for_bug_id(bug_id))
796         self.browser.select_form(name="changeform")
797         if comment_text:
798             self.browser['comment'] = comment_text
799         self.browser['bug_status'] = ['RESOLVED']
800         self.browser['resolution'] = ['FIXED']
801         self.browser.submit()
802
803     def _has_control(self, form, id):
804         return id in [control.id for control in form.controls]
805
806     def reassign_bug(self, bug_id, assignee=None, comment_text=None):
807         self.authenticate()
808
809         if not assignee:
810             assignee = self.username
811
812         log("Assigning bug %s to %s" % (bug_id, assignee))
813         self.browser.open(self.bug_url_for_bug_id(bug_id))
814         self.browser.select_form(name="changeform")
815
816         if not self._has_control(self.browser, "assigned_to"):
817             log("""Failed to assign bug to you (can't find assigned_to) control.
818 Do you have EditBugs privileges at bugs.webkit.org?
819 https://bugs.webkit.org/userprefs.cgi?tab=permissions
820
821 If not, you should email webkit-committers@lists.webkit.org or ask in #webkit
822 for someone to add EditBugs to your bugs.webkit.org account.""")
823             return
824
825         if comment_text:
826             log(comment_text)
827             self.browser["comment"] = comment_text
828         self.browser["assigned_to"] = assignee
829         self.browser.submit()
830
831     def reopen_bug(self, bug_id, comment_text):
832         self.authenticate()
833
834         log("Re-opening bug %s" % bug_id)
835         # Bugzilla requires a comment when re-opening a bug, so we know it will
836         # never be None.
837         log(comment_text)
838         self.browser.open(self.bug_url_for_bug_id(bug_id))
839         self.browser.select_form(name="changeform")
840         bug_status = self.browser.find_control("bug_status", type="select")
841         # This is a hack around the fact that ClientForm.ListControl seems to
842         # have no simpler way to ask if a control has an item named "REOPENED"
843         # without using exceptions for control flow.
844         possible_bug_statuses = map(lambda item: item.name, bug_status.items)
845         if "REOPENED" in possible_bug_statuses:
846             bug_status.value = ["REOPENED"]
847         # If the bug was never confirmed it will not have a "REOPENED"
848         # state, but only an "UNCONFIRMED" state.
849         elif "UNCONFIRMED" in possible_bug_statuses:
850             bug_status.value = ["UNCONFIRMED"]
851         else:
852             # FIXME: This logic is slightly backwards.  We won't print this
853             # message if the bug is already open with state "UNCONFIRMED".
854             log("Did not reopen bug %s, it appears to already be open with status %s." % (bug_id, bug_status.value))
855         self.browser['comment'] = comment_text
856         self.browser.submit()