Remove use of deprecated logging from webkitpy.common and webkitpy.layout_tests
[WebKit-https.git] / Tools / Scripts / webkitpy / common / net / bugzilla / bugzilla.py
1 # Copyright (c) 2011 Google Inc. All rights reserved.
2 # Copyright (c) 2009 Apple Inc. All rights reserved.
3 # Copyright (c) 2010 Research In Motion Limited. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 #
31 # WebKit's Python module for interacting with Bugzilla
32
33 import logging
34 import mimetypes
35 import re
36 import StringIO
37 import socket
38 import urllib
39
40 from datetime import datetime # used in timestamp()
41
42 from .attachment import Attachment
43 from .bug import Bug
44
45 from webkitpy.common.config import committers
46 import webkitpy.common.config.urls as config_urls
47 from webkitpy.common.net.credentials import Credentials
48 from webkitpy.common.system.user import User
49 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, SoupStrainer
50
51 _log = logging.getLogger(__name__)
52
53
54 class EditUsersParser(object):
55     def __init__(self):
56         self._group_name_to_group_string_cache = {}
57
58     def _login_and_uid_from_row(self, row):
59         first_cell = row.find("td")
60         # The first row is just headers, we skip it.
61         if not first_cell:
62             return None
63         # When there were no results, we have a fake "<none>" entry in the table.
64         if first_cell.find(text="<none>"):
65             return None
66         # Otherwise the <td> contains a single <a> which contains the login name or a single <i> with the string "<none>".
67         anchor_tag = first_cell.find("a")
68         login = unicode(anchor_tag.string).strip()
69         user_id = int(re.search(r"userid=(\d+)", str(anchor_tag['href'])).group(1))
70         return (login, user_id)
71
72     def login_userid_pairs_from_edit_user_results(self, results_page):
73         soup = BeautifulSoup(results_page, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
74         results_table = soup.find(id="admin_table")
75         login_userid_pairs = [self._login_and_uid_from_row(row) for row in results_table('tr')]
76         # Filter out None from the logins.
77         return filter(lambda pair: bool(pair), login_userid_pairs)
78
79     def _group_name_and_string_from_row(self, row):
80         label_element = row.find('label')
81         group_string = unicode(label_element['for'])
82         group_name = unicode(label_element.find('strong').string).rstrip(':')
83         return (group_name, group_string)
84
85     def user_dict_from_edit_user_page(self, page):
86         soup = BeautifulSoup(page, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
87         user_table = soup.find("table", {'class': 'main'})
88         user_dict = {}
89         for row in user_table('tr'):
90             label_element = row.find('label')
91             if not label_element:
92                 continue  # This must not be a row we know how to parse.
93             if row.find('table'):
94                 continue  # Skip the <tr> holding the groups table.
95
96             key = label_element['for']
97             if "group" in key:
98                 key = "groups"
99                 value = user_dict.get('groups', set())
100                 # We must be parsing a "tr" inside the inner group table.
101                 (group_name, _) = self._group_name_and_string_from_row(row)
102                 if row.find('input', {'type': 'checkbox', 'checked': 'checked'}):
103                     value.add(group_name)
104             else:
105                 value = unicode(row.find('td').string).strip()
106             user_dict[key] = value
107         return user_dict
108
109     def _group_rows_from_edit_user_page(self, edit_user_page):
110         soup = BeautifulSoup(edit_user_page, convertEntities=BeautifulSoup.HTML_ENTITIES)
111         return soup('td', {'class': 'groupname'})
112
113     def group_string_from_name(self, edit_user_page, group_name):
114         # Bugzilla uses "group_NUMBER" strings, which may be different per install
115         # so we just look them up once and cache them.
116         if not self._group_name_to_group_string_cache:
117             rows = self._group_rows_from_edit_user_page(edit_user_page)
118             name_string_pairs = map(self._group_name_and_string_from_row, rows)
119             self._group_name_to_group_string_cache = dict(name_string_pairs)
120         return self._group_name_to_group_string_cache[group_name]
121
122
123 def timestamp():
124     return datetime.now().strftime("%Y%m%d%H%M%S")
125
126
127 # A container for all of the logic for making and parsing bugzilla queries.
128 class BugzillaQueries(object):
129
130     def __init__(self, bugzilla):
131         self._bugzilla = bugzilla
132
133     def _is_xml_bugs_form(self, form):
134         # ClientForm.HTMLForm.find_control throws if the control is not found,
135         # so we do a manual search instead:
136         return "xml" in [control.id for control in form.controls]
137
138     # This is kinda a hack.  There is probably a better way to get this information from bugzilla.
139     def _parse_result_count(self, results_page):
140         result_count_text = BeautifulSoup(results_page).find(attrs={'class': 'bz_result_count'}).string
141         result_count_parts = result_count_text.strip().split(" ")
142         if result_count_parts[0] == "Zarro":
143             return 0
144         if result_count_parts[0] == "One":
145             return 1
146         return int(result_count_parts[0])
147
148     # Note: _load_query, _fetch_bug and _fetch_bugs_from_advanced_query
149     # are the only methods which access self._bugzilla.
150
151     def _load_query(self, query):
152         self._bugzilla.authenticate()
153         full_url = "%s%s" % (config_urls.bug_server_url, query)
154         return self._bugzilla.browser.open(full_url)
155
156     def _fetch_bugs_from_advanced_query(self, query):
157         results_page = self._load_query(query)
158         # Some simple searches can return a single result.
159         results_url = results_page.geturl()
160         if results_url.find("/show_bug.cgi?id=") != -1:
161             bug_id = int(results_url.split("=")[-1])
162             return [self._fetch_bug(bug_id)]
163         if not self._parse_result_count(results_page):
164             return []
165         # Bugzilla results pages have an "XML" submit button at the bottom
166         # which can be used to get an XML page containing all of the <bug> elements.
167         # This is slighty lame that this assumes that _load_query used
168         # self._bugzilla.browser and that it's in an acceptable state.
169         self._bugzilla.browser.select_form(predicate=self._is_xml_bugs_form)
170         bugs_xml = self._bugzilla.browser.submit()
171         return self._bugzilla._parse_bugs_from_xml(bugs_xml)
172
173     def _fetch_bug(self, bug_id):
174         return self._bugzilla.fetch_bug(bug_id)
175
176     def _fetch_bug_ids_advanced_query(self, query):
177         soup = BeautifulSoup(self._load_query(query))
178         # The contents of the <a> inside the cells in the first column happen
179         # to be the bug id.
180         return [int(bug_link_cell.find("a").string)
181                 for bug_link_cell in soup('td', "first-child")]
182
183     def _parse_attachment_ids_request_query(self, page):
184         digits = re.compile("\d+")
185         attachment_href = re.compile("attachment.cgi\?id=\d+&action=review")
186         attachment_links = SoupStrainer("a", href=attachment_href)
187         return [int(digits.search(tag["href"]).group(0))
188                 for tag in BeautifulSoup(page, parseOnlyThese=attachment_links)]
189
190     def _fetch_attachment_ids_request_query(self, query):
191         return self._parse_attachment_ids_request_query(self._load_query(query))
192
193     def _parse_quips(self, page):
194         soup = BeautifulSoup(page, convertEntities=BeautifulSoup.HTML_ENTITIES)
195         quips = soup.find(text=re.compile(r"Existing quips:")).findNext("ul").findAll("li")
196         return [unicode(quip_entry.string) for quip_entry in quips]
197
198     def fetch_quips(self):
199         return self._parse_quips(self._load_query("/quips.cgi?action=show"))
200
201     # List of all r+'d bugs.
202     def fetch_bug_ids_from_pending_commit_list(self):
203         needs_commit_query_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review%2B"
204         return self._fetch_bug_ids_advanced_query(needs_commit_query_url)
205
206     def fetch_bugs_matching_quicksearch(self, search_string):
207         # We may want to use a more explicit query than "quicksearch".
208         # If quicksearch changes we should probably change to use
209         # a normal buglist.cgi?query_format=advanced query.
210         quicksearch_url = "buglist.cgi?quicksearch=%s" % urllib.quote(search_string)
211         return self._fetch_bugs_from_advanced_query(quicksearch_url)
212
213     # Currently this returns all bugs across all components.
214     # In the future we may wish to extend this API to construct more restricted searches.
215     def fetch_bugs_matching_search(self, search_string):
216         query = "buglist.cgi?query_format=advanced"
217         if search_string:
218             query += "&short_desc_type=allwordssubstr&short_desc=%s" % urllib.quote(search_string)
219         return self._fetch_bugs_from_advanced_query(query)
220
221     def fetch_patches_from_pending_commit_list(self):
222         return sum([self._fetch_bug(bug_id).reviewed_patches()
223             for bug_id in self.fetch_bug_ids_from_pending_commit_list()], [])
224
225     def fetch_bugs_from_review_queue(self, cc_email=None):
226         query = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review?"
227
228         if cc_email:
229             query += "&emailcc1=1&emailtype1=substring&email1=%s" % urllib.quote(cc_email)
230
231         return self._fetch_bugs_from_advanced_query(query)
232
233     def fetch_bug_ids_from_commit_queue(self):
234         commit_queue_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=commit-queue%2B&order=Last+Changed"
235         return self._fetch_bug_ids_advanced_query(commit_queue_url)
236
237     def fetch_patches_from_commit_queue(self):
238         # This function will only return patches which have valid committers
239         # set.  It won't reject patches with invalid committers/reviewers.
240         return sum([self._fetch_bug(bug_id).commit_queued_patches()
241                     for bug_id in self.fetch_bug_ids_from_commit_queue()], [])
242
243     def fetch_bug_ids_from_review_queue(self):
244         review_queue_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review?"
245         return self._fetch_bug_ids_advanced_query(review_queue_url)
246
247     # This method will make several requests to bugzilla.
248     def fetch_patches_from_review_queue(self, limit=None):
249         # [:None] returns the whole array.
250         return sum([self._fetch_bug(bug_id).unreviewed_patches()
251             for bug_id in self.fetch_bug_ids_from_review_queue()[:limit]], [])
252
253     # NOTE: This is the only client of _fetch_attachment_ids_request_query
254     # This method only makes one request to bugzilla.
255     def fetch_attachment_ids_from_review_queue(self):
256         review_queue_url = "request.cgi?action=queue&type=review&group=type"
257         return self._fetch_attachment_ids_request_query(review_queue_url)
258
259     # This only works if your account has edituser privileges.
260     # We could easily parse https://bugs.webkit.org/userprefs.cgi?tab=permissions to
261     # check permissions, but bugzilla will just return an error if we don't have them.
262     def fetch_login_userid_pairs_matching_substring(self, search_string):
263         review_queue_url = "editusers.cgi?action=list&matchvalue=login_name&matchstr=%s&matchtype=substr" % urllib.quote(search_string)
264         results_page = self._load_query(review_queue_url)
265         # We could pull the EditUsersParser off Bugzilla if needed.
266         return EditUsersParser().login_userid_pairs_from_edit_user_results(results_page)
267
268     # FIXME: We should consider adding a BugzillaUser class.
269     def fetch_logins_matching_substring(self, search_string):
270         pairs = self.fetch_login_userid_pairs_matching_substring(search_string)
271         return map(lambda pair: pair[0], pairs)
272
273
274 class Bugzilla(object):
275     def __init__(self, committers=committers.CommitterList()):
276         self.authenticated = False
277         self.queries = BugzillaQueries(self)
278         self.committers = committers
279         self.cached_quips = []
280         self.edit_user_parser = EditUsersParser()
281         self._browser = None
282
283     def _get_browser(self):
284         if not self._browser:
285             self.setdefaulttimeout(600)
286             from webkitpy.thirdparty.autoinstalled.mechanize import Browser
287             self._browser = Browser()
288             # Ignore bugs.webkit.org/robots.txt until we fix it to allow this script.
289             self._browser.set_handle_robots(False)
290         return self._browser
291
292     def _set_browser(self, value):
293         self._browser = value
294
295     browser = property(_get_browser, _set_browser)
296
297     def setdefaulttimeout(self, value):
298         socket.setdefaulttimeout(value)
299
300     def fetch_user(self, user_id):
301         self.authenticate()
302         edit_user_page = self.browser.open(self.edit_user_url_for_id(user_id))
303         return self.edit_user_parser.user_dict_from_edit_user_page(edit_user_page)
304
305     def add_user_to_groups(self, user_id, group_names):
306         self.authenticate()
307         user_edit_page = self.browser.open(self.edit_user_url_for_id(user_id))
308         self.browser.select_form(nr=1)
309         for group_name in group_names:
310             group_string = self.edit_user_parser.group_string_from_name(user_edit_page, group_name)
311             self.browser.find_control(group_string).items[0].selected = True
312         self.browser.submit()
313
314     def quips(self):
315         # We only fetch and parse the list of quips once per instantiation
316         # so that we do not burden bugs.webkit.org.
317         if not self.cached_quips:
318             self.cached_quips = self.queries.fetch_quips()
319         return self.cached_quips
320
321     def bug_url_for_bug_id(self, bug_id, xml=False):
322         if not bug_id:
323             return None
324         content_type = "&ctype=xml&excludefield=attachmentdata" if xml else ""
325         return "%sshow_bug.cgi?id=%s%s" % (config_urls.bug_server_url, bug_id, content_type)
326
327     def short_bug_url_for_bug_id(self, bug_id):
328         if not bug_id:
329             return None
330         return "http://webkit.org/b/%s" % bug_id
331
332     def add_attachment_url(self, bug_id):
333         return "%sattachment.cgi?action=enter&bugid=%s" % (config_urls.bug_server_url, bug_id)
334
335     def attachment_url_for_id(self, attachment_id, action="view"):
336         if not attachment_id:
337             return None
338         action_param = ""
339         if action and action != "view":
340             action_param = "&action=%s" % action
341         return "%sattachment.cgi?id=%s%s" % (config_urls.bug_server_url,
342                                              attachment_id,
343                                              action_param)
344
345     def edit_user_url_for_id(self, user_id):
346         return "%seditusers.cgi?action=edit&userid=%s" % (config_urls.bug_server_url, user_id)
347
348     def _parse_attachment_flag(self,
349                                element,
350                                flag_name,
351                                attachment,
352                                result_key):
353         flag = element.find('flag', attrs={'name': flag_name})
354         if flag:
355             attachment[flag_name] = flag['status']
356             if flag['status'] == '+':
357                 attachment[result_key] = flag['setter']
358         # Sadly show_bug.cgi?ctype=xml does not expose the flag modification date.
359
360     def _string_contents(self, soup):
361         # WebKit's bugzilla instance uses UTF-8.
362         # BeautifulStoneSoup always returns Unicode strings, however
363         # the .string method returns a (unicode) NavigableString.
364         # NavigableString can confuse other parts of the code, so we
365         # convert from NavigableString to a real unicode() object using unicode().
366         return unicode(soup.string)
367
368     # Example: 2010-01-20 14:31 PST
369     # FIXME: Some bugzilla dates seem to have seconds in them?
370     # Python does not support timezones out of the box.
371     # Assume that bugzilla always uses PST (which is true for bugs.webkit.org)
372     _bugzilla_date_format = "%Y-%m-%d %H:%M:%S"
373
374     @classmethod
375     def _parse_date(cls, date_string):
376         (date, time, time_zone) = date_string.split(" ")
377         if time.count(':') == 1:
378             # Add seconds into the time.
379             time += ':0'
380         # Ignore the timezone because python doesn't understand timezones out of the box.
381         date_string = "%s %s" % (date, time)
382         return datetime.strptime(date_string, cls._bugzilla_date_format)
383
384     def _date_contents(self, soup):
385         return self._parse_date(self._string_contents(soup))
386
387     def _parse_attachment_element(self, element, bug_id):
388         attachment = {}
389         attachment['bug_id'] = bug_id
390         attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
391         attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
392         attachment['id'] = int(element.find('attachid').string)
393         # FIXME: No need to parse out the url here.
394         attachment['url'] = self.attachment_url_for_id(attachment['id'])
395         attachment["attach_date"] = self._date_contents(element.find("date"))
396         attachment['name'] = self._string_contents(element.find('desc'))
397         attachment['attacher_email'] = self._string_contents(element.find('attacher'))
398         attachment['type'] = self._string_contents(element.find('type'))
399         self._parse_attachment_flag(
400                 element, 'review', attachment, 'reviewer_email')
401         self._parse_attachment_flag(
402                 element, 'commit-queue', attachment, 'committer_email')
403         return attachment
404
405     def _parse_log_descr_element(self, element):
406         comment = {}
407         comment['comment_email'] = self._string_contents(element.find('who'))
408         comment['comment_date'] = self._date_contents(element.find('bug_when'))
409         comment['text'] = self._string_contents(element.find('thetext'))
410         return comment
411
412     def _parse_bugs_from_xml(self, page):
413         soup = BeautifulSoup(page)
414         # Without the unicode() call, BeautifulSoup occasionally complains of being
415         # passed None for no apparent reason.
416         return [Bug(self._parse_bug_dictionary_from_xml(unicode(bug_xml)), self) for bug_xml in soup('bug')]
417
418     def _parse_bug_dictionary_from_xml(self, page):
419         soup = BeautifulStoneSoup(page, convertEntities=BeautifulStoneSoup.XML_ENTITIES)
420         bug = {}
421         bug["id"] = int(soup.find("bug_id").string)
422         bug["title"] = self._string_contents(soup.find("short_desc"))
423         bug["bug_status"] = self._string_contents(soup.find("bug_status"))
424         dup_id = soup.find("dup_id")
425         if dup_id:
426             bug["dup_id"] = self._string_contents(dup_id)
427         bug["reporter_email"] = self._string_contents(soup.find("reporter"))
428         bug["assigned_to_email"] = self._string_contents(soup.find("assigned_to"))
429         bug["cc_emails"] = [self._string_contents(element) for element in soup.findAll('cc')]
430         bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')]
431         bug["comments"] = [self._parse_log_descr_element(element) for element in soup.findAll('long_desc')]
432
433         return bug
434
435     # Makes testing fetch_*_from_bug() possible until we have a better
436     # BugzillaNetwork abstration.
437
438     def _fetch_bug_page(self, bug_id):
439         bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
440         _log.info("Fetching: %s" % bug_url)
441         return self.browser.open(bug_url)
442
443     def fetch_bug_dictionary(self, bug_id):
444         try:
445             return self._parse_bug_dictionary_from_xml(self._fetch_bug_page(bug_id))
446         except KeyboardInterrupt:
447             raise
448         except:
449             self.authenticate()
450             return self._parse_bug_dictionary_from_xml(self._fetch_bug_page(bug_id))
451
452     # FIXME: A BugzillaCache object should provide all these fetch_ methods.
453
454     def fetch_bug(self, bug_id):
455         return Bug(self.fetch_bug_dictionary(bug_id), self)
456
457     def fetch_attachment_contents(self, attachment_id):
458         attachment_url = self.attachment_url_for_id(attachment_id)
459         # We need to authenticate to download patches from security bugs.
460         self.authenticate()
461         return self.browser.open(attachment_url).read()
462
463     def _parse_bug_id_from_attachment_page(self, page):
464         # The "Up" relation happens to point to the bug.
465         up_link = BeautifulSoup(page).find('link', rel='Up')
466         if not up_link:
467             # This attachment does not exist (or you don't have permissions to
468             # view it).
469             return None
470         match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href'])
471         return int(match.group('bug_id'))
472
473     def bug_id_for_attachment_id(self, attachment_id):
474         self.authenticate()
475
476         attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
477         _log.info("Fetching: %s" % attachment_url)
478         page = self.browser.open(attachment_url)
479         return self._parse_bug_id_from_attachment_page(page)
480
481     # FIXME: This should just return Attachment(id), which should be able to
482     # lazily fetch needed data.
483
484     def fetch_attachment(self, attachment_id):
485         # We could grab all the attachment details off of the attachment edit
486         # page but we already have working code to do so off of the bugs page,
487         # so re-use that.
488         bug_id = self.bug_id_for_attachment_id(attachment_id)
489         if not bug_id:
490             return None
491         attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True)
492         for attachment in attachments:
493             if attachment.id() == int(attachment_id):
494                 return attachment
495         return None # This should never be hit.
496
497     def authenticate(self):
498         if self.authenticated:
499             return
500
501         credentials = Credentials(config_urls.bug_server_host, git_prefix="bugzilla")
502
503         attempts = 0
504         while not self.authenticated:
505             attempts += 1
506             username, password = credentials.read_credentials()
507
508             _log.info("Logging in as %s..." % username)
509             self.browser.open(config_urls.bug_server_url +
510                               "index.cgi?GoAheadAndLogIn=1")
511             self.browser.select_form(name="login")
512             self.browser['Bugzilla_login'] = username
513             self.browser['Bugzilla_password'] = password
514             self.browser.find_control("Bugzilla_restrictlogin").items[0].selected = False
515             response = self.browser.submit()
516
517             match = re.search("<title>(.+?)</title>", response.read())
518             # If the resulting page has a title, and it contains the word
519             # "invalid" assume it's the login failure page.
520             if match and re.search("Invalid", match.group(1), re.IGNORECASE):
521                 errorMessage = "Bugzilla login failed: %s" % match.group(1)
522                 # raise an exception only if this was the last attempt
523                 if attempts < 5:
524                     _log.error(errorMessage)
525                 else:
526                     raise Exception(errorMessage)
527             else:
528                 self.authenticated = True
529                 self.username = username
530
531     # FIXME: Use enum instead of two booleans
532     def _commit_queue_flag(self, mark_for_landing, mark_for_commit_queue):
533         if mark_for_landing:
534             user = self.committers.account_by_email(self.username)
535             mark_for_commit_queue = True
536             if not user:
537                 _log.warning("Your Bugzilla login is not listed in committers.py. Uploading with cq? instead of cq+")
538                 mark_for_landing = False
539             elif not user.can_commit:
540                 _log.warning("You're not a committer yet or haven't updated committers.py yet. Uploading with cq? instead of cq+")
541                 mark_for_landing = False
542
543         if mark_for_landing:
544             return '+'
545         if mark_for_commit_queue:
546             return '?'
547         return 'X'
548
549     # FIXME: mark_for_commit_queue and mark_for_landing should be joined into a single commit_flag argument.
550     def _fill_attachment_form(self,
551                               description,
552                               file_object,
553                               mark_for_review=False,
554                               mark_for_commit_queue=False,
555                               mark_for_landing=False,
556                               is_patch=False,
557                               filename=None,
558                               mimetype=None):
559         self.browser['description'] = description
560         if is_patch:
561             self.browser['ispatch'] = ("1",)
562         # FIXME: Should this use self._find_select_element_for_flag?
563         self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
564         self.browser['flag_type-3'] = (self._commit_queue_flag(mark_for_landing, mark_for_commit_queue),)
565
566         filename = filename or "%s.patch" % timestamp()
567         if not mimetype:
568             mimetypes.add_type('text/plain', '.patch')  # Make sure mimetypes knows about .patch
569             mimetype, _ = mimetypes.guess_type(filename)
570         if not mimetype:
571             mimetype = "text/plain"  # Bugzilla might auto-guess for us and we might not need this?
572         self.browser.add_file(file_object, mimetype, filename, 'data')
573
574     def _file_object_for_upload(self, file_or_string):
575         if hasattr(file_or_string, 'read'):
576             return file_or_string
577         # Only if file_or_string is not already encoded do we want to encode it.
578         if isinstance(file_or_string, unicode):
579             file_or_string = file_or_string.encode('utf-8')
580         return StringIO.StringIO(file_or_string)
581
582     # timestamp argument is just for unittests.
583     def _filename_for_upload(self, file_object, bug_id, extension="txt", timestamp=timestamp):
584         if hasattr(file_object, "name"):
585             return file_object.name
586         return "bug-%s-%s.%s" % (bug_id, timestamp(), extension)
587
588     def add_attachment_to_bug(self, bug_id, file_or_string, description, filename=None, comment_text=None, mimetype=None):
589         self.authenticate()
590         _log.info('Adding attachment "%s" to %s' % (description, self.bug_url_for_bug_id(bug_id)))
591         self.browser.open(self.add_attachment_url(bug_id))
592         self.browser.select_form(name="entryform")
593         file_object = self._file_object_for_upload(file_or_string)
594         filename = filename or self._filename_for_upload(file_object, bug_id)
595         self._fill_attachment_form(description, file_object, filename=filename, mimetype=mimetype)
596         if comment_text:
597             _log.info(comment_text)
598             self.browser['comment'] = comment_text
599         self.browser.submit()
600
601     # FIXME: The arguments to this function should be simplified and then
602     # this should be merged into add_attachment_to_bug
603     def add_patch_to_bug(self,
604                          bug_id,
605                          file_or_string,
606                          description,
607                          comment_text=None,
608                          mark_for_review=False,
609                          mark_for_commit_queue=False,
610                          mark_for_landing=False):
611         self.authenticate()
612         _log.info('Adding patch "%s" to %s' % (description, self.bug_url_for_bug_id(bug_id)))
613
614         self.browser.open(self.add_attachment_url(bug_id))
615         self.browser.select_form(name="entryform")
616         file_object = self._file_object_for_upload(file_or_string)
617         filename = self._filename_for_upload(file_object, bug_id, extension="patch")
618         self._fill_attachment_form(description,
619                                    file_object,
620                                    mark_for_review=mark_for_review,
621                                    mark_for_commit_queue=mark_for_commit_queue,
622                                    mark_for_landing=mark_for_landing,
623                                    is_patch=True,
624                                    filename=filename)
625         if comment_text:
626             _log.info(comment_text)
627             self.browser['comment'] = comment_text
628         self.browser.submit()
629
630     # FIXME: There has to be a more concise way to write this method.
631     def _check_create_bug_response(self, response_html):
632         match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>",
633                           response_html)
634         if match:
635             return match.group('bug_id')
636
637         match = re.search(
638             '<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">',
639             response_html,
640             re.DOTALL)
641         error_message = "FAIL"
642         if match:
643             text_lines = BeautifulSoup(
644                     match.group('error_message')).findAll(text=True)
645             error_message = "\n" + '\n'.join(
646                     ["  " + line.strip()
647                      for line in text_lines if line.strip()])
648         raise Exception("Bug not created: %s" % error_message)
649
650     def create_bug(self,
651                    bug_title,
652                    bug_description,
653                    component=None,
654                    diff=None,
655                    patch_description=None,
656                    cc=None,
657                    blocked=None,
658                    assignee=None,
659                    mark_for_review=False,
660                    mark_for_commit_queue=False):
661         self.authenticate()
662
663         _log.info('Creating bug with title "%s"' % bug_title)
664         self.browser.open(config_urls.bug_server_url + "enter_bug.cgi?product=WebKit")
665         self.browser.select_form(name="Create")
666         component_items = self.browser.find_control('component').items
667         component_names = map(lambda item: item.name, component_items)
668         if not component:
669             component = "New Bugs"
670         if component not in component_names:
671             component = User.prompt_with_list("Please pick a component:", component_names)
672         self.browser["component"] = [component]
673         if cc:
674             self.browser["cc"] = cc
675         if blocked:
676             self.browser["blocked"] = unicode(blocked)
677         if not assignee:
678             assignee = self.username
679         if assignee and not self.browser.find_control("assigned_to").disabled:
680             self.browser["assigned_to"] = assignee
681         self.browser["short_desc"] = bug_title
682         self.browser["comment"] = bug_description
683
684         if diff:
685             # _fill_attachment_form expects a file-like object
686             # Patch files are already binary, so no encoding needed.
687             assert(isinstance(diff, str))
688             patch_file_object = StringIO.StringIO(diff)
689             self._fill_attachment_form(
690                     patch_description,
691                     patch_file_object,
692                     mark_for_review=mark_for_review,
693                     mark_for_commit_queue=mark_for_commit_queue,
694                     is_patch=True)
695
696         response = self.browser.submit()
697
698         bug_id = self._check_create_bug_response(response.read())
699         _log.info("Bug %s created." % bug_id)
700         _log.info("%sshow_bug.cgi?id=%s" % (config_urls.bug_server_url, bug_id))
701         return bug_id
702
703     def _find_select_element_for_flag(self, flag_name):
704         # FIXME: This will break if we ever re-order attachment flags
705         if flag_name == "review":
706             return self.browser.find_control(type='select', nr=0)
707         elif flag_name == "commit-queue":
708             return self.browser.find_control(type='select', nr=1)
709         raise Exception("Don't know how to find flag named \"%s\"" % flag_name)
710
711     def clear_attachment_flags(self,
712                                attachment_id,
713                                additional_comment_text=None):
714         self.authenticate()
715
716         comment_text = "Clearing flags on attachment: %s" % attachment_id
717         if additional_comment_text:
718             comment_text += "\n\n%s" % additional_comment_text
719         _log.info(comment_text)
720
721         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
722         self.browser.select_form(nr=1)
723         self.browser.set_value(comment_text, name='comment', nr=0)
724         self._find_select_element_for_flag('review').value = ("X",)
725         self._find_select_element_for_flag('commit-queue').value = ("X",)
726         self.browser.submit()
727
728     def set_flag_on_attachment(self,
729                                attachment_id,
730                                flag_name,
731                                flag_value,
732                                comment_text=None,
733                                additional_comment_text=None):
734         # FIXME: We need a way to test this function on a live bugzilla
735         # instance.
736
737         self.authenticate()
738
739         # FIXME: additional_comment_text seems useless and should be merged into comment-text.
740         if additional_comment_text:
741             comment_text += "\n\n%s" % additional_comment_text
742         _log.info(comment_text)
743
744         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
745         self.browser.select_form(nr=1)
746
747         if comment_text:
748             self.browser.set_value(comment_text, name='comment', nr=0)
749
750         self._find_select_element_for_flag(flag_name).value = (flag_value,)
751         self.browser.submit()
752
753     # FIXME: All of these bug editing methods have a ridiculous amount of
754     # copy/paste code.
755
756     def obsolete_attachment(self, attachment_id, comment_text=None):
757         self.authenticate()
758
759         _log.info("Obsoleting attachment: %s" % attachment_id)
760         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
761         self.browser.select_form(nr=1)
762         self.browser.find_control('isobsolete').items[0].selected = True
763         # Also clear any review flag (to remove it from review/commit queues)
764         self._find_select_element_for_flag('review').value = ("X",)
765         self._find_select_element_for_flag('commit-queue').value = ("X",)
766         if comment_text:
767             _log.info(comment_text)
768             # Bugzilla has two textareas named 'comment', one is somehow
769             # hidden.  We want the first.
770             self.browser.set_value(comment_text, name='comment', nr=0)
771         self.browser.submit()
772
773     def add_cc_to_bug(self, bug_id, email_address_list):
774         self.authenticate()
775
776         _log.info("Adding %s to the CC list for bug %s" % (email_address_list, bug_id))
777         self.browser.open(self.bug_url_for_bug_id(bug_id))
778         self.browser.select_form(name="changeform")
779         self.browser["newcc"] = ", ".join(email_address_list)
780         self.browser.submit()
781
782     def post_comment_to_bug(self, bug_id, comment_text, cc=None):
783         self.authenticate()
784
785         _log.info("Adding comment to bug %s" % bug_id)
786         self.browser.open(self.bug_url_for_bug_id(bug_id))
787         self.browser.select_form(name="changeform")
788         self.browser["comment"] = comment_text
789         if cc:
790             self.browser["newcc"] = ", ".join(cc)
791         self.browser.submit()
792
793     def close_bug_as_fixed(self, bug_id, comment_text=None):
794         self.authenticate()
795
796         _log.info("Closing bug %s as fixed" % bug_id)
797         self.browser.open(self.bug_url_for_bug_id(bug_id))
798         self.browser.select_form(name="changeform")
799         if comment_text:
800             self.browser['comment'] = comment_text
801         self.browser['bug_status'] = ['RESOLVED']
802         self.browser['resolution'] = ['FIXED']
803         self.browser.submit()
804
805     def _has_control(self, form, id):
806         return id in [control.id for control in form.controls]
807
808     def reassign_bug(self, bug_id, assignee=None, comment_text=None):
809         self.authenticate()
810
811         if not assignee:
812             assignee = self.username
813
814         _log.info("Assigning bug %s to %s" % (bug_id, assignee))
815         self.browser.open(self.bug_url_for_bug_id(bug_id))
816         self.browser.select_form(name="changeform")
817
818         if not self._has_control(self.browser, "assigned_to"):
819             _log.warning("""Failed to assign bug to you (can't find assigned_to) control.
820 Do you have EditBugs privileges at bugs.webkit.org?
821 https://bugs.webkit.org/userprefs.cgi?tab=permissions
822
823 If not, you should email webkit-committers@lists.webkit.org or ask in #webkit
824 for someone to add EditBugs to your bugs.webkit.org account.""")
825             return
826
827         if comment_text:
828             _log.info(comment_text)
829             self.browser["comment"] = comment_text
830         self.browser["assigned_to"] = assignee
831         self.browser.submit()
832
833     def reopen_bug(self, bug_id, comment_text):
834         self.authenticate()
835
836         _log.info("Re-opening bug %s" % bug_id)
837         # Bugzilla requires a comment when re-opening a bug, so we know it will
838         # never be None.
839         _log.info(comment_text)
840         self.browser.open(self.bug_url_for_bug_id(bug_id))
841         self.browser.select_form(name="changeform")
842         bug_status = self.browser.find_control("bug_status", type="select")
843         # This is a hack around the fact that ClientForm.ListControl seems to
844         # have no simpler way to ask if a control has an item named "REOPENED"
845         # without using exceptions for control flow.
846         possible_bug_statuses = map(lambda item: item.name, bug_status.items)
847         if "REOPENED" in possible_bug_statuses:
848             bug_status.value = ["REOPENED"]
849         # If the bug was never confirmed it will not have a "REOPENED"
850         # state, but only an "UNCONFIRMED" state.
851         elif "UNCONFIRMED" in possible_bug_statuses:
852             bug_status.value = ["UNCONFIRMED"]
853         else:
854             # FIXME: This logic is slightly backwards.  We won't print this
855             # message if the bug is already open with state "UNCONFIRMED".
856             _log.info("Did not reopen bug %s, it appears to already be open with status %s." % (bug_id, bug_status.value))
857         self.browser['comment'] = comment_text
858         self.browser.submit()