Update scripts to reference contributors.json instead of committers.py in messaging
[WebKit-https.git] / Tools / Scripts / webkitpy / common / net / bugzilla / bugzilla.py
1 # Copyright (c) 2011 Google Inc. All rights reserved.
2 # Copyright (c) 2009 Apple Inc. All rights reserved.
3 # Copyright (c) 2010 Research In Motion Limited. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 #
31 # WebKit's Python module for interacting with Bugzilla
32
33 import logging
34 import mimetypes
35 import re
36 import StringIO
37 import socket
38 import urllib
39
40 from datetime import datetime # used in timestamp()
41
42 from .attachment import Attachment
43 from .bug import Bug
44
45 from webkitpy.common.config import committers
46 import webkitpy.common.config.urls as config_urls
47 from webkitpy.common.net.credentials import Credentials
48 from webkitpy.common.system.user import User
49 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, SoupStrainer
50
51 _log = logging.getLogger(__name__)
52
53
54 class EditUsersParser(object):
55     def __init__(self):
56         self._group_name_to_group_string_cache = {}
57
58     def _login_and_uid_from_row(self, row):
59         first_cell = row.find("td")
60         # The first row is just headers, we skip it.
61         if not first_cell:
62             return None
63         # When there were no results, we have a fake "<none>" entry in the table.
64         if first_cell.find(text="<none>"):
65             return None
66         # Otherwise the <td> contains a single <a> which contains the login name or a single <i> with the string "<none>".
67         anchor_tag = first_cell.find("a")
68         login = unicode(anchor_tag.string).strip()
69         user_id = int(re.search(r"userid=(\d+)", str(anchor_tag['href'])).group(1))
70         return (login, user_id)
71
72     def login_userid_pairs_from_edit_user_results(self, results_page):
73         soup = BeautifulSoup(results_page, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
74         results_table = soup.find(id="admin_table")
75         login_userid_pairs = [self._login_and_uid_from_row(row) for row in results_table('tr')]
76         # Filter out None from the logins.
77         return filter(lambda pair: bool(pair), login_userid_pairs)
78
79     def _group_name_and_string_from_row(self, row):
80         label_element = row.find('label')
81         group_string = unicode(label_element['for'])
82         group_name = unicode(label_element.find('strong').string).rstrip(':')
83         return (group_name, group_string)
84
85     def user_dict_from_edit_user_page(self, page):
86         soup = BeautifulSoup(page, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
87         user_table = soup.find("table", {'class': 'main'})
88         user_dict = {}
89         for row in user_table('tr'):
90             label_element = row.find('label')
91             if not label_element:
92                 continue  # This must not be a row we know how to parse.
93             if row.find('table'):
94                 continue  # Skip the <tr> holding the groups table.
95
96             key = label_element['for']
97             if "group" in key:
98                 key = "groups"
99                 value = user_dict.get('groups', set())
100                 # We must be parsing a "tr" inside the inner group table.
101                 (group_name, _) = self._group_name_and_string_from_row(row)
102                 if row.find('input', {'type': 'checkbox', 'checked': 'checked'}):
103                     value.add(group_name)
104             else:
105                 value = unicode(row.find('td').string).strip()
106             user_dict[key] = value
107         return user_dict
108
109     def _group_rows_from_edit_user_page(self, edit_user_page):
110         soup = BeautifulSoup(edit_user_page, convertEntities=BeautifulSoup.HTML_ENTITIES)
111         return soup('td', {'class': 'groupname'})
112
113     def group_string_from_name(self, edit_user_page, group_name):
114         # Bugzilla uses "group_NUMBER" strings, which may be different per install
115         # so we just look them up once and cache them.
116         if not self._group_name_to_group_string_cache:
117             rows = self._group_rows_from_edit_user_page(edit_user_page)
118             name_string_pairs = map(self._group_name_and_string_from_row, rows)
119             self._group_name_to_group_string_cache = dict(name_string_pairs)
120         return self._group_name_to_group_string_cache[group_name]
121
122
123 def timestamp():
124     return datetime.now().strftime("%Y%m%d%H%M%S")
125
126
127 # A container for all of the logic for making and parsing bugzilla queries.
128 class BugzillaQueries(object):
129
130     def __init__(self, bugzilla):
131         self._bugzilla = bugzilla
132
133     def _is_xml_bugs_form(self, form):
134         # ClientForm.HTMLForm.find_control throws if the control is not found,
135         # so we do a manual search instead:
136         return "xml" in [control.id for control in form.controls]
137
138     # This is kinda a hack.  There is probably a better way to get this information from bugzilla.
139     def _parse_result_count(self, results_page):
140         result_count_text = BeautifulSoup(results_page).find(attrs={'class': 'bz_result_count'}).string
141         result_count_parts = result_count_text.strip().split(" ")
142         if result_count_parts[0] == "Zarro":
143             return 0
144         if result_count_parts[0] == "One":
145             return 1
146         return int(result_count_parts[0])
147
148     # Note: _load_query, _fetch_bug and _fetch_bugs_from_advanced_query
149     # are the only methods which access self._bugzilla.
150
151     def _load_query(self, query):
152         self._bugzilla.authenticate()
153         full_url = "%s%s" % (config_urls.bug_server_url, query)
154         return self._bugzilla.browser.open(full_url)
155
156     def _fetch_bugs_from_advanced_query(self, query):
157         results_page = self._load_query(query)
158         # Some simple searches can return a single result.
159         results_url = results_page.geturl()
160         if results_url.find("/show_bug.cgi?id=") != -1:
161             bug_id = int(results_url.split("=")[-1])
162             return [self._fetch_bug(bug_id)]
163         if not self._parse_result_count(results_page):
164             return []
165         # Bugzilla results pages have an "XML" submit button at the bottom
166         # which can be used to get an XML page containing all of the <bug> elements.
167         # This is slighty lame that this assumes that _load_query used
168         # self._bugzilla.browser and that it's in an acceptable state.
169         self._bugzilla.browser.select_form(predicate=self._is_xml_bugs_form)
170         bugs_xml = self._bugzilla.browser.submit()
171         return self._bugzilla._parse_bugs_from_xml(bugs_xml)
172
173     def _fetch_bug(self, bug_id):
174         return self._bugzilla.fetch_bug(bug_id)
175
176     def _fetch_bug_ids_advanced_query(self, query):
177         soup = BeautifulSoup(self._load_query(query))
178         # The contents of the <a> inside the cells in the first column happen
179         # to be the bug id.
180         return [int(bug_link_cell.find("a").string)
181                 for bug_link_cell in soup('td', "first-child")]
182
183     def _parse_attachment_ids_request_query(self, page):
184         digits = re.compile("\d+")
185         attachment_href = re.compile("attachment.cgi\?id=\d+&action=review")
186         attachment_links = SoupStrainer("a", href=attachment_href)
187         return [int(digits.search(tag["href"]).group(0))
188                 for tag in BeautifulSoup(page, parseOnlyThese=attachment_links)]
189
190     def _fetch_attachment_ids_request_query(self, query):
191         return self._parse_attachment_ids_request_query(self._load_query(query))
192
193     def _parse_quips(self, page):
194         soup = BeautifulSoup(page, convertEntities=BeautifulSoup.HTML_ENTITIES)
195         quips = soup.find(text=re.compile(r"Existing quips:")).findNext("ul").findAll("li")
196         return [unicode(quip_entry.string) for quip_entry in quips]
197
198     def fetch_quips(self):
199         return self._parse_quips(self._load_query("/quips.cgi?action=show"))
200
201     # List of all r+'d bugs.
202     def fetch_bug_ids_from_pending_commit_list(self):
203         needs_commit_query_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review%2B"
204         return self._fetch_bug_ids_advanced_query(needs_commit_query_url)
205
206     def fetch_bugs_matching_quicksearch(self, search_string):
207         # We may want to use a more explicit query than "quicksearch".
208         # If quicksearch changes we should probably change to use
209         # a normal buglist.cgi?query_format=advanced query.
210         quicksearch_url = "buglist.cgi?quicksearch=%s" % urllib.quote(search_string)
211         return self._fetch_bugs_from_advanced_query(quicksearch_url)
212
213     # Currently this returns all bugs across all components.
214     # In the future we may wish to extend this API to construct more restricted searches.
215     def fetch_bugs_matching_search(self, search_string):
216         query = "buglist.cgi?query_format=advanced"
217         if search_string:
218             query += "&short_desc_type=allwordssubstr&short_desc=%s" % urllib.quote(search_string)
219         return self._fetch_bugs_from_advanced_query(query)
220
221     def fetch_patches_from_pending_commit_list(self):
222         return sum([self._fetch_bug(bug_id).reviewed_patches()
223             for bug_id in self.fetch_bug_ids_from_pending_commit_list()], [])
224
225     def fetch_bugs_from_review_queue(self, cc_email=None):
226         query = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review?"
227
228         if cc_email:
229             query += "&emailcc1=1&emailtype1=substring&email1=%s" % urllib.quote(cc_email)
230
231         return self._fetch_bugs_from_advanced_query(query)
232
233     def fetch_bug_ids_from_commit_queue(self):
234         commit_queue_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=commit-queue%2B&order=Last+Changed"
235         return self._fetch_bug_ids_advanced_query(commit_queue_url)
236
237     def fetch_patches_from_commit_queue(self):
238         # This function will only return patches which have valid committers
239         # set.  It won't reject patches with invalid committers/reviewers.
240         return sum([self._fetch_bug(bug_id).commit_queued_patches()
241                     for bug_id in self.fetch_bug_ids_from_commit_queue()], [])
242
243     def fetch_bug_ids_from_review_queue(self):
244         review_queue_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review?"
245         return self._fetch_bug_ids_advanced_query(review_queue_url)
246
247     # This method will make several requests to bugzilla.
248     def fetch_patches_from_review_queue(self, limit=None):
249         # [:None] returns the whole array.
250         return sum([self._fetch_bug(bug_id).unreviewed_patches()
251             for bug_id in self.fetch_bug_ids_from_review_queue()[:limit]], [])
252
253     # NOTE: This is the only client of _fetch_attachment_ids_request_query
254     # This method only makes one request to bugzilla.
255     def fetch_attachment_ids_from_review_queue(self):
256         review_queue_url = "request.cgi?action=queue&type=review&group=type"
257         return self._fetch_attachment_ids_request_query(review_queue_url)
258
259     # This only works if your account has edituser privileges.
260     # We could easily parse https://bugs.webkit.org/userprefs.cgi?tab=permissions to
261     # check permissions, but bugzilla will just return an error if we don't have them.
262     def fetch_login_userid_pairs_matching_substring(self, search_string):
263         review_queue_url = "editusers.cgi?action=list&matchvalue=login_name&matchstr=%s&matchtype=substr" % urllib.quote(search_string)
264         results_page = self._load_query(review_queue_url)
265         # We could pull the EditUsersParser off Bugzilla if needed.
266         return EditUsersParser().login_userid_pairs_from_edit_user_results(results_page)
267
268     # FIXME: We should consider adding a BugzillaUser class.
269     def fetch_logins_matching_substring(self, search_string):
270         pairs = self.fetch_login_userid_pairs_matching_substring(search_string)
271         return map(lambda pair: pair[0], pairs)
272
273
274 class Bugzilla(object):
275     def __init__(self, committers=committers.CommitterList()):
276         self.authenticated = False
277         self.queries = BugzillaQueries(self)
278         self.committers = committers
279         self.cached_quips = []
280         self.edit_user_parser = EditUsersParser()
281         self._browser = None
282
283     def _get_browser(self):
284         if not self._browser:
285             self.setdefaulttimeout(600)
286             from webkitpy.thirdparty.autoinstalled.mechanize import Browser
287             self._browser = Browser()
288             # Ignore bugs.webkit.org/robots.txt until we fix it to allow this script.
289             self._browser.set_handle_robots(False)
290         return self._browser
291
292     def _set_browser(self, value):
293         self._browser = value
294
295     browser = property(_get_browser, _set_browser)
296
297     def setdefaulttimeout(self, value):
298         socket.setdefaulttimeout(value)
299
300     def fetch_user(self, user_id):
301         self.authenticate()
302         edit_user_page = self.browser.open(self.edit_user_url_for_id(user_id))
303         return self.edit_user_parser.user_dict_from_edit_user_page(edit_user_page)
304
305     def add_user_to_groups(self, user_id, group_names):
306         self.authenticate()
307         user_edit_page = self.browser.open(self.edit_user_url_for_id(user_id))
308         self.browser.select_form(nr=1)
309         for group_name in group_names:
310             group_string = self.edit_user_parser.group_string_from_name(user_edit_page, group_name)
311             self.browser.find_control(group_string).items[0].selected = True
312         self.browser.submit()
313
314     def quips(self):
315         # We only fetch and parse the list of quips once per instantiation
316         # so that we do not burden bugs.webkit.org.
317         if not self.cached_quips:
318             self.cached_quips = self.queries.fetch_quips()
319         return self.cached_quips
320
321     def bug_url_for_bug_id(self, bug_id, xml=False):
322         if not bug_id:
323             return None
324         content_type = "&ctype=xml&excludefield=attachmentdata" if xml else ""
325         return "%sshow_bug.cgi?id=%s%s" % (config_urls.bug_server_url, bug_id, content_type)
326
327     def short_bug_url_for_bug_id(self, bug_id):
328         if not bug_id:
329             return None
330         return "http://webkit.org/b/%s" % bug_id
331
332     def add_attachment_url(self, bug_id):
333         return "%sattachment.cgi?action=enter&bugid=%s" % (config_urls.bug_server_url, bug_id)
334
335     def attachment_url_for_id(self, attachment_id, action="view"):
336         if not attachment_id:
337             return None
338         action_param = ""
339         if action and action != "view":
340             action_param = "&action=%s" % action
341         return "%sattachment.cgi?id=%s%s" % (config_urls.bug_server_url,
342                                              attachment_id,
343                                              action_param)
344
345     def edit_user_url_for_id(self, user_id):
346         return "%seditusers.cgi?action=edit&userid=%s" % (config_urls.bug_server_url, user_id)
347
348     def _parse_attachment_flag(self,
349                                element,
350                                flag_name,
351                                attachment,
352                                result_key):
353         flag = element.find('flag', attrs={'name': flag_name})
354         if flag:
355             attachment[flag_name] = flag['status']
356             if flag['status'] == '+':
357                 attachment[result_key] = flag['setter']
358         # Sadly show_bug.cgi?ctype=xml does not expose the flag modification date.
359
360     def _string_contents(self, soup):
361         # WebKit's bugzilla instance uses UTF-8.
362         # BeautifulStoneSoup always returns Unicode strings, however
363         # the .string method returns a (unicode) NavigableString.
364         # NavigableString can confuse other parts of the code, so we
365         # convert from NavigableString to a real unicode() object using unicode().
366         return unicode(soup.string)
367
368     # Example: 2010-01-20 14:31 PST
369     # FIXME: Some bugzilla dates seem to have seconds in them?
370     # Python does not support timezones out of the box.
371     # Assume that bugzilla always uses PST (which is true for bugs.webkit.org)
372     _bugzilla_date_format = "%Y-%m-%d %H:%M:%S"
373
374     @classmethod
375     def _parse_date(cls, date_string):
376         (date, time, time_zone) = date_string.split(" ")
377         if time.count(':') == 1:
378             # Add seconds into the time.
379             time += ':0'
380         # Ignore the timezone because python doesn't understand timezones out of the box.
381         date_string = "%s %s" % (date, time)
382         return datetime.strptime(date_string, cls._bugzilla_date_format)
383
384     def _date_contents(self, soup):
385         return self._parse_date(self._string_contents(soup))
386
387     def _parse_attachment_element(self, element, bug_id):
388         attachment = {}
389         attachment['bug_id'] = bug_id
390         attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
391         attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
392         attachment['id'] = int(element.find('attachid').string)
393         # FIXME: No need to parse out the url here.
394         attachment['url'] = self.attachment_url_for_id(attachment['id'])
395         attachment["attach_date"] = self._date_contents(element.find("date"))
396         attachment['name'] = self._string_contents(element.find('desc'))
397         attachment['attacher_email'] = self._string_contents(element.find('attacher'))
398         attachment['type'] = self._string_contents(element.find('type'))
399         self._parse_attachment_flag(
400                 element, 'review', attachment, 'reviewer_email')
401         self._parse_attachment_flag(
402                 element, 'commit-queue', attachment, 'committer_email')
403         return attachment
404
405     def _parse_log_descr_element(self, element):
406         comment = {}
407         comment['comment_email'] = self._string_contents(element.find('who'))
408         comment['comment_date'] = self._date_contents(element.find('bug_when'))
409         comment['text'] = self._string_contents(element.find('thetext'))
410         return comment
411
412     def _parse_bugs_from_xml(self, page):
413         soup = BeautifulSoup(page)
414         # Without the unicode() call, BeautifulSoup occasionally complains of being
415         # passed None for no apparent reason.
416         return [Bug(self._parse_bug_dictionary_from_xml(unicode(bug_xml)), self) for bug_xml in soup('bug')]
417
418     def _parse_bug_dictionary_from_xml(self, page):
419         soup = BeautifulStoneSoup(page, convertEntities=BeautifulStoneSoup.XML_ENTITIES)
420         bug = {}
421         bug["id"] = int(soup.find("bug_id").string)
422         bug["title"] = self._string_contents(soup.find("short_desc"))
423         bug["bug_status"] = self._string_contents(soup.find("bug_status"))
424         dup_id = soup.find("dup_id")
425         if dup_id:
426             bug["dup_id"] = self._string_contents(dup_id)
427         bug["reporter_email"] = self._string_contents(soup.find("reporter"))
428         bug["assigned_to_email"] = self._string_contents(soup.find("assigned_to"))
429         bug["cc_emails"] = [self._string_contents(element) for element in soup.findAll('cc')]
430         bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')]
431         bug["comments"] = [self._parse_log_descr_element(element) for element in soup.findAll('long_desc')]
432
433         return bug
434
435     # Makes testing fetch_*_from_bug() possible until we have a better
436     # BugzillaNetwork abstration.
437
438     def _fetch_bug_page(self, bug_id):
439         bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
440         _log.info("Fetching: %s" % bug_url)
441         return self.browser.open(bug_url)
442
443     def fetch_bug_dictionary(self, bug_id):
444         try:
445             return self._parse_bug_dictionary_from_xml(self._fetch_bug_page(bug_id))
446         except KeyboardInterrupt:
447             raise
448         except:
449             self.authenticate()
450             return self._parse_bug_dictionary_from_xml(self._fetch_bug_page(bug_id))
451
452     # FIXME: A BugzillaCache object should provide all these fetch_ methods.
453
454     def fetch_bug(self, bug_id):
455         return Bug(self.fetch_bug_dictionary(bug_id), self)
456
457     def fetch_attachment_contents(self, attachment_id):
458         attachment_url = self.attachment_url_for_id(attachment_id)
459         # We need to authenticate to download patches from security bugs.
460         self.authenticate()
461         return self.browser.open(attachment_url).read()
462
463     def _parse_bug_id_from_attachment_page(self, page):
464         # The "Up" relation happens to point to the bug.
465         up_link = BeautifulSoup(page).find('link', rel='Up')
466         if not up_link:
467             # This attachment does not exist (or you don't have permissions to
468             # view it).
469             return None
470         match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href'])
471         return int(match.group('bug_id'))
472
473     def bug_id_for_attachment_id(self, attachment_id):
474         self.authenticate()
475
476         attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
477         _log.info("Fetching: %s" % attachment_url)
478         page = self.browser.open(attachment_url)
479         return self._parse_bug_id_from_attachment_page(page)
480
481     # FIXME: This should just return Attachment(id), which should be able to
482     # lazily fetch needed data.
483
484     def fetch_attachment(self, attachment_id):
485         # We could grab all the attachment details off of the attachment edit
486         # page but we already have working code to do so off of the bugs page,
487         # so re-use that.
488         bug_id = self.bug_id_for_attachment_id(attachment_id)
489         if not bug_id:
490             return None
491         attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True)
492         for attachment in attachments:
493             if attachment.id() == int(attachment_id):
494                 return attachment
495         return None # This should never be hit.
496
497     def authenticate(self):
498         if self.authenticated:
499             return
500
501         credentials = Credentials(config_urls.bug_server_host, git_prefix="bugzilla")
502
503         attempts = 0
504         while not self.authenticated:
505             attempts += 1
506             username, password = credentials.read_credentials()
507
508             _log.info("Logging in as %s..." % username)
509             self.browser.open(config_urls.bug_server_url +
510                               "index.cgi?GoAheadAndLogIn=1")
511             self.browser.select_form(name="login")
512             self.browser['Bugzilla_login'] = username
513             self.browser['Bugzilla_password'] = password
514             self.browser.find_control("Bugzilla_restrictlogin").items[0].selected = False
515             response = self.browser.submit()
516
517             match = re.search("<title>(.+?)</title>", response.read())
518             # If the resulting page has a title, and it contains the word
519             # "invalid" assume it's the login failure page.
520             if match and re.search("Invalid", match.group(1), re.IGNORECASE):
521                 errorMessage = "Bugzilla login failed: %s" % match.group(1)
522                 # raise an exception only if this was the last attempt
523                 if attempts < 5:
524                     _log.error(errorMessage)
525                 else:
526                     raise Exception(errorMessage)
527             else:
528                 self.authenticated = True
529                 self.username = username
530
531     # FIXME: Use enum instead of two booleans
532     def _commit_queue_flag(self, mark_for_landing, mark_for_commit_queue):
533         if mark_for_landing:
534             user = self.committers.contributor_by_email(self.username)
535             mark_for_commit_queue = True
536             if not user:
537                 _log.warning("Your Bugzilla login is not listed in contributors.json. Uploading with cq? instead of cq+")
538                 mark_for_landing = False
539             elif not user.can_commit:
540                 _log.warning("You're not a committer yet or haven't updated contributors.json yet. Uploading with cq? instead of cq+")
541                 mark_for_landing = False
542
543         if mark_for_landing:
544             return '+'
545         if mark_for_commit_queue:
546             return '?'
547         return 'X'
548
549     # FIXME: mark_for_commit_queue and mark_for_landing should be joined into a single commit_flag argument.
550     def _fill_attachment_form(self,
551                               description,
552                               file_object,
553                               mark_for_review=False,
554                               mark_for_commit_queue=False,
555                               mark_for_landing=False,
556                               is_patch=False,
557                               filename=None,
558                               mimetype=None):
559         self.browser['description'] = description
560         if is_patch:
561             self.browser['ispatch'] = ("1",)
562         # FIXME: Should this use self._find_select_element_for_flag?
563         self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
564         self.browser['flag_type-3'] = (self._commit_queue_flag(mark_for_landing, mark_for_commit_queue),)
565
566         filename = filename or "%s.patch" % timestamp()
567         if not mimetype:
568             mimetypes.add_type('text/plain', '.patch')  # Make sure mimetypes knows about .patch
569             mimetype, _ = mimetypes.guess_type(filename)
570         if not mimetype:
571             mimetype = "text/plain"  # Bugzilla might auto-guess for us and we might not need this?
572         self.browser.add_file(file_object, mimetype, filename, 'data')
573
574     def _file_object_for_upload(self, file_or_string):
575         if hasattr(file_or_string, 'read'):
576             return file_or_string
577         # Only if file_or_string is not already encoded do we want to encode it.
578         if isinstance(file_or_string, unicode):
579             file_or_string = file_or_string.encode('utf-8')
580         return StringIO.StringIO(file_or_string)
581
582     # timestamp argument is just for unittests.
583     def _filename_for_upload(self, file_object, bug_id, extension="txt", timestamp=timestamp):
584         if hasattr(file_object, "name"):
585             return file_object.name
586         return "bug-%s-%s.%s" % (bug_id, timestamp(), extension)
587
588     def add_attachment_to_bug(self, bug_id, file_or_string, description, filename=None, comment_text=None, mimetype=None):
589         self.authenticate()
590         _log.info('Adding attachment "%s" to %s' % (description, self.bug_url_for_bug_id(bug_id)))
591         self.browser.open(self.add_attachment_url(bug_id))
592         self.browser.select_form(name="entryform")
593         file_object = self._file_object_for_upload(file_or_string)
594         filename = filename or self._filename_for_upload(file_object, bug_id)
595         self._fill_attachment_form(description, file_object, filename=filename, mimetype=mimetype)
596         if comment_text:
597             _log.info(comment_text)
598             self.browser['comment'] = comment_text
599         self.browser.submit()
600
601     # FIXME: The arguments to this function should be simplified and then
602     # this should be merged into add_attachment_to_bug
603     def add_patch_to_bug(self,
604                          bug_id,
605                          file_or_string,
606                          description,
607                          comment_text=None,
608                          mark_for_review=False,
609                          mark_for_commit_queue=False,
610                          mark_for_landing=False):
611         self.authenticate()
612         _log.info('Adding patch "%s" to %s' % (description, self.bug_url_for_bug_id(bug_id)))
613
614         self.browser.open(self.add_attachment_url(bug_id))
615         self.browser.select_form(name="entryform")
616         file_object = self._file_object_for_upload(file_or_string)
617         filename = self._filename_for_upload(file_object, bug_id, extension="patch")
618         self._fill_attachment_form(description,
619                                    file_object,
620                                    mark_for_review=mark_for_review,
621                                    mark_for_commit_queue=mark_for_commit_queue,
622                                    mark_for_landing=mark_for_landing,
623                                    is_patch=True,
624                                    filename=filename)
625         if comment_text:
626             _log.info(comment_text)
627             self.browser['comment'] = comment_text
628         self.browser.submit()
629
630     # FIXME: There has to be a more concise way to write this method.
631     def _check_create_bug_response(self, response_html):
632         match = re.search("<title>Bug (?P<bug_id>\d+) Submitted[^<]*</title>",
633                           response_html)
634         if match:
635             return match.group('bug_id')
636
637         match = re.search(
638             '<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">',
639             response_html,
640             re.DOTALL)
641         error_message = "FAIL"
642         if match:
643             text_lines = BeautifulSoup(
644                     match.group('error_message')).findAll(text=True)
645             error_message = "\n" + '\n'.join(
646                     ["  " + line.strip()
647                      for line in text_lines if line.strip()])
648         raise Exception("Bug not created: %s" % error_message)
649
650     def create_bug(self,
651                    bug_title,
652                    bug_description,
653                    component=None,
654                    diff=None,
655                    patch_description=None,
656                    cc=None,
657                    blocked=None,
658                    assignee=None,
659                    mark_for_review=False,
660                    mark_for_commit_queue=False):
661         self.authenticate()
662
663         _log.info('Creating bug with title "%s"' % bug_title)
664         self.browser.open(config_urls.bug_server_url + "enter_bug.cgi?product=WebKit")
665         self.browser.select_form(name="Create")
666         component_items = self.browser.find_control('component').items
667         component_names = map(lambda item: item.name, component_items)
668         if not component:
669             component = "New Bugs"
670         if component not in component_names:
671             component = User.prompt_with_list("Please pick a component:", component_names)
672         self.browser["component"] = [component]
673         if cc:
674             self.browser["cc"] = cc
675         if blocked:
676             self.browser["blocked"] = unicode(blocked)
677         if not assignee:
678             assignee = self.username
679         if assignee and not self.browser.find_control("assigned_to").disabled:
680             self.browser["assigned_to"] = assignee
681         self.browser["short_desc"] = bug_title
682         self.browser["comment"] = bug_description
683
684         if diff:
685             # _fill_attachment_form expects a file-like object
686             # Patch files are already binary, so no encoding needed.
687             assert(isinstance(diff, str))
688             patch_file_object = StringIO.StringIO(diff)
689             self._fill_attachment_form(
690                     patch_description,
691                     patch_file_object,
692                     mark_for_review=mark_for_review,
693                     mark_for_commit_queue=mark_for_commit_queue,
694                     is_patch=True)
695
696         response = self.browser.submit()
697
698         bug_id = self._check_create_bug_response(response.read())
699         _log.info("Bug %s created." % bug_id)
700         _log.info("%sshow_bug.cgi?id=%s" % (config_urls.bug_server_url, bug_id))
701         return bug_id
702
703     def _find_select_element_for_flag(self, flag_name):
704         # FIXME: This will break if we ever re-order attachment flags
705         if flag_name == "review":
706             return self.browser.find_control(type='select', nr=0)
707         elif flag_name == "commit-queue":
708             return self.browser.find_control(type='select', nr=1)
709         raise Exception("Don't know how to find flag named \"%s\"" % flag_name)
710
711     def clear_attachment_flags(self,
712                                attachment_id,
713                                additional_comment_text=None):
714         self.authenticate()
715
716         comment_text = "Clearing flags on attachment: %s" % attachment_id
717         if additional_comment_text:
718             comment_text += "\n\n%s" % additional_comment_text
719         _log.info(comment_text)
720
721         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
722         self.browser.select_form(nr=1)
723         self.browser.set_value(comment_text, name='comment', nr=0)
724         self._find_select_element_for_flag('review').value = ("X",)
725         self._find_select_element_for_flag('commit-queue').value = ("X",)
726         self.browser.submit()
727
728     def set_flag_on_attachment(self,
729                                attachment_id,
730                                flag_name,
731                                flag_value,
732                                comment_text=None):
733         # FIXME: We need a way to test this function on a live bugzilla
734         # instance.
735
736         self.authenticate()
737         _log.info(comment_text)
738         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
739         self.browser.select_form(nr=1)
740
741         if comment_text:
742             self.browser.set_value(comment_text, name='comment', nr=0)
743
744         self._find_select_element_for_flag(flag_name).value = (flag_value,)
745         self.browser.submit()
746
747     # FIXME: All of these bug editing methods have a ridiculous amount of
748     # copy/paste code.
749
750     def obsolete_attachment(self, attachment_id, comment_text=None):
751         self.authenticate()
752
753         _log.info("Obsoleting attachment: %s" % attachment_id)
754         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
755         self.browser.select_form(nr=1)
756         self.browser.find_control('isobsolete').items[0].selected = True
757         # Also clear any review flag (to remove it from review/commit queues)
758         self._find_select_element_for_flag('review').value = ("X",)
759         self._find_select_element_for_flag('commit-queue').value = ("X",)
760         if comment_text:
761             _log.info(comment_text)
762             # Bugzilla has two textareas named 'comment', one is somehow
763             # hidden.  We want the first.
764             self.browser.set_value(comment_text, name='comment', nr=0)
765         self.browser.submit()
766
767     def add_cc_to_bug(self, bug_id, email_address_list):
768         self.authenticate()
769
770         _log.info("Adding %s to the CC list for bug %s" % (email_address_list, bug_id))
771         self.browser.open(self.bug_url_for_bug_id(bug_id))
772         self.browser.select_form(name="changeform")
773         self.browser["newcc"] = ", ".join(email_address_list)
774         self.browser.submit()
775
776     def post_comment_to_bug(self, bug_id, comment_text, cc=None):
777         self.authenticate()
778
779         _log.info("Adding comment to bug %s" % bug_id)
780         self.browser.open(self.bug_url_for_bug_id(bug_id))
781         self.browser.select_form(name="changeform")
782         self.browser["comment"] = comment_text
783         if cc:
784             self.browser["newcc"] = ", ".join(cc)
785         self.browser.submit()
786
787     def close_bug_as_fixed(self, bug_id, comment_text=None):
788         self.authenticate()
789
790         _log.info("Closing bug %s as fixed" % bug_id)
791         self.browser.open(self.bug_url_for_bug_id(bug_id))
792         self.browser.select_form(name="changeform")
793         if comment_text:
794             self.browser['comment'] = comment_text
795         self.browser['bug_status'] = ['RESOLVED']
796         self.browser['resolution'] = ['FIXED']
797         self.browser.submit()
798
799     def _has_control(self, form, id):
800         return id in [control.id for control in form.controls]
801
802     def reassign_bug(self, bug_id, assignee=None, comment_text=None):
803         self.authenticate()
804
805         if not assignee:
806             assignee = self.username
807
808         _log.info("Assigning bug %s to %s" % (bug_id, assignee))
809         self.browser.open(self.bug_url_for_bug_id(bug_id))
810         self.browser.select_form(name="changeform")
811
812         if not self._has_control(self.browser, "assigned_to"):
813             _log.warning("""Failed to assign bug to you (can't find assigned_to) control.
814 Ignore this message if you don't have EditBugs privileges (https://bugs.webkit.org/userprefs.cgi?tab=permissions)""")
815             return
816
817         if comment_text:
818             _log.info(comment_text)
819             self.browser["comment"] = comment_text
820         self.browser["assigned_to"] = assignee
821         self.browser.submit()
822
823     def reopen_bug(self, bug_id, comment_text):
824         self.authenticate()
825
826         _log.info("Re-opening bug %s" % bug_id)
827         # Bugzilla requires a comment when re-opening a bug, so we know it will
828         # never be None.
829         _log.info(comment_text)
830         self.browser.open(self.bug_url_for_bug_id(bug_id))
831         self.browser.select_form(name="changeform")
832         bug_status = self.browser.find_control("bug_status", type="select")
833         # This is a hack around the fact that ClientForm.ListControl seems to
834         # have no simpler way to ask if a control has an item named "REOPENED"
835         # without using exceptions for control flow.
836         possible_bug_statuses = map(lambda item: item.name, bug_status.items)
837         if "REOPENED" in possible_bug_statuses:
838             bug_status.value = ["REOPENED"]
839         # If the bug was never confirmed it will not have a "REOPENED"
840         # state, but only an "UNCONFIRMED" state.
841         elif "UNCONFIRMED" in possible_bug_statuses:
842             bug_status.value = ["UNCONFIRMED"]
843         else:
844             # FIXME: This logic is slightly backwards.  We won't print this
845             # message if the bug is already open with state "UNCONFIRMED".
846             _log.info("Did not reopen bug %s, it appears to already be open with status %s." % (bug_id, bug_status.value))
847         self.browser['comment'] = comment_text
848         self.browser.submit()