b10c96fbed5ea93309d0165c5eb593ded6d6c7bb
[WebKit-https.git] / Tools / Scripts / webkitpy / common / net / bugzilla / bugzilla.py
1 # Copyright (c) 2011 Google Inc. All rights reserved.
2 # Copyright (c) 2009 Apple Inc. All rights reserved.
3 # Copyright (c) 2010 Research In Motion Limited. All rights reserved.
4 # Copyright (c) 2013 University of Szeged. All rights reserved.
5 #
6 # Redistribution and use in source and binary forms, with or without
7 # modification, are permitted provided that the following conditions are
8 # met:
9 #
10 #     * Redistributions of source code must retain the above copyright
11 # notice, this list of conditions and the following disclaimer.
12 #     * Redistributions in binary form must reproduce the above
13 # copyright notice, this list of conditions and the following disclaimer
14 # in the documentation and/or other materials provided with the
15 # distribution.
16 #     * Neither the name of Google Inc. nor the names of its
17 # contributors may be used to endorse or promote products derived from
18 # this software without specific prior written permission.
19 #
20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #
32 # WebKit's Python module for interacting with Bugzilla
33
34 import logging
35 import mimetypes
36 import re
37 import StringIO
38 import socket
39 import urllib
40
41 from datetime import datetime  # used in timestamp()
42
43 from .attachment import Attachment
44 from .bug import Bug
45
46 from webkitpy.common.config import committers
47 import webkitpy.common.config.urls as config_urls
48 from webkitpy.common.net.credentials import Credentials
49 from webkitpy.common.system.user import User
50 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, SoupStrainer
51
52 _log = logging.getLogger(__name__)
53
54
55 class EditUsersParser(object):
56     def __init__(self):
57         self._group_name_to_group_string_cache = {}
58
59     def _login_and_uid_from_row(self, row):
60         first_cell = row.find("td")
61         # The first row is just headers, we skip it.
62         if not first_cell:
63             return None
64         # When there were no results, we have a fake "<none>" entry in the table.
65         if first_cell.find(text="<none>"):
66             return None
67         # Otherwise the <td> contains a single <a> which contains the login name or a single <i> with the string "<none>".
68         anchor_tag = first_cell.find("a")
69         login = unicode(anchor_tag.string).strip()
70         user_id = int(re.search(r"userid=(\d+)", str(anchor_tag['href'])).group(1))
71         return (login, user_id)
72
73     def login_userid_pairs_from_edit_user_results(self, results_page):
74         soup = BeautifulSoup(results_page, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
75         results_table = soup.find(id="admin_table")
76         login_userid_pairs = [self._login_and_uid_from_row(row) for row in results_table('tr')]
77         # Filter out None from the logins.
78         return filter(lambda pair: bool(pair), login_userid_pairs)
79
80     def _group_name_and_string_from_row(self, row):
81         label_element = row.find('label')
82         group_string = unicode(label_element['for'])
83         group_name = unicode(label_element.find('strong').string).rstrip(':')
84         return (group_name, group_string)
85
86     def user_dict_from_edit_user_page(self, page):
87         soup = BeautifulSoup(page, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
88         user_table = soup.find("table", {'class': 'main'})
89         user_dict = {}
90         for row in user_table('tr'):
91             label_element = row.find('label')
92             if not label_element:
93                 continue  # This must not be a row we know how to parse.
94             if row.find('table'):
95                 continue  # Skip the <tr> holding the groups table.
96
97             key = label_element['for']
98             if "group" in key:
99                 key = "groups"
100                 value = user_dict.get('groups', set())
101                 # We must be parsing a "tr" inside the inner group table.
102                 (group_name, _) = self._group_name_and_string_from_row(row)
103                 if row.find('input', {'type': 'checkbox', 'checked': 'checked'}):
104                     value.add(group_name)
105             else:
106                 value = unicode(row.find('td').string).strip()
107             user_dict[key] = value
108         return user_dict
109
110     def _group_rows_from_edit_user_page(self, edit_user_page):
111         soup = BeautifulSoup(edit_user_page, convertEntities=BeautifulSoup.HTML_ENTITIES)
112         return soup('td', {'class': 'groupname'})
113
114     def group_string_from_name(self, edit_user_page, group_name):
115         # Bugzilla uses "group_NUMBER" strings, which may be different per install
116         # so we just look them up once and cache them.
117         if not self._group_name_to_group_string_cache:
118             rows = self._group_rows_from_edit_user_page(edit_user_page)
119             name_string_pairs = map(self._group_name_and_string_from_row, rows)
120             self._group_name_to_group_string_cache = dict(name_string_pairs)
121         return self._group_name_to_group_string_cache[group_name]
122
123
124 def timestamp():
125     return datetime.now().strftime("%Y%m%d%H%M%S")
126
127
128 # A container for all of the logic for making and parsing bugzilla queries.
129 class BugzillaQueries(object):
130
131     def __init__(self, bugzilla):
132         self._bugzilla = bugzilla
133
134     def _is_xml_bugs_form(self, form):
135         # ClientForm.HTMLForm.find_control throws if the control is not found,
136         # so we do a manual search instead:
137         return "xml" in [control.id for control in form.controls]
138
139     # This is kinda a hack.  There is probably a better way to get this information from bugzilla.
140     def _parse_result_count(self, results_page):
141         result_count_text = BeautifulSoup(results_page).find(attrs={'class': 'bz_result_count'}).string
142         result_count_parts = result_count_text.strip().split(" ")
143         if result_count_parts[0] == "Zarro":
144             return 0
145         if result_count_parts[0] == "One":
146             return 1
147         return int(result_count_parts[0])
148
149     # Note: _load_query, _fetch_bug and _fetch_bugs_from_advanced_query
150     # are the only methods which access self._bugzilla.
151
152     def _load_query(self, query):
153         self._bugzilla.authenticate()
154         full_url = "%s%s" % (config_urls.bug_server_url, query)
155         return self._bugzilla.browser.open(full_url)
156
157     def _fetch_bugs_from_advanced_query(self, query):
158         results_page = self._load_query(query)
159         # Some simple searches can return a single result.
160         results_url = results_page.geturl()
161         if results_url.find("/show_bug.cgi?id=") != -1:
162             bug_id = int(results_url.split("=")[-1])
163             return [self._fetch_bug(bug_id)]
164         if not self._parse_result_count(results_page):
165             return []
166         # Bugzilla results pages have an "XML" submit button at the bottom
167         # which can be used to get an XML page containing all of the <bug> elements.
168         # This is slighty lame that this assumes that _load_query used
169         # self._bugzilla.browser and that it's in an acceptable state.
170         self._bugzilla.browser.select_form(predicate=self._is_xml_bugs_form)
171         bugs_xml = self._bugzilla.browser.submit()
172         return self._bugzilla._parse_bugs_from_xml(bugs_xml)
173
174     def _fetch_bug(self, bug_id):
175         return self._bugzilla.fetch_bug(bug_id)
176
177     def _fetch_bug_ids_advanced_query(self, query):
178         soup = BeautifulSoup(self._load_query(query))
179         # The contents of the <a> inside the cells in the first column happen
180         # to be the bug id.
181         return [int(bug_link_cell.find("a").string)
182                 for bug_link_cell in soup('td', "first-child")]
183
184     def _parse_attachment_ids_request_query(self, page, since=None):
185         # Formats
186         digits = re.compile("\d+")
187         attachment_href = re.compile("attachment.cgi\?id=\d+&action=review")
188         # if no date is given, return all ids
189         if not since:
190             attachment_links = SoupStrainer("a", href=attachment_href)
191             return [int(digits.search(tag["href"]).group(0))
192                 for tag in BeautifulSoup(page, parseOnlyThese=attachment_links)]
193
194         # Parse the main table only
195         date_format = re.compile("\d{4}-\d{2}-\d{2} \d{2}:\d{2}")
196         mtab = SoupStrainer("table", {"class": "requests"})
197         soup = BeautifulSoup(page, parseOnlyThese=mtab)
198         patch_ids = []
199
200         for row in soup.findAll("tr"):
201             patch_tag = row.find("a", {"href": attachment_href})
202             if not patch_tag:
203                 continue
204             patch_id = int(digits.search(patch_tag["href"]).group(0))
205             date_tag = row.find("td", text=date_format)
206             if date_tag and datetime.strptime(date_format.search(date_tag).group(0), "%Y-%m-%d %H:%M") < since:
207                 continue
208             patch_ids.append(patch_id)
209         return patch_ids
210
211     def _fetch_attachment_ids_request_query(self, query, since=None):
212         return self._parse_attachment_ids_request_query(self._load_query(query), since)
213
214     def _parse_quips(self, page):
215         soup = BeautifulSoup(page, convertEntities=BeautifulSoup.HTML_ENTITIES)
216         quips = soup.find(text=re.compile(r"Existing quips:")).findNext("ul").findAll("li")
217         return [unicode(quip_entry.string) for quip_entry in quips]
218
219     def fetch_quips(self):
220         return self._parse_quips(self._load_query("/quips.cgi?action=show"))
221
222     # List of all r+'d bugs.
223     def fetch_bug_ids_from_pending_commit_list(self):
224         needs_commit_query_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review%2B"
225         return self._fetch_bug_ids_advanced_query(needs_commit_query_url)
226
227     def fetch_bugs_matching_quicksearch(self, search_string):
228         # We may want to use a more explicit query than "quicksearch".
229         # If quicksearch changes we should probably change to use
230         # a normal buglist.cgi?query_format=advanced query.
231         quicksearch_url = "buglist.cgi?quicksearch=%s" % urllib.quote(search_string)
232         return self._fetch_bugs_from_advanced_query(quicksearch_url)
233
234     # Currently this returns all bugs across all components.
235     # In the future we may wish to extend this API to construct more restricted searches.
236     def fetch_bugs_matching_search(self, search_string):
237         query = "buglist.cgi?query_format=advanced"
238         if search_string:
239             query += "&short_desc_type=allwordssubstr&short_desc=%s" % urllib.quote(search_string)
240         return self._fetch_bugs_from_advanced_query(query)
241
242     def fetch_patches_from_pending_commit_list(self):
243         return sum([self._fetch_bug(bug_id).reviewed_patches()
244             for bug_id in self.fetch_bug_ids_from_pending_commit_list()], [])
245
246     def fetch_bugs_from_review_queue(self, cc_email=None):
247         query = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review?"
248
249         if cc_email:
250             query += "&emailcc1=1&emailtype1=substring&email1=%s" % urllib.quote(cc_email)
251
252         return self._fetch_bugs_from_advanced_query(query)
253
254     def fetch_bug_ids_from_commit_queue(self):
255         commit_queue_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=commit-queue%2B&order=Last+Changed"
256         return self._fetch_bug_ids_advanced_query(commit_queue_url)
257
258     def fetch_patches_from_commit_queue(self):
259         # This function will only return patches which have valid committers
260         # set.  It won't reject patches with invalid committers/reviewers.
261         return sum([self._fetch_bug(bug_id).commit_queued_patches()
262                     for bug_id in self.fetch_bug_ids_from_commit_queue()], [])
263
264     def fetch_bug_ids_from_review_queue(self):
265         review_queue_url = "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review?"
266         return self._fetch_bug_ids_advanced_query(review_queue_url)
267
268     # This method will make several requests to bugzilla.
269     def fetch_patches_from_review_queue(self, limit=None):
270         # [:None] returns the whole array.
271         return sum([self._fetch_bug(bug_id).unreviewed_patches()
272             for bug_id in self.fetch_bug_ids_from_review_queue()[:limit]], [])
273
274     # NOTE: This is the only client of _fetch_attachment_ids_request_query
275     # This method only makes one request to bugzilla.
276     def fetch_attachment_ids_from_review_queue(self, since=None):
277         review_queue_url = "request.cgi?action=queue&type=review&group=type"
278         return self._fetch_attachment_ids_request_query(review_queue_url, since)
279
280     # This only works if your account has edituser privileges.
281     # We could easily parse https://bugs.webkit.org/userprefs.cgi?tab=permissions to
282     # check permissions, but bugzilla will just return an error if we don't have them.
283     def fetch_login_userid_pairs_matching_substring(self, search_string):
284         review_queue_url = "editusers.cgi?action=list&matchvalue=login_name&matchstr=%s&matchtype=substr" % urllib.quote(search_string)
285         results_page = self._load_query(review_queue_url)
286         # We could pull the EditUsersParser off Bugzilla if needed.
287         return EditUsersParser().login_userid_pairs_from_edit_user_results(results_page)
288
289     def is_invalid_bugzilla_email(self, search_string):
290         review_queue_url = "request.cgi?action=queue&requester=%s&product=&type=review&requestee=&component=&group=requestee" % urllib.quote(search_string)
291         results_page = self._load_query(review_queue_url)
292         return bool(re.search("did not match anything", results_page.read()))
293
294
295 class CommitQueueFlag(object):
296     mark_for_nothing = 0
297     mark_for_commit_queue = 1
298     mark_for_landing = 2
299
300
301 class Bugzilla(object):
302     def __init__(self, committers=committers.CommitterList()):
303         self.authenticated = False
304         self.queries = BugzillaQueries(self)
305         self.committers = committers
306         self.cached_quips = []
307         self.edit_user_parser = EditUsersParser()
308         self._browser = None
309
310     def _get_browser(self):
311         if not self._browser:
312             self.setdefaulttimeout(600)
313             from webkitpy.thirdparty.autoinstalled.mechanize import Browser
314             self._browser = Browser()
315             self._browser.set_handle_robots(False)
316         return self._browser
317
318     def _set_browser(self, value):
319         self._browser = value
320
321     browser = property(_get_browser, _set_browser)
322
323     def setdefaulttimeout(self, value):
324         socket.setdefaulttimeout(value)
325
326     def fetch_user(self, user_id):
327         self.authenticate()
328         edit_user_page = self.browser.open(self.edit_user_url_for_id(user_id))
329         return self.edit_user_parser.user_dict_from_edit_user_page(edit_user_page)
330
331     def add_user_to_groups(self, user_id, group_names):
332         self.authenticate()
333         user_edit_page = self.browser.open(self.edit_user_url_for_id(user_id))
334         self.browser.select_form(nr=1)
335         for group_name in group_names:
336             group_string = self.edit_user_parser.group_string_from_name(user_edit_page, group_name)
337             self.browser.find_control(group_string).items[0].selected = True
338         self.browser.submit()
339
340     def quips(self):
341         # We only fetch and parse the list of quips once per instantiation
342         # so that we do not burden bugs.webkit.org.
343         if not self.cached_quips:
344             self.cached_quips = self.queries.fetch_quips()
345         return self.cached_quips
346
347     def bug_url_for_bug_id(self, bug_id, xml=False):
348         if not bug_id:
349             return None
350         content_type = "&ctype=xml&excludefield=attachmentdata" if xml else ""
351         return "%sshow_bug.cgi?id=%s%s" % (config_urls.bug_server_url, bug_id, content_type)
352
353     def short_bug_url_for_bug_id(self, bug_id):
354         if not bug_id:
355             return None
356         return "http://webkit.org/b/%s" % bug_id
357
358     def add_attachment_url(self, bug_id):
359         return "%sattachment.cgi?action=enter&bugid=%s" % (config_urls.bug_server_url, bug_id)
360
361     def attachment_url_for_id(self, attachment_id, action="view"):
362         if not attachment_id:
363             return None
364         action_param = ""
365         if action and action != "view":
366             action_param = "&action=%s" % action
367         return "%sattachment.cgi?id=%s%s" % (config_urls.bug_server_url,
368                                              attachment_id,
369                                              action_param)
370
371     def edit_user_url_for_id(self, user_id):
372         return "%seditusers.cgi?action=edit&userid=%s" % (config_urls.bug_server_url, user_id)
373
374     def _parse_attachment_flag(self,
375                                element,
376                                flag_name,
377                                attachment,
378                                result_key):
379         flag = element.find('flag', attrs={'name': flag_name})
380         if flag:
381             attachment[flag_name] = flag['status']
382             if flag['status'] == '+':
383                 attachment[result_key] = flag['setter']
384         # Sadly show_bug.cgi?ctype=xml does not expose the flag modification date.
385
386     def _string_contents(self, soup):
387         # WebKit's bugzilla instance uses UTF-8.
388         # BeautifulStoneSoup always returns Unicode strings, however
389         # the .string method returns a (unicode) NavigableString.
390         # NavigableString can confuse other parts of the code, so we
391         # convert from NavigableString to a real unicode() object using unicode().
392         return unicode(soup.string)
393
394     # Example: 2010-01-20 14:31 PST
395     # FIXME: Some bugzilla dates seem to have seconds in them?
396     # Python does not support timezones out of the box.
397     # Assume that bugzilla always uses PST (which is true for bugs.webkit.org)
398     _bugzilla_date_format = "%Y-%m-%d %H:%M:%S"
399
400     @classmethod
401     def _parse_date(cls, date_string):
402         (date, time, time_zone) = date_string.split(" ")
403         if time.count(':') == 1:
404             # Add seconds into the time.
405             time += ':0'
406         # Ignore the timezone because python doesn't understand timezones out of the box.
407         date_string = "%s %s" % (date, time)
408         return datetime.strptime(date_string, cls._bugzilla_date_format)
409
410     def _date_contents(self, soup):
411         return self._parse_date(self._string_contents(soup))
412
413     def _parse_attachment_element(self, element, bug_id):
414         attachment = {}
415         attachment['bug_id'] = bug_id
416         attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
417         attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
418         attachment['id'] = int(element.find('attachid').string)
419         # FIXME: No need to parse out the url here.
420         attachment['url'] = self.attachment_url_for_id(attachment['id'])
421         attachment["attach_date"] = self._date_contents(element.find("date"))
422         attachment['name'] = self._string_contents(element.find('desc'))
423         attachment['attacher_email'] = self._string_contents(element.find('attacher'))
424         attachment['type'] = self._string_contents(element.find('type'))
425         self._parse_attachment_flag(
426                 element, 'review', attachment, 'reviewer_email')
427         self._parse_attachment_flag(
428                 element, 'commit-queue', attachment, 'committer_email')
429         return attachment
430
431     def _parse_log_descr_element(self, element):
432         comment = {}
433         comment['comment_email'] = self._string_contents(element.find('who'))
434         comment['comment_date'] = self._date_contents(element.find('bug_when'))
435         comment['text'] = self._string_contents(element.find('thetext'))
436         return comment
437
438     def _parse_bugs_from_xml(self, page):
439         soup = BeautifulSoup(page)
440         # Without the unicode() call, BeautifulSoup occasionally complains of being
441         # passed None for no apparent reason.
442         return [Bug(self._parse_bug_dictionary_from_xml(unicode(bug_xml)), self) for bug_xml in soup('bug')]
443
444     def _parse_bug_dictionary_from_xml(self, page):
445         soup = BeautifulStoneSoup(page, convertEntities=BeautifulStoneSoup.XML_ENTITIES)
446         bug = {}
447         bug["id"] = int(soup.find("bug_id").string)
448         bug["title"] = self._string_contents(soup.find("short_desc"))
449         bug["bug_status"] = self._string_contents(soup.find("bug_status"))
450         dup_id = soup.find("dup_id")
451         if dup_id:
452             bug["dup_id"] = self._string_contents(dup_id)
453         bug["reporter_email"] = self._string_contents(soup.find("reporter"))
454         bug["assigned_to_email"] = self._string_contents(soup.find("assigned_to"))
455         bug["cc_emails"] = [self._string_contents(element) for element in soup.findAll('cc')]
456         bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')]
457         bug["comments"] = [self._parse_log_descr_element(element) for element in soup.findAll('long_desc')]
458
459         return bug
460
461     # Makes testing fetch_*_from_bug() possible until we have a better
462     # BugzillaNetwork abstration.
463
464     def _fetch_bug_page(self, bug_id):
465         bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
466         _log.info("Fetching: %s" % bug_url)
467         return self.browser.open(bug_url)
468
469     def fetch_bug_dictionary(self, bug_id):
470         try:
471             self.authenticate()
472             return self._parse_bug_dictionary_from_xml(self._fetch_bug_page(bug_id))
473         except KeyboardInterrupt:
474             raise
475
476     # FIXME: A BugzillaCache object should provide all these fetch_ methods.
477
478     def fetch_bug(self, bug_id):
479         return Bug(self.fetch_bug_dictionary(bug_id), self)
480
481     def fetch_attachment_contents(self, attachment_id):
482         attachment_url = self.attachment_url_for_id(attachment_id)
483         # We need to authenticate to download patches from security bugs.
484         self.authenticate()
485         return self.browser.open(attachment_url).read()
486
487     def _parse_bug_id_from_attachment_page(self, page):
488         # The "Up" relation happens to point to the bug.
489         title = BeautifulSoup(page).find('div', attrs={'id':'bug_title'})
490         if not title :
491             _log.warning("This attachment does not exist (or you don't have permissions to view it).")
492             return None
493         match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", str(title))
494         if not match:
495             _log.warning("Unable to parse bug id from attachment")
496             return None
497         return int(match.group('bug_id'))
498
499     def bug_id_for_attachment_id(self, attachment_id):
500         self.authenticate()
501
502         attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
503         _log.info("Fetching: %s" % attachment_url)
504         page = self.browser.open(attachment_url)
505         return self._parse_bug_id_from_attachment_page(page)
506
507     # FIXME: This should just return Attachment(id), which should be able to
508     # lazily fetch needed data.
509
510     def fetch_attachment(self, attachment_id):
511         # We could grab all the attachment details off of the attachment edit
512         # page but we already have working code to do so off of the bugs page,
513         # so re-use that.
514         bug_id = self.bug_id_for_attachment_id(attachment_id)
515         if not bug_id:
516             _log.warning("Unable to parse bug_id from attachment {}".format(attachment_id))
517             return None
518         attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True)
519         for attachment in attachments:
520             if attachment.id() == int(attachment_id):
521                 return attachment
522         _log.error("Error in fetching attachment {}, bug_id: {}".format(attachment_id, bug_id))
523         return None  # This should never be hit.
524
525     def authenticate(self):
526         if self.authenticated:
527             return
528
529         credentials = Credentials(config_urls.bug_server_host, git_prefix="bugzilla")
530
531         attempts = 0
532         while not self.authenticated:
533             attempts += 1
534             username, password = credentials.read_credentials(use_stored_credentials=attempts == 1)
535
536             _log.info("Logging in as %s..." % username)
537             self.browser.open(config_urls.bug_server_url +
538                               "index.cgi?GoAheadAndLogIn=1")
539             self.browser.select_form(name="login")
540             self.browser['Bugzilla_login'] = username
541             self.browser['Bugzilla_password'] = password
542             self.browser.find_control("Bugzilla_restrictlogin").items[0].selected = False
543             response = self.browser.submit()
544
545             match = re.search("<title>(.+?)</title>", response.read())
546             # If the resulting page has a title, and it contains the word
547             # "invalid" assume it's the login failure page.
548             if match and re.search("Invalid", match.group(1), re.IGNORECASE):
549                 errorMessage = "Bugzilla login failed: %s" % match.group(1)
550                 # raise an exception only if this was the last attempt
551                 if attempts < 5:
552                     _log.error(errorMessage)
553                 else:
554                     raise Exception(errorMessage)
555             else:
556                 self.authenticated = True
557                 self.username = username
558
559     def _commit_queue_flag(self, commit_flag):
560         if commit_flag == CommitQueueFlag.mark_for_landing:
561             user = self.committers.contributor_by_email(self.username)
562             if not user:
563                 _log.warning("Your Bugzilla login is not listed in contributors.json. Uploading with cq? instead of cq+")
564             elif not user.can_commit:
565                 _log.warning("You're not a committer yet or haven't updated contributors.json yet. Uploading with cq? instead of cq+")
566             else:
567                 return '+'
568
569         if commit_flag != CommitQueueFlag.mark_for_nothing:
570             return '?'
571         return 'X'
572
573     def _fill_attachment_form(self,
574                               description,
575                               file_object,
576                               mark_for_review=False,
577                               commit_flag=CommitQueueFlag.mark_for_nothing,
578                               is_patch=False,
579                               filename=None,
580                               mimetype=None):
581         self.browser['description'] = description
582         if is_patch:
583             self.browser['ispatch'] = ("1",)
584         # FIXME: Should this use self._find_select_element_for_flag?
585         self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
586         self.browser['flag_type-3'] = (self._commit_queue_flag(commit_flag),)
587
588         filename = filename or "%s.patch" % timestamp()
589         if not mimetype:
590             mimetypes.add_type('text/plain', '.patch')  # Make sure mimetypes knows about .patch
591             mimetype, _ = mimetypes.guess_type(filename)
592         if not mimetype:
593             mimetype = "text/plain"  # Bugzilla might auto-guess for us and we might not need this?
594         self.browser.add_file(file_object, mimetype, filename, 'data')
595
596     def _file_object_for_upload(self, file_or_string):
597         if hasattr(file_or_string, 'read'):
598             return file_or_string
599         # Only if file_or_string is not already encoded do we want to encode it.
600         if isinstance(file_or_string, unicode):
601             file_or_string = file_or_string.encode('utf-8')
602         return StringIO.StringIO(file_or_string)
603
604     # timestamp argument is just for unittests.
605     def _filename_for_upload(self, file_object, bug_id, extension="txt", timestamp=timestamp):
606         if hasattr(file_object, "name"):
607             return file_object.name
608         return "bug-%s-%s.%s" % (bug_id, timestamp(), extension)
609
610     def add_attachment_to_bug(self, bug_id, file_or_string, description, filename=None, comment_text=None, mimetype=None):
611         self.authenticate()
612         _log.info('Adding attachment "%s" to %s' % (description, self.bug_url_for_bug_id(bug_id)))
613         self.browser.open(self.add_attachment_url(bug_id))
614         self.browser.select_form(name="entryform")
615         file_object = self._file_object_for_upload(file_or_string)
616         filename = filename or self._filename_for_upload(file_object, bug_id)
617         self._fill_attachment_form(description, file_object, filename=filename, mimetype=mimetype)
618         if comment_text:
619             _log.info(comment_text)
620             self.browser['comment'] = comment_text
621         self.browser.submit()
622
623     @staticmethod
624     def _parse_attachment_id_from_add_patch_to_bug_response(response_html):
625         match = re.search('<title>Attachment (?P<attachment_id>\d+) added to Bug \d+</title>', response_html)
626         if match:
627             return match.group('attachment_id')
628         _log.warning('Unable to parse attachment id')
629         return None
630
631     # FIXME: The arguments to this function should be simplified and then
632     # this should be merged into add_attachment_to_bug
633     def add_patch_to_bug(self,
634                          bug_id,
635                          file_or_string,
636                          description,
637                          comment_text=None,
638                          mark_for_review=False,
639                          mark_for_commit_queue=False,
640                          mark_for_landing=False):
641         self.authenticate()
642         _log.info('Adding patch "%s" to %s' % (description, self.bug_url_for_bug_id(bug_id)))
643
644         self.browser.open(self.add_attachment_url(bug_id))
645         self.browser.select_form(name="entryform")
646         file_object = self._file_object_for_upload(file_or_string)
647         filename = self._filename_for_upload(file_object, bug_id, extension="patch")
648         commit_flag = CommitQueueFlag.mark_for_nothing
649         if mark_for_landing:
650             commit_flag = CommitQueueFlag.mark_for_landing
651         elif mark_for_commit_queue:
652             commit_flag = CommitQueueFlag.mark_for_commit_queue
653
654         self._fill_attachment_form(description,
655                                    file_object,
656                                    mark_for_review=mark_for_review,
657                                    commit_flag=commit_flag,
658                                    is_patch=True,
659                                    filename=filename)
660         if comment_text:
661             _log.info(comment_text)
662             self.browser['comment'] = comment_text
663         response = self.browser.submit()
664         return self._parse_attachment_id_from_add_patch_to_bug_response(response.read())
665
666     # FIXME: There has to be a more concise way to write this method.
667     def _check_create_bug_response(self, response_html):
668         match = re.search("<title>Bug (?P<bug_id>\d+) Submitted[^<]*</title>",
669                           response_html)
670         if match:
671             return match.group('bug_id')
672
673         match = re.search(
674             '<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">',
675             response_html,
676             re.DOTALL)
677         error_message = "FAIL"
678         if match:
679             text_lines = BeautifulSoup(
680                     match.group('error_message')).findAll(text=True)
681             error_message = "\n" + '\n'.join(
682                     ["  " + line.strip()
683                      for line in text_lines if line.strip()])
684         raise Exception("Bug not created: %s" % error_message)
685
686     def create_bug(self,
687                    bug_title,
688                    bug_description,
689                    component=None,
690                    diff=None,
691                    patch_description=None,
692                    cc=None,
693                    blocked=None,
694                    assignee=None,
695                    mark_for_review=False,
696                    mark_for_commit_queue=False):
697         self.authenticate()
698
699         _log.info('Creating bug with title "%s"' % bug_title)
700         self.browser.open(config_urls.bug_server_url + "enter_bug.cgi?product=WebKit")
701         self.browser.select_form(name="Create")
702         component_items = self.browser.find_control('component').items
703         component_names = map(lambda item: item.name, component_items)
704         if not component:
705             component = "New Bugs"
706         if component not in component_names:
707             component = User.prompt_with_list("Please pick a component:", component_names)
708         self.browser["component"] = [component]
709         if cc:
710             self.browser["cc"] = cc
711         if blocked:
712             self.browser["blocked"] = unicode(blocked)
713         if not assignee:
714             assignee = self.username
715         if assignee and not self.browser.find_control("assigned_to").disabled:
716             self.browser["assigned_to"] = assignee
717         self.browser["short_desc"] = bug_title
718         self.browser["comment"] = bug_description
719
720         if diff:
721             # _fill_attachment_form expects a file-like object
722             # Patch files are already binary, so no encoding needed.
723             assert(isinstance(diff, str))
724             patch_file_object = StringIO.StringIO(diff)
725             commit_flag = CommitQueueFlag.mark_for_nothing
726             if mark_for_commit_queue:
727                 commit_flag = CommitQueueFlag.mark_for_commit_queue
728
729             self._fill_attachment_form(
730                     patch_description,
731                     patch_file_object,
732                     mark_for_review=mark_for_review,
733                     commit_flag=commit_flag,
734                     is_patch=True)
735
736         response = self.browser.submit()
737
738         bug_id = self._check_create_bug_response(response.read())
739         _log.info("Bug %s created." % bug_id)
740         _log.info("%sshow_bug.cgi?id=%s" % (config_urls.bug_server_url, bug_id))
741         return bug_id
742
743     def _find_select_element_for_flag(self, flag_name):
744         # FIXME: This will break if we ever re-order attachment flags
745         if flag_name == "review":
746             return self.browser.find_control(type='select', nr=0)
747         elif flag_name == "commit-queue":
748             return self.browser.find_control(type='select', nr=1)
749         raise Exception("Don't know how to find flag named \"%s\"" % flag_name)
750
751     def clear_attachment_flags(self,
752                                attachment_id,
753                                additional_comment_text=None):
754         self.authenticate()
755
756         comment_text = "Clearing flags on attachment: %s" % attachment_id
757         if additional_comment_text:
758             comment_text += "\n\n%s" % additional_comment_text
759         _log.info(comment_text)
760
761         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
762         self.browser.select_form(nr=1)
763         self.browser.set_value(comment_text, name='comment', nr=1)
764         self._find_select_element_for_flag('review').value = ("X",)
765         self._find_select_element_for_flag('commit-queue').value = ("X",)
766         self.browser.submit()
767
768     def set_flag_on_attachment(self,
769                                attachment_id,
770                                flag_name,
771                                flag_value,
772                                comment_text=None):
773         # FIXME: We need a way to test this function on a live bugzilla
774         # instance.
775
776         self.authenticate()
777         _log.info(comment_text)
778         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
779         self.browser.select_form(nr=1)
780
781         if comment_text:
782             self.browser.set_value(comment_text, name='comment', nr=1)
783
784         self._find_select_element_for_flag(flag_name).value = (flag_value,)
785         self.browser.submit()
786
787     # FIXME: All of these bug editing methods have a ridiculous amount of
788     # copy/paste code.
789
790     def obsolete_attachment(self, attachment_id, comment_text=None):
791         self.authenticate()
792
793         _log.info("Obsoleting attachment: %s" % attachment_id)
794         self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
795         self.browser.select_form(nr=1)
796         self.browser.find_control('isobsolete').items[0].selected = True
797         # Also clear any review flag (to remove it from review/commit queues)
798         self._find_select_element_for_flag('review').value = ("X",)
799         self._find_select_element_for_flag('commit-queue').value = ("X",)
800         if comment_text:
801             _log.info(comment_text)
802             # Bugzilla has two textareas named 'comment', one is somehow
803             # hidden.  We want the first.
804             self.browser.set_value(comment_text, name='comment', nr=1)
805         self.browser.submit()
806
807     def add_cc_to_bug(self, bug_id, email_address_list):
808         self.authenticate()
809
810         _log.info("Adding %s to the CC list for bug %s" % (email_address_list, bug_id))
811         self.browser.open(self.bug_url_for_bug_id(bug_id))
812         self.browser.select_form(name="changeform")
813         self.browser["newcc"] = ", ".join(email_address_list)
814         self.browser.submit()
815
816     def post_comment_to_bug(self, bug_id, comment_text, cc=None):
817         self.authenticate()
818
819         _log.info("Adding comment to bug %s" % bug_id)
820         self.browser.open(self.bug_url_for_bug_id(bug_id))
821         self.browser.select_form(name="changeform")
822         self.browser["comment"] = comment_text
823         if cc:
824             self.browser["newcc"] = ", ".join(cc)
825         self.browser.submit()
826
827     def close_bug_as_fixed(self, bug_id, comment_text=None):
828         self.authenticate()
829
830         _log.info("Closing bug %s as fixed" % bug_id)
831         self.browser.open(self.bug_url_for_bug_id(bug_id))
832         self.browser.select_form(name="changeform")
833         if comment_text:
834             self.browser['comment'] = comment_text
835         self.browser['bug_status'] = ['RESOLVED']
836         self.browser['resolution'] = ['FIXED']
837         self.browser.submit()
838
839     def _has_control(self, form, id):
840         return id in [control.id for control in form.controls]
841
842     def reassign_bug(self, bug_id, assignee=None, comment_text=None):
843         self.authenticate()
844
845         if not assignee:
846             assignee = self.username
847
848         _log.info("Assigning bug %s to %s" % (bug_id, assignee))
849         self.browser.open(self.bug_url_for_bug_id(bug_id))
850         self.browser.select_form(name="changeform")
851
852         if not self._has_control(self.browser, "assigned_to"):
853             _log.warning("""Failed to assign bug to you (can't find assigned_to) control.
854 Ignore this message if you don't have EditBugs privileges (https://bugs.webkit.org/userprefs.cgi?tab=permissions)""")
855             return
856
857         if comment_text:
858             _log.info(comment_text)
859             self.browser["comment"] = comment_text
860         self.browser["assigned_to"] = assignee
861         self.browser.submit()
862
863     def reopen_bug(self, bug_id, comment_text):
864         self.authenticate()
865
866         _log.info("Re-opening bug %s" % bug_id)
867         # Bugzilla requires a comment when re-opening a bug, so we know it will
868         # never be None.
869         _log.info(comment_text)
870         self.browser.open(self.bug_url_for_bug_id(bug_id))
871         self.browser.select_form(name="changeform")
872         bug_status = self.browser.find_control("bug_status", type="select")
873         # This is a hack around the fact that ClientForm.ListControl seems to
874         # have no simpler way to ask if a control has an item named "REOPENED"
875         # without using exceptions for control flow.
876         possible_bug_statuses = map(lambda item: item.name, bug_status.items)
877         if "REOPENED" in possible_bug_statuses:
878             bug_status.value = ["REOPENED"]
879         # If the bug was never confirmed it will not have a "REOPENED"
880         # state, but only an "UNCONFIRMED" state.
881         elif "UNCONFIRMED" in possible_bug_statuses:
882             bug_status.value = ["UNCONFIRMED"]
883         else:
884             # FIXME: This logic is slightly backwards.  We won't print this
885             # message if the bug is already open with state "UNCONFIRMED".
886             _log.info("Did not reopen bug %s, it appears to already be open with status %s." % (bug_id, bug_status.value))
887         self.browser['comment'] = comment_text
888         self.browser.submit()