Failure when building WebKit for appletvsimulator.
[WebKit-https.git] / Websites / perf.webkit.org / tools / pull-svn.py
1 #!/usr/bin/python
2
3 import json
4 import re
5 import subprocess
6 import sys
7 import time
8 import urllib2
9
10 from xml.dom.minidom import parseString as parseXmlString
11
12
13 def main(argv):
14     if len(argv) < 7:
15         sys.exit('Usage: pull-svn <repository-name> <repository-URL> <dashboard-URL> <slave-name> <slave-password> <seconds-to-sleep> [<account-to-name-helper>]')
16
17     repository_name = argv[1]
18     repository_url = argv[2]
19     dashboard_url = argv[3]
20     slave_name = argv[4]
21     slave_password = argv[5]
22     seconds_to_sleep = float(argv[6])
23     account_to_name_helper = argv[7] if len(argv) > 7 else None
24
25     print "Submitting revision logs for %s at %s to %s" % (repository_name, repository_url, dashboard_url)
26
27     revision_to_fetch = determine_first_revision_to_fetch(dashboard_url, repository_name)
28     print "Start fetching commits at r%d" % revision_to_fetch
29
30     pending_commits_to_send = []
31
32     while True:
33         commit = fetch_commit_and_resolve_author(repository_name, repository_url, account_to_name_helper, revision_to_fetch)
34
35         if commit:
36             print "Fetched r%d." % revision_to_fetch
37             pending_commits_to_send += [commit]
38             revision_to_fetch += 1
39         else:
40             print "Revision %d not found" % revision_to_fetch
41
42         if not commit or len(pending_commits_to_send) >= 10:
43             if pending_commits_to_send:
44                 print "Submitting the above commits to %s..." % dashboard_url
45                 submit_commits(pending_commits_to_send, dashboard_url, slave_name, slave_password)
46                 print "Successfully submitted."
47             pending_commits_to_send = []
48             time.sleep(seconds_to_sleep)
49
50
51 def determine_first_revision_to_fetch(dashboard_url, repository_name):
52     try:
53         last_reported_revision = fetch_revision_from_dasbhoard(dashboard_url, repository_name, 'last-reported')
54     except Exception as error:
55         sys.exit('Failed to fetch the latest reported commit: ' + str(error))
56
57     if last_reported_revision:
58         return last_reported_revision + 1
59
60     # FIXME: This is a problematic if dashboard can get results for revisions older than oldest_revision
61     # in the future because we never refetch older revisions.
62     try:
63         return fetch_revision_from_dasbhoard(dashboard_url, repository_name, 'oldest') or 1
64     except Exception as error:
65         sys.exit('Failed to fetch the oldest commit: ' + str(error))
66
67
68 def fetch_revision_from_dasbhoard(dashboard_url, repository_name, filter):
69     result = urllib2.urlopen(dashboard_url + '/api/commits/' + repository_name + '/' + filter).read()
70     parsed_result = json.loads(result)
71     if parsed_result['status'] != 'OK' and parsed_result['status'] != 'RepositoryNotFound':
72         raise Exception(result)
73     commits = parsed_result.get('commits')
74     return int(commits[0]['revision']) if commits else None
75
76
77 def fetch_commit_and_resolve_author(repository_name, repository_url, account_to_name_helper, revision_to_fetch):
78     try:
79         commit = fetch_commit(repository_name, repository_url, revision_to_fetch)
80     except Exception as error:
81         sys.exit('Failed to fetch the commit %d: %s' % (revision_to_fetch, str(error)))
82
83     if not commit:
84         return None
85
86     account = commit['author']['account']
87     try:
88         name = resolve_author_name_from_account(account_to_name_helper, account) if account_to_name_helper else None
89         if name:
90             commit['author']['name'] = name
91     except Exception as error:
92         sys.exit('Failed to resolve the author name from an account %s: %s' % (account, str(error)))
93
94     return commit
95
96
97 def fetch_commit(repository_name, repository_url, revision):
98     args = ['svn', 'log', '--revision', str(revision), '--xml', repository_url]
99     try:
100         output = subprocess.check_output(args, stderr=subprocess.STDOUT)
101     except subprocess.CalledProcessError as error:
102         if (': No such revision ' + str(revision)) in error.output:
103             return None
104         raise error
105     xml = parseXmlString(output)
106     time = textContent(xml.getElementsByTagName("date")[0])
107     author_account = textContent(xml.getElementsByTagName("author")[0])
108     message = textContent(xml.getElementsByTagName("msg")[0])
109     return {
110         'repository': repository_name,
111         'revision': revision,
112         'time': time,
113         'author': {'account': author_account},
114         'message': message,
115     }
116
117
118 def textContent(element):
119     text = ''
120     for child in element.childNodes:
121         if child.nodeType == child.TEXT_NODE:
122             text += child.data
123         else:
124             text += textContent(child)
125     return text
126
127
128 name_account_compound_regex = re.compile(r'^\s*(?P<name>(\".+\"|[^<]+?))\s*\<(?P<account>.+)\>\s*$')
129
130
131 def resolve_author_name_from_account(helper, account):
132     output = subprocess.check_output(helper + ' ' + account, shell=True)
133     match = name_account_compound_regex.match(output)
134     if match:
135         return match.group('name').strip('"')
136     return output.strip()
137
138
139 def submit_commits(commits, dashboard_url, slave_name, slave_password):
140     try:
141         payload = json.dumps({
142             'slaveName': slave_name,
143             'slavePassword': slave_password,
144             'commits': commits,
145         })
146         request = urllib2.Request(dashboard_url + '/api/report-commits')
147         request.add_header('Content-Type', 'application/json')
148         request.add_header('Content-Length', len(payload))
149
150         output = urllib2.urlopen(request, payload).read()
151         try:
152             result = json.loads(output)
153         except Exception, error:
154             raise Exception(error, output)
155
156         if result.get('status') != 'OK':
157             raise Exception(result)
158     except Exception as error:
159         sys.exit('Failed to submit commits: %s' % str(error))
160
161
162 if __name__ == "__main__":
163     main(sys.argv)