Versioning.
[WebKit-https.git] / Websites / perf.webkit.org / tools / pull-svn.py
1 #!/usr/bin/python
2
3 import json
4 import re
5 import subprocess
6 import sys
7 import time
8 import urllib2
9
10 from xml.dom.minidom import parseString as parseXmlString
11
12
13 def main(argv):
14     if len(argv) < 7:
15         sys.exit('Usage: pull-svn <repository-name> <repository-URL> <dashboard-URL> <builder-name> <builder-password> <seconds-to-sleep> [<email-to-name-helper>]')
16
17     repository_name = argv[1]
18     repository_url = argv[2]
19     dashboard_url = argv[3]
20     builder_name = argv[4]
21     builder_password = argv[5]
22     seconds_to_sleep = float(argv[6])
23     email_to_name_helper = argv[7] if len(argv) > 7 else None
24
25     print "Submitting revision logs for %s at %s to %s" % (repository_name, repository_url, dashboard_url)
26
27     revision_to_fetch = determine_first_revision_to_fetch(dashboard_url, repository_name)
28     print "Start fetching commits at r%d" % revision_to_fetch
29
30     pending_commits_to_send = []
31
32     while True:
33         commit = fetch_commit_and_resolve_author(repository_name, repository_url, email_to_name_helper, revision_to_fetch)
34
35         if commit:
36             print "Fetched r%d." % revision_to_fetch
37             pending_commits_to_send += [commit]
38             revision_to_fetch += 1
39         else:
40             print "Revision %d not found" % revision_to_fetch
41
42         if not commit or len(pending_commits_to_send) >= 10:
43             if pending_commits_to_send:
44                 print "Submitting the above commits to %s..." % dashboard_url
45                 submit_commits(pending_commits_to_send, dashboard_url, builder_name, builder_password)
46                 print "Successfully submitted."
47             pending_commits_to_send = []
48             time.sleep(seconds_to_sleep)
49
50
51 def determine_first_revision_to_fetch(dashboard_url, repository_name):
52     try:
53         last_reported_revision = fetch_revision_from_dasbhoard(dashboard_url, repository_name, 'last-reported')
54     except Exception as error:
55         sys.exit('Failed to fetch the latest reported commit: ' + str(error))
56
57     if last_reported_revision:
58         return last_reported_revision + 1
59
60     # FIXME: This is a problematic if dashboard can get results for revisions older than oldest_revision
61     # in the future because we never refetch older revisions.
62     try:
63         return fetch_revision_from_dasbhoard(dashboard_url, repository_name, 'oldest') or 1
64     except Exception as error:
65         sys.exit('Failed to fetch the oldest commit: ' + str(error))
66
67
68 def fetch_revision_from_dasbhoard(dashboard_url, repository_name, filter):
69     result = urllib2.urlopen(dashboard_url + '/api/commits/' + repository_name + '/' + filter).read()
70     parsed_result = json.loads(result)
71     if parsed_result['status'] != 'OK' and parsed_result['status'] != 'RepositoryNotFound':
72         raise Exception(result)
73     commits = parsed_result.get('commits')
74     return int(commits[0]['revision']) if commits else None
75
76
77 def fetch_commit_and_resolve_author(repository_name, repository_url, email_to_name_helper, revision_to_fetch):
78     try:
79         commit = fetch_commit(repository_name, repository_url, revision_to_fetch)
80     except Exception as error:
81         sys.exit('Failed to fetch the commit %d: %s' % (revision_to_fetch, str(error)))
82
83     if not commit:
84         return None
85
86     email = commit['author']['email']
87     try:
88         name = resolve_author_name_from_email(email_to_name_helper, email) if email_to_name_helper else None
89         if name:
90             commit['author']['name'] = name
91     except Exception as error:
92         sys.exit('Failed to resolve the author name from an email %s: %s' % (email, str(error)))
93
94     return commit
95
96
97 def fetch_commit(repository_name, repository_url, revision):
98     args = ['svn', 'log', '--revision', str(revision), '--xml', repository_url]
99     try:
100         output = subprocess.check_output(args, stderr=subprocess.STDOUT)
101     except subprocess.CalledProcessError as error:
102         if (': No such revision ' + str(revision)) in error.output:
103             return None
104         raise error
105     xml = parseXmlString(output)
106     time = textContent(xml.getElementsByTagName("date")[0])
107     author_email = textContent(xml.getElementsByTagName("author")[0])
108     message = textContent(xml.getElementsByTagName("msg")[0])
109     return {
110         'repository': repository_name,
111         'revision': revision,
112         'time': time,
113         'author': {'email': author_email},
114         'message': message,
115     }
116
117
118 def textContent(element):
119     text = ''
120     for child in element.childNodes:
121         if child.nodeType == child.TEXT_NODE:
122             text += child.data
123         else:
124             text += textContent(child)
125     return text
126
127
128 name_email_compound_regex = re.compile(r'^\s*(?P<name>(\".+\"|[^<]+?))\s*\<(?P<email>.+)\>\s*$')
129
130
131 def resolve_author_name_from_email(helper, email):
132     output = subprocess.check_output(helper + ' ' + email, shell=True)
133     match = name_email_compound_regex.match(output)
134     if match:
135         return match.group('name').strip('"')
136     return output.strip()
137
138
139 def submit_commits(commits, dashboard_url, builder_name, builder_password):
140     try:
141         payload = json.dumps({
142             'builderName': builder_name,
143             'builderPassword': builder_password,
144             'commits': commits,
145         })
146         request = urllib2.Request(dashboard_url + '/api/report-commits')
147         request.add_header('Content-Type', 'application/json')
148         request.add_header('Content-Length', len(payload))
149
150         output = urllib2.urlopen(request, payload).read()
151         try:
152             result = json.loads(output)
153         except Exception, error:
154             raise Exception(error, output)
155
156         if result.get('status') != 'OK':
157             raise Exception(result)
158     except Exception as error:
159         sys.exit('Failed to submit commits: %s' % str(error))
160
161
162 if __name__ == "__main__":
163     main(sys.argv)