build.webkit.org/dashboard: Move layoutTestResultsDirectoryURLForIteration implementa...
[WebKit-https.git] / Websites / perf.webkit.org / tools / pull-svn.py
1 #!/usr/bin/python
2
3 import argparse
4 import json
5 import re
6 import subprocess
7 import sys
8 import time
9 import urllib2
10
11 from xml.dom.minidom import parseString as parseXmlString
12 from util import load_server_config
13 from util import submit_commits
14 from util import text_content
15
16
17 def main(argv):
18     parser = argparse.ArgumentParser()
19     parser.add_argument('--svn-config-json', required=True, help='The path to a JSON file that specifies subversion syncing options')
20     parser.add_argument('--server-config-json', required=True, help='The path to a JSON file that specifies the perf dashboard')
21     parser.add_argument('--seconds-to-sleep', type=float, default=900, help='The seconds to sleep between iterations')
22     parser.add_argument('--max-fetch-count', type=int, default=10, help='The number of commits to fetch at once')
23     args = parser.parse_args()
24
25     with open(args.svn_config_json) as svn_config_json:
26         svn_config = json.load(svn_config_json)
27
28     while True:
29         server_config = load_server_config(args.server_config_json)
30         for repository_info in svn_config:
31             fetch_commits_and_submit(repository_info, server_config, args.max_fetch_count)
32         print "Sleeping for %d seconds..." % args.seconds_to_sleep
33         time.sleep(args.seconds_to_sleep)
34
35
36 def fetch_commits_and_submit(repository, server_config, max_fetch_count):
37     assert 'name' in repository, 'The repository name should be specified'
38     assert 'url' in repository, 'The SVN repository URL should be specified'
39
40     if 'revisionToFetch' not in repository:
41         print "Determining the stating revision for %s" % repository['name']
42         repository['revisionToFecth'] = determine_first_revision_to_fetch(server_config['server']['url'], repository['name'])
43
44     if 'useServerAuth' in repository:
45         repository['username'] = server_config['server']['auth']['username']
46         repository['password'] = server_config['server']['auth']['password']
47
48     pending_commits = []
49     for unused in range(max_fetch_count):
50         commit = fetch_commit_and_resolve_author(repository, repository.get('accountNameFinderScript', None), repository['revisionToFecth'])
51         if not commit:
52             break
53         pending_commits += [commit]
54         repository['revisionToFecth'] += 1
55
56     if not pending_commits:
57         print "No new revision found for %s (waiting for r%d)" % (repository['name'], repository['revisionToFecth'])
58         return
59
60     revision_list = 'r' + ', r'.join(map(lambda commit: str(commit['revision']), pending_commits))
61     print "Submitting revisions %s for %s to %s" % (revision_list, repository['name'], server_config['server']['url'])
62     submit_commits(pending_commits, server_config['server']['url'], server_config['slave']['name'], server_config['slave']['password'])
63     print "Successfully submitted."
64     print
65
66
67 def determine_first_revision_to_fetch(dashboard_url, repository_name):
68     try:
69         last_reported_revision = fetch_revision_from_dasbhoard(dashboard_url, repository_name, 'last-reported')
70     except Exception as error:
71         sys.exit('Failed to fetch the latest reported commit: ' + str(error))
72
73     if last_reported_revision:
74         return last_reported_revision + 1
75
76     # FIXME: This is a problematic if dashboard can get results for revisions older than oldest_revision
77     # in the future because we never refetch older revisions.
78     try:
79         return fetch_revision_from_dasbhoard(dashboard_url, repository_name, 'oldest') or 1
80     except Exception as error:
81         sys.exit('Failed to fetch the oldest commit: ' + str(error))
82
83
84 def fetch_revision_from_dasbhoard(dashboard_url, repository_name, filter):
85     result = urllib2.urlopen(dashboard_url + '/api/commits/' + repository_name + '/' + filter).read()
86     parsed_result = json.loads(result)
87     if parsed_result['status'] != 'OK' and parsed_result['status'] != 'RepositoryNotFound':
88         raise Exception(result)
89     commits = parsed_result.get('commits')
90     return int(commits[0]['revision']) if commits else None
91
92
93 def fetch_commit_and_resolve_author(repository, account_to_name_helper, revision_to_fetch):
94     try:
95         commit = fetch_commit(repository, revision_to_fetch)
96     except Exception as error:
97         sys.exit('Failed to fetch the commit %d: %s' % (revision_to_fetch, str(error)))
98
99     if not commit:
100         return None
101
102     account = commit['author']['account']
103     try:
104         name = resolve_author_name_from_account(account_to_name_helper, account) if account_to_name_helper else None
105         if name:
106             commit['author']['name'] = name
107     except Exception as error:
108         sys.exit('Failed to resolve the author name from an account %s: %s' % (account, str(error)))
109
110     return commit
111
112
113 def fetch_commit(repository, revision):
114     args = ['svn', 'log', '--revision', str(revision), '--xml', repository['url'], '--non-interactive']
115     if 'username' in repository and 'password' in repository:
116         args += ['--no-auth-cache', '--username', repository['username'], '--password', repository['password']]
117     if repository.get('trustCertificate', False):
118         args += ['--trust-server-cert']
119
120     try:
121         output = subprocess.check_output(args, stderr=subprocess.STDOUT)
122     except subprocess.CalledProcessError as error:
123         if (': No such revision ' + str(revision)) in error.output:
124             return None
125         raise error
126     xml = parseXmlString(output)
127     time = text_content(xml.getElementsByTagName("date")[0])
128     author_account = text_content(xml.getElementsByTagName("author")[0])
129     message = text_content(xml.getElementsByTagName("msg")[0])
130     return {
131         'repository': repository['name'],
132         'revision': revision,
133         'time': time,
134         'author': {'account': author_account},
135         'message': message,
136     }
137
138
139 name_account_compound_regex = re.compile(r'^\s*(?P<name>(\".+\"|[^<]+?))\s*\<(?P<account>.+)\>\s*$')
140
141
142 def resolve_author_name_from_account(helper, account):
143     output = subprocess.check_output(helper + [account])
144     match = name_account_compound_regex.match(output)
145     if match:
146         return match.group('name').strip('"')
147     return output.strip()
148
149
150 if __name__ == "__main__":
151     main(sys.argv)