12ebb23432f90177cc09c405aecd756bdea39be6
[WebKit-https.git] / Websites / perf.webkit.org / tools / pull-os-versions.py
1 #!/usr/bin/python
2
3 import argparse
4 import json
5 import re
6 import sys
7 import subprocess
8 import time
9 import urllib
10 import urllib2
11
12 from xml.dom.minidom import parseString as parseXmlString
13 from util import submit_commits
14 from util import text_content
15
16
17 HTTP_AUTH_HANDLERS = {
18     'basic': urllib2.HTTPBasicAuthHandler,
19     'digest': urllib2.HTTPDigestAuthHandler,
20 }
21
22
23 def main(argv):
24     parser = argparse.ArgumentParser()
25     parser.add_argument('--config', required=True, help='Path to a config JSON file')
26     args = parser.parse_args()
27
28     with open(args.config) as config_file:
29         config = json.load(config_file)
30
31     setup_auth(config['server'])
32
33     submission_size = config['submissionSize']
34     reported_revisions = set()
35
36     while True:
37         if 'customCommands' in config:
38             available_builds = []
39             for command in config['customCommands']:
40                 print "Executing", ' '.join(command['command'])
41                 available_builds += available_builds_from_command(config['repositoryName'], command['command'], command['linesToIgnore'])
42                 print "Got %d builds" % len(available_builds)
43         else:
44             url = config['buildSourceURL']
45             print "Fetching available builds from", url
46             available_builds = fetch_available_builds(config['repositoryName'], url, config['trainVersionMap'])
47
48         available_builds = filter(lambda commit: commit['revision'] not in reported_revisions, available_builds)
49         print "%d builds available" % len(available_builds)
50
51         while available_builds:
52             commits_to_submit = available_builds[:submission_size]
53             revisions_to_report = map(lambda commit: commit['revision'], commits_to_submit)
54             print "Submitting builds (%d remaining):" % len(available_builds), json.dumps(revisions_to_report)
55             available_builds = available_builds[submission_size:]
56
57             submit_commits(commits_to_submit, config['server']['url'], config['slave']['name'], config['slave']['password'])
58             reported_revisions |= set(revisions_to_report)
59
60             time.sleep(config['submissionInterval'])
61
62         print "Sleeping for %d seconds", config['fetchInterval']
63         time.sleep(config['fetchInterval'])
64
65
66 def setup_auth(server):
67     auth = server.get('auth')
68     if not auth:
69         return
70
71     password_manager = urllib2.HTTPPasswordMgr()
72     password_manager.add_password(realm=auth['realm'], uri=server['url'], user=auth['username'], passwd=auth['password'])
73     auth_handler = HTTP_AUTH_HANDLERS[auth['type']](password_manager)
74     urllib2.install_opener(urllib2.build_opener(auth_handler))
75
76
77 def available_builds_from_command(repository_name, command, lines_to_ignore):
78     try:
79         output = subprocess.check_output(command, stderr=subprocess.STDOUT)
80     except subprocess.CalledProcessError as error:
81         print "Failed:", str(error)
82
83     regex = re.compile(lines_to_ignore)
84     return [{'repository': repository_name, 'revision': line} for line in output.split('\n') if not regex.match(line)]
85
86
87 def fetch_available_builds(repository_name, url, train_version_map):
88     output = urllib2.urlopen(url).read()
89     try:
90         xml = parseXmlString(output)
91     except Exception, error:
92         raise Exception(error, output)
93     available_builds = []
94     for image in xml.getElementsByTagName('baseASR'):
95         id = text_content(image.getElementsByTagName('id')[0])
96         train = text_content(image.getElementsByTagName('train')[0])
97         build = text_content(image.getElementsByTagName('build')[0])
98         if train not in train_version_map:
99             continue
100         available_builds.append({
101             'repository': repository_name,
102             'revision': train_version_map[train] + ' ' + build})
103
104     return available_builds
105
106
107 if __name__ == "__main__":
108     main(sys.argv)