17c5738ea5a33ac880c012b848cad0f3cb134010
[WebKit-https.git] / Websites / perf.webkit.org / tools / pull-os-versions.py
1 #!/usr/bin/python
2
3 import argparse
4 import json
5 import re
6 import sys
7 import subprocess
8 import time
9 import urllib
10 import urllib2
11
12 from xml.dom.minidom import parseString as parseXmlString
13 from util import submit_commits
14 from util import text_content
15 from util import setup_auth
16
17
18 def main(argv):
19     parser = argparse.ArgumentParser()
20     parser.add_argument('--config', required=True, help='Path to a config JSON file')
21     args = parser.parse_args()
22
23     with open(args.config) as config_file:
24         config = json.load(config_file)
25
26     setup_auth(config['server'])
27
28     submission_size = config['submissionSize']
29     reported_revisions = set()
30
31     while True:
32         if 'customCommands' in config:
33             available_builds = []
34             for command in config['customCommands']:
35                 print "Executing", ' '.join(command['command'])
36                 available_builds += available_builds_from_command(config['repositoryName'], command['command'], command['linesToIgnore'])
37                 print "Got %d builds" % len(available_builds)
38         else:
39             url = config['buildSourceURL']
40             print "Fetching available builds from", url
41             available_builds = fetch_available_builds(config['repositoryName'], url, config['trainVersionMap'])
42
43         available_builds = filter(lambda commit: commit['revision'] not in reported_revisions, available_builds)
44         print "%d builds available" % len(available_builds)
45
46         while available_builds:
47             commits_to_submit = available_builds[:submission_size]
48             revisions_to_report = map(lambda commit: commit['revision'], commits_to_submit)
49             print "Submitting builds (%d remaining):" % len(available_builds), json.dumps(revisions_to_report)
50             available_builds = available_builds[submission_size:]
51
52             submit_commits(commits_to_submit, config['server']['url'], config['slave']['name'], config['slave']['password'])
53             reported_revisions |= set(revisions_to_report)
54
55             time.sleep(config['submissionInterval'])
56
57         print "Sleeping for %d seconds" % config['fetchInterval']
58         time.sleep(config['fetchInterval'])
59
60
61 def available_builds_from_command(repository_name, command, lines_to_ignore):
62     try:
63         output = subprocess.check_output(command, stderr=subprocess.STDOUT)
64     except subprocess.CalledProcessError as error:
65         print "Failed:", str(error)
66
67     regex = re.compile(lines_to_ignore)
68     return [{'repository': repository_name, 'revision': line} for line in output.split('\n') if not regex.match(line)]
69
70
71 def fetch_available_builds(repository_name, url, train_version_map):
72     output = urllib2.urlopen(url).read()
73     try:
74         xml = parseXmlString(output)
75     except Exception, error:
76         raise Exception(error, output)
77     available_builds = []
78     for image in xml.getElementsByTagName('baseASR'):
79         id = text_content(image.getElementsByTagName('id')[0])
80         train = text_content(image.getElementsByTagName('train')[0])
81         build = text_content(image.getElementsByTagName('build')[0])
82         if train not in train_version_map:
83             continue
84         available_builds.append({
85             'repository': repository_name,
86             'revision': train_version_map[train] + ' ' + build})
87
88     return available_builds
89
90
91 if __name__ == "__main__":
92     main(sys.argv)