Refactor duplicate code for calling into media controls
[WebKit-https.git] / Websites / perf.webkit.org / tools / pull-os-versions.py
1 #!/usr/bin/python
2
3 import argparse
4 import json
5 import operator
6 import re
7 import sys
8 import subprocess
9 import time
10 import urllib
11 import urllib2
12
13 from datetime import datetime
14 from xml.dom.minidom import parseString as parseXmlString
15 from util import submit_commits
16 from util import text_content
17 from util import load_server_config
18
19
20 def main(argv):
21     parser = argparse.ArgumentParser()
22     parser.add_argument('--os-config-json', required=True, help='The path to a JSON that specifies how to fetch OS build information')
23     parser.add_argument('--server-config-json', required=True, help='The path to a JSON file that specifies the perf dashboard')
24     parser.add_argument('--seconds-to-sleep', type=float, default=43200, help='The seconds to sleep between iterations')
25     args = parser.parse_args()
26
27     with open(args.os_config_json) as os_config_json:
28         os_config_list = json.load(os_config_json)
29
30     fetchers = [OSBuildFetcher(os_config) for os_config in os_config_list]
31
32     while True:
33         server_config = load_server_config(args.server_config_json)
34         for fetcher in fetchers:
35             fetcher.fetch_and_report_new_builds(server_config)
36         print "Sleeping for %d seconds" % args.seconds_to_sleep
37         time.sleep(args.seconds_to_sleep)
38
39
40 # FIXME: Move other static functions into this class.
41 class OSBuildFetcher:
42     def __init__(self, os_config):
43         self._os_config = os_config
44         self._reported_revisions = set()
45
46     def _fetch_available_builds(self):
47         config = self._os_config
48         repository_name = self._os_config['name']
49
50         if 'customCommands' in config:
51             available_builds = []
52             for command in config['customCommands']:
53                 print "Executing", ' '.join(command['command'])
54                 available_builds += available_builds_from_command(repository_name, command['command'], command['linesToIgnore'])
55         else:
56             url = config['buildSourceURL']
57             print "Fetching available builds from", url
58             available_builds = fetch_available_builds(repository_name, url, config['trainVersionMap'])
59         return available_builds
60
61     def fetch_and_report_new_builds(self, server_config):
62         available_builds = self._fetch_available_builds()
63         reported_revisions = self._reported_revisions
64         print 'Found %d builds' % len(available_builds)
65
66         available_builds = filter(lambda commit: commit['revision'] not in reported_revisions, available_builds)
67         self._assign_order(available_builds)
68
69         print "Submitting %d builds" % len(available_builds)
70         submit_commits(available_builds, server_config['server']['url'], server_config['slave']['name'], server_config['slave']['password'])
71         reported_revisions |= set(map(lambda commit: commit['revision'], available_builds))
72
73     @staticmethod
74     def _assign_order(builds):
75         build_name_regex = re.compile(r'(?P<major>\d+)(?P<kind>\w)(?P<minor>\d+)(?P<variant>\w*)')
76         for commit in builds:
77             match = build_name_regex.search(commit['revision'])
78             major = int(match.group('major'))
79             kind = ord(match.group('kind').upper()) - ord('A')
80             minor = int(match.group('minor'))
81             variant = ord(match.group('variant').upper()) - ord('A') + 1 if match.group('variant') else 0
82             commit['order'] = ((major * 100 + kind) * 10000 + minor) * 100 + variant
83
84
85 def available_builds_from_command(repository_name, command, lines_to_ignore):
86     try:
87         output = subprocess.check_output(command, stderr=subprocess.STDOUT)
88     except subprocess.CalledProcessError as error:
89         print "Failed:", str(error)
90         return []
91
92     regex = re.compile(lines_to_ignore)
93     return [{'repository': repository_name, 'revision': line} for line in output.split('\n') if not regex.match(line)]
94
95
96 def fetch_available_builds(repository_name, url, train_version_map):
97     output = urllib2.urlopen(url).read()
98     try:
99         xml = parseXmlString(output)
100     except Exception, error:
101         raise Exception(error, output)
102     available_builds = []
103     for image in xml.getElementsByTagName('baseASR'):
104         id = text_content(image.getElementsByTagName('id')[0])
105         train = text_content(image.getElementsByTagName('train')[0])
106         build = text_content(image.getElementsByTagName('build')[0])
107         if train not in train_version_map:
108             continue
109         available_builds.append({
110             'repository': repository_name,
111             'revision': train_version_map[train] + ' ' + build})
112
113     return available_builds
114
115
116 if __name__ == "__main__":
117     main(sys.argv)