build-requests should use conform to JSON API format
[WebKit-https.git] / Websites / perf.webkit.org / tools / sync-with-buildbot.py
1 #!/usr/bin/python
2
3 import argparse
4 import base64
5 import copy
6 import json
7 import sys
8 import time
9 import urllib
10 import urllib2
11
12 from util import setup_auth
13
14
15 def main():
16     parser = argparse.ArgumentParser()
17     parser.add_argument('--triggerable', required=True, help='The name of the triggerable to process. e.g. build-webkit')
18     parser.add_argument('--buildbot-url', required=True, help='URL for a buildbot builder; e.g. "https://build.webkit.org/"')
19     parser.add_argument('--builder-config-json', required=True, help='The path to a JSON file that specifies which test and platform will be posted to which builder. '
20         'The JSON should contain an array of dictionaries with keys "platform", "test", and "builder" '
21         'with the platform name (e.g. mountainlion), the test path (e.g. ["Parser", "html5-full-render"]), and the builder name (e.g. Apple MountainLion Release (Perf)) as values.')
22     parser.add_argument('--server-config-json', required=True, help='The path to a JSON file that specifies the perf dashboard.')
23
24     parser.add_argument('--lookback-count', type=int, default=10, help='The number of builds to look back when finding in-progress builds on the buildbot')
25     parser.add_argument('--seconds-to-sleep', type=float, default=120, help='The seconds to sleep between iterations')
26     args = parser.parse_args()
27
28     configurations = load_config(args.builder_config_json, args.buildbot_url.strip('/'))
29
30     with open(args.server_config_json) as server_config_json:
31         server_config = json.load(server_config_json)
32         setup_auth(server_config['server'])
33
34     build_requests_url = server_config['server']['url'] + '/api/build-requests/' + args.triggerable
35
36     request_updates = {}
37     while True:
38         request_updates.update(find_request_updates(configurations, args.lookback_count))
39         if request_updates:
40             print 'Updating the build requests %s...' % ', '.join(map(str, request_updates.keys()))
41         else:
42             print 'No updates...'
43
44         payload = {
45             'buildRequestUpdates': request_updates,
46             'slaveName': server_config['slave']['name'],
47             'slavePassword': server_config['slave']['password']}
48         response = update_and_fetch_build_requests(build_requests_url, payload)
49         open_requests = response.get('buildRequests', [])
50
51         root_sets = organize_root_sets_by_id_and_repository_names(response.get('rootSets', {}), response.get('roots', []))
52
53         for request in filter(lambda request: request['status'] == 'pending', open_requests):
54             config = config_for_request(configurations, request)
55             if not config:
56                 print >> sys.stderr, "Failed to find the configuration for request %s: %s" % (str(request['id']), json.dumps(request))
57                 continue
58             if config and len(config['scheduledRequests']) < 1:
59                 print "Scheduling the build request %s..." % str(request['id'])
60                 schedule_request(config, request, root_sets)
61
62         request_updates = find_stale_request_updates(configurations, open_requests, request_updates.keys())
63         if request_updates:
64             print "Found stale build requests %s..." % ', '.join(map(str, request_updates.keys()))
65
66         time.sleep(args.seconds_to_sleep)
67
68
69 def load_config(config_json_path, buildbot_url):
70     with open(config_json_path) as config_json:
71         configurations = json.load(config_json)
72
73     for config in configurations:
74         escaped_builder_name = urllib.quote(config['builder'])
75         config['url'] = '%s/builders/%s/' % (buildbot_url, escaped_builder_name)
76         config['jsonURL'] = '%s/json/builders/%s/' % (buildbot_url, escaped_builder_name)
77         config['scheduledRequests'] = set()
78
79     return configurations
80
81
82 def find_request_updates(configurations, lookback_count):
83     request_updates = {}
84
85     for config in configurations:
86         try:
87             pending_builds = fetch_json(config['jsonURL'] + 'pendingBuilds')
88             scheduled_requests = filter(None, [request_id_from_build(config, build) for build in pending_builds])
89             for request_id in scheduled_requests:
90                 request_updates[request_id] = {'status': 'scheduled', 'url': config['url']}
91             config['scheduledRequests'] = set(scheduled_requests)
92         except (IOError, ValueError) as error:
93             print >> sys.stderr, "Failed to fetch pending builds for %s: %s" % (config['builder'], str(error))
94
95     for config in configurations:
96         for i in range(1, lookback_count + 1):
97             build_error = None
98             build_index = -i
99             try:
100                 build = fetch_json(config['jsonURL'] + 'builds/%d' % build_index)
101                 request_id = request_id_from_build(config, build)
102                 if not request_id:
103                     continue
104
105                 in_progress = build.get('currentStep')
106                 if in_progress:
107                     request_updates[request_id] = {'status': 'running', 'url': config['url']}
108                     config['scheduledRequests'].discard(request_id)
109                 else:
110                     url = config['url'] + 'builds/' + str(build['number'])
111                     request_updates[request_id] = {'status': 'failedIfNotCompleted', 'url': url}
112             except urllib2.HTTPError as error:
113                 if error.code == 404:
114                     break
115                 else:
116                     build_error = error
117             except ValueError as error:
118                 build_error = error
119             if build_error:
120                 print >> sys.stderr, "Failed to fetch build %d for %s: %s" % (build_index, config['builder'], str(build_error))
121
122     return request_updates
123
124
125 def update_and_fetch_build_requests(build_requests_url, payload):
126     try:
127         response = fetch_json(build_requests_url, payload=json.dumps(payload))
128         if response['status'] != 'OK':
129             raise ValueError(response['status'])
130         return response
131     except (IOError, ValueError) as error:
132         print >> sys.stderr, 'Failed to update or fetch build requests at %s: %s' % (build_requests_url, str(error))
133     return {}
134
135
136 def find_stale_request_updates(configurations, open_requests, requests_on_buildbot):
137     request_updates = {}
138     for request in open_requests:
139         request_id = int(request['id'])
140         should_be_on_buildbot = request['status'] in ('scheduled', 'running')
141         if should_be_on_buildbot and request_id not in requests_on_buildbot:
142             config = config_for_request(configurations, request)
143             if config:
144                 request_updates[request_id] = {'status': 'failed', 'url': config['url']}
145     return request_updates
146
147
148 def organize_root_sets_by_id_and_repository_names(root_sets, roots):
149     result = {}
150     root_by_id = {}
151     for root in roots:
152         root_by_id[root['id']] = root
153
154     for root_set in root_sets:
155         roots_by_repository = {}
156         for root_id in root_set['roots']:
157             root = root_by_id[root_id]
158             roots_by_repository[root['repository']] = root
159         result[root_set['id']] = roots_by_repository
160
161     return result
162
163
164 def schedule_request(config, request, root_sets):
165     roots = root_sets[request['rootSet']]
166     payload = {}
167     for property_name, property_value in config['arguments'].iteritems():
168         if not isinstance(property_value, dict):
169             payload[property_name] = property_value
170         elif 'root' in property_value:
171             repository_name = property_value['root']
172             if repository_name in roots:
173                 payload[property_name] = roots[repository_name]['revision']
174         elif 'rootsExcluding' in property_value:
175             excluded_roots = property_value['rootsExcluding']
176             filtered_roots = {}
177             for root_name in roots:
178                 if root_name not in excluded_roots:
179                     filtered_roots[root_name] = roots[root_name]
180             payload[property_name] = json.dumps(filtered_roots)
181         else:
182             print >> sys.stderr, "Failed to process an argument %s: %s" % (property_name, property_value)
183             return
184     payload[config['buildRequestArgument']] = request['id']
185
186     try:
187         urllib2.urlopen(urllib2.Request(config['url'] + 'force'), urllib.urlencode(payload))
188         config['scheduledRequests'].add(request['id'])
189     except (IOError, ValueError) as error:
190         print >> sys.stderr, "Failed to fetch pending builds for %s: %s" % (config['builder'], str(error))
191
192
193 def config_for_request(configurations, request):
194     for config in configurations:
195         if config['platform'] == request['platform'] and config['test'] == request['test']:
196             return config
197     return None
198
199
200 def fetch_json(url, payload=None):
201     request = urllib2.Request(url)
202     response = urllib2.urlopen(request, payload).read()
203     try:
204         return json.loads(response)
205     except ValueError as error:
206         raise ValueError(str(error) + '\n' + response)
207
208
209 def property_value_from_build(build, name):
210     for prop in build.get('properties', []):
211         if prop[0] == name:
212             return prop[1]
213     return None
214
215
216 def request_id_from_build(config, build):
217     job_id = property_value_from_build(build, config['buildRequestArgument'])
218     return int(job_id) if job_id and job_id.isdigit() else None
219
220
221 if __name__ == "__main__":
222     main()