build.webkit.org/dashboard: Move layoutTestResultsDirectoryURLForIteration implementa...
[WebKit-https.git] / Websites / perf.webkit.org / tools / sync-with-buildbot.py
1 #!/usr/bin/python
2
3 import argparse
4 import base64
5 import copy
6 import json
7 import sys
8 import time
9 import urllib
10 import urllib2
11
12 from util import load_server_config
13
14
15 def main():
16     parser = argparse.ArgumentParser()
17     parser.add_argument('--triggerable', required=True, help='The name of the triggerable to process. e.g. build-webkit')
18     parser.add_argument('--buildbot-url', required=True, help='URL for a buildbot builder; e.g. "https://build.webkit.org/"')
19     parser.add_argument('--builder-config-json', required=True, help='The path to a JSON file that specifies which test and platform will be posted to which builder. '
20         'The JSON should contain an array of dictionaries with keys "platform", "test", and "builder" '
21         'with the platform name (e.g. mountainlion), the test path (e.g. ["Parser", "html5-full-render"]), and the builder name (e.g. Apple MountainLion Release (Perf)) as values.')
22     parser.add_argument('--server-config-json', required=True, help='The path to a JSON file that specifies the perf dashboard.')
23
24     parser.add_argument('--lookback-count', type=int, default=10, help='The number of builds to look back when finding in-progress builds on the buildbot')
25     parser.add_argument('--seconds-to-sleep', type=float, default=120, help='The seconds to sleep between iterations')
26     args = parser.parse_args()
27
28     configurations = load_config(args.builder_config_json, args.buildbot_url.strip('/'))
29
30     request_updates = {}
31     while True:
32         server_config = load_server_config(args.server_config_json)
33         request_updates.update(find_request_updates(configurations, args.lookback_count))
34         if request_updates:
35             print 'Updating the build requests %s...' % ', '.join(map(str, request_updates.keys()))
36         else:
37             print 'No updates...'
38
39         payload = {
40             'buildRequestUpdates': request_updates,
41             'slaveName': server_config['slave']['name'],
42             'slavePassword': server_config['slave']['password']}
43
44         build_requests_url = server_config['server']['url'] + '/api/build-requests/' + args.triggerable
45         response = update_and_fetch_build_requests(build_requests_url, payload)
46         open_requests = response.get('buildRequests', [])
47
48         root_sets = organize_root_sets_by_id_and_repository_names(response.get('rootSets', {}), response.get('roots', []))
49
50         for request in filter(lambda request: request['status'] == 'pending', open_requests):
51             config = config_for_request(configurations, request)
52             if not config:
53                 print >> sys.stderr, "Failed to find the configuration for request %s: %s" % (str(request['id']), json.dumps(request))
54                 continue
55             if config and len(config['scheduledRequests']) < 1:
56                 print "Scheduling the build request %s..." % str(request['id'])
57                 schedule_request(config, request, root_sets)
58
59         request_updates = find_stale_request_updates(configurations, open_requests, request_updates.keys())
60         if request_updates:
61             print "Found stale build requests %s..." % ', '.join(map(str, request_updates.keys()))
62
63         time.sleep(args.seconds_to_sleep)
64
65
66 def load_config(config_json_path, buildbot_url):
67     with open(config_json_path) as config_json:
68         configurations = json.load(config_json)
69
70     for config in configurations:
71         escaped_builder_name = urllib.quote(config['builder'])
72         config['url'] = '%s/builders/%s/' % (buildbot_url, escaped_builder_name)
73         config['jsonURL'] = '%s/json/builders/%s/' % (buildbot_url, escaped_builder_name)
74         config['scheduledRequests'] = set()
75
76     return configurations
77
78
79 def find_request_updates(configurations, lookback_count):
80     request_updates = {}
81
82     for config in configurations:
83         try:
84             pending_builds = fetch_json(config['jsonURL'] + 'pendingBuilds')
85             scheduled_requests = filter(None, [request_id_from_build(config, build) for build in pending_builds])
86             for request_id in scheduled_requests:
87                 request_updates[request_id] = {'status': 'scheduled', 'url': config['url']}
88             config['scheduledRequests'] = set(scheduled_requests)
89         except (IOError, ValueError) as error:
90             print >> sys.stderr, "Failed to fetch pending builds for %s: %s" % (config['builder'], str(error))
91
92     for config in configurations:
93         for i in range(1, lookback_count + 1):
94             build_error = None
95             build_index = -i
96             try:
97                 build = fetch_json(config['jsonURL'] + 'builds/%d' % build_index)
98                 request_id = request_id_from_build(config, build)
99                 if not request_id:
100                     continue
101
102                 in_progress = build.get('currentStep')
103                 if in_progress:
104                     request_updates[request_id] = {'status': 'running', 'url': config['url']}
105                     config['scheduledRequests'].discard(request_id)
106                 else:
107                     url = config['url'] + 'builds/' + str(build['number'])
108                     request_updates[request_id] = {'status': 'failedIfNotCompleted', 'url': url}
109             except urllib2.HTTPError as error:
110                 if error.code == 404:
111                     break
112                 else:
113                     build_error = error
114             except ValueError as error:
115                 build_error = error
116             if build_error:
117                 print >> sys.stderr, "Failed to fetch build %d for %s: %s" % (build_index, config['builder'], str(build_error))
118
119     return request_updates
120
121
122 def update_and_fetch_build_requests(build_requests_url, payload):
123     try:
124         response = fetch_json(build_requests_url, payload=json.dumps(payload))
125         if response['status'] != 'OK':
126             raise ValueError(response['status'])
127         return response
128     except (IOError, ValueError) as error:
129         print >> sys.stderr, 'Failed to update or fetch build requests at %s: %s' % (build_requests_url, str(error))
130     return {}
131
132
133 def find_stale_request_updates(configurations, open_requests, requests_on_buildbot):
134     request_updates = {}
135     for request in open_requests:
136         request_id = int(request['id'])
137         should_be_on_buildbot = request['status'] in ('scheduled', 'running')
138         if should_be_on_buildbot and request_id not in requests_on_buildbot:
139             config = config_for_request(configurations, request)
140             if config:
141                 request_updates[request_id] = {'status': 'failed', 'url': config['url']}
142     return request_updates
143
144
145 def organize_root_sets_by_id_and_repository_names(root_sets, roots):
146     result = {}
147     root_by_id = {}
148     for root in roots:
149         root_by_id[root['id']] = root
150
151     for root_set in root_sets:
152         roots_by_repository = {}
153         for root_id in root_set['roots']:
154             root = root_by_id[root_id]
155             roots_by_repository[root['repository']] = root
156         result[root_set['id']] = roots_by_repository
157
158     return result
159
160
161 def schedule_request(config, request, root_sets):
162     roots = root_sets[request['rootSet']]
163     payload = {}
164     for property_name, property_value in config['arguments'].iteritems():
165         if not isinstance(property_value, dict):
166             payload[property_name] = property_value
167         elif 'root' in property_value:
168             repository_name = property_value['root']
169             if repository_name in roots:
170                 payload[property_name] = roots[repository_name]['revision']
171         elif 'rootsExcluding' in property_value:
172             excluded_roots = property_value['rootsExcluding']
173             filtered_roots = {}
174             for root_name in roots:
175                 if root_name not in excluded_roots:
176                     filtered_roots[root_name] = roots[root_name]
177             payload[property_name] = json.dumps(filtered_roots)
178         else:
179             print >> sys.stderr, "Failed to process an argument %s: %s" % (property_name, property_value)
180             return
181     payload[config['buildRequestArgument']] = request['id']
182
183     try:
184         urllib2.urlopen(urllib2.Request(config['url'] + 'force'), urllib.urlencode(payload))
185         config['scheduledRequests'].add(request['id'])
186     except (IOError, ValueError) as error:
187         print >> sys.stderr, "Failed to fetch pending builds for %s: %s" % (config['builder'], str(error))
188
189
190 def config_for_request(configurations, request):
191     for config in configurations:
192         if config['platform'] == request['platform'] and config['test'] == request['test']:
193             return config
194     return None
195
196
197 def fetch_json(url, payload=None):
198     request = urllib2.Request(url)
199     response = urllib2.urlopen(request, payload).read()
200     try:
201         return json.loads(response)
202     except ValueError as error:
203         raise ValueError(str(error) + '\n' + response)
204
205
206 def property_value_from_build(build, name):
207     for prop in build.get('properties', []):
208         if prop[0] == name:
209             return prop[1]
210     return None
211
212
213 def request_id_from_build(config, build):
214     job_id = property_value_from_build(build, config['buildRequestArgument'])
215     return int(job_id) if job_id and job_id.isdigit() else None
216
217
218 if __name__ == "__main__":
219     main()