Websites/perf.webkit.org/tools/sync-with-buildbot.py

   1 #!/usr/bin/python
   2
   3 import argparse
   4 import base64
   5 import copy
   6 import json
   7 import sys
   8 import time
   9 import urllib
  10 import urllib2
  11
  12 from util import setup_auth
  13
  14
  15 def main():
  16     parser = argparse.ArgumentParser()
  17     parser.add_argument('--triggerable', required=True, help='The name of the triggerable to process. e.g. build-webkit')
  18     parser.add_argument('--buildbot-url', required=True, help='URL for a buildbot builder; e.g. "https://build.webkit.org/"')
  19     parser.add_argument('--builder-config-json', required=True, help='The path to a JSON file that specifies which test and platform will be posted to which builder. '
  20         'The JSON should contain an array of dictionaries with keys "platform", "test", and "builder" '
  21         'with the platform name (e.g. mountainlion), the test path (e.g. ["Parser", "html5-full-render"]), and the builder name (e.g. Apple MountainLion Release (Perf)) as values.')
  22     parser.add_argument('--server-config-json', required=True, help='The path to a JSON file that specifies the perf dashboard.')
  23
  24     parser.add_argument('--lookback-count', type=int, default=10, help='The number of builds to look back when finding in-progress builds on the buildbot')
  25     parser.add_argument('--seconds-to-sleep', type=float, default=120, help='The seconds to sleep between iterations')
  26     args = parser.parse_args()
  27
  28     configurations = load_config(args.builder_config_json, args.buildbot_url.strip('/'))
  29
  30     with open(args.server_config_json) as server_config_json:
  31         server_config = json.load(server_config_json)
  32         setup_auth(server_config['server'])
  33
  34     build_requests_url = server_config['server']['url'] + '/api/build-requests/' + args.triggerable
  35
  36     request_updates = {}
  37     while True:
  38         request_updates.update(find_request_updates(configurations, args.lookback_count))
  39         if request_updates:
  40             print 'Updating the build requests %s...' % ', '.join(map(str, request_updates.keys()))
  41         else:
  42             print 'No updates...'
  43
  44         payload = {
  45             'buildRequestUpdates': request_updates,
  46             'slaveName': server_config['slave']['name'],
  47             'slavePassword': server_config['slave']['password']}
  48         response = update_and_fetch_build_requests(build_requests_url, payload)
  49         open_requests = response.get('buildRequests', [])
  50
  51         root_sets = organize_root_sets_by_id_and_repository_names(response.get('rootSets', {}), response.get('roots', []))
  52
  53         for request in filter(lambda request: request['status'] == 'pending', open_requests):
  54             config = config_for_request(configurations, request)
  55             if not config:
  56                 print >> sys.stderr, "Failed to find the configuration for request %s: %s" % (str(request['id']), json.dumps(request))
  57                 continue
  58             if config and len(config['scheduledRequests']) < 1:
  59                 print "Scheduling the build request %s..." % str(request['id'])
  60                 schedule_request(config, request, root_sets)
  61
  62         request_updates = find_stale_request_updates(configurations, open_requests, request_updates.keys())
  63         if request_updates:
  64             print "Found stale build requests %s..." % ', '.join(map(str, request_updates.keys()))
  65
  66         time.sleep(args.seconds_to_sleep)
  67
  68
  69 def load_config(config_json_path, buildbot_url):
  70     with open(config_json_path) as config_json:
  71         configurations = json.load(config_json)
  72
  73     for config in configurations:
  74         escaped_builder_name = urllib.quote(config['builder'])
  75         config['url'] = '%s/builders/%s/' % (buildbot_url, escaped_builder_name)
  76         config['jsonURL'] = '%s/json/builders/%s/' % (buildbot_url, escaped_builder_name)
  77         config['scheduledRequests'] = set()
  78
  79     return configurations
  80
  81
  82 def find_request_updates(configurations, lookback_count):
  83     request_updates = {}
  84
  85     for config in configurations:
  86         try:
  87             pending_builds = fetch_json(config['jsonURL'] + 'pendingBuilds')
  88             scheduled_requests = filter(None, [request_id_from_build(config, build) for build in pending_builds])
  89             for request_id in scheduled_requests:
  90                 request_updates[request_id] = {'status': 'scheduled', 'url': config['url']}
  91             config['scheduledRequests'] = set(scheduled_requests)
  92         except (IOError, ValueError) as error:
  93             print >> sys.stderr, "Failed to fetch pending builds for %s: %s" % (config['builder'], str(error))
  94
  95     for config in configurations:
  96         for i in range(1, lookback_count + 1):
  97             build_error = None
  98             build_index = -i
  99             try:
 100                 build = fetch_json(config['jsonURL'] + 'builds/%d' % build_index)
 101                 request_id = request_id_from_build(config, build)
 102                 if not request_id:
 103                     continue
 104
 105                 in_progress = build.get('currentStep')
 106                 if in_progress:
 107                     request_updates[request_id] = {'status': 'running', 'url': config['url']}
 108                     config['scheduledRequests'].discard(request_id)
 109                 else:
 110                     url = config['url'] + 'builds/' + str(build['number'])
 111                     request_updates[request_id] = {'status': 'failedIfNotCompleted', 'url': url}
 112             except urllib2.HTTPError as error:
 113                 if error.code == 404:
 114                     break
 115                 else:
 116                     build_error = error
 117             except ValueError as error:
 118                 build_error = error
 119             if build_error:
 120                 print >> sys.stderr, "Failed to fetch build %d for %s: %s" % (build_index, config['builder'], str(build_error))
 121
 122     return request_updates
 123
 124
 125 def update_and_fetch_build_requests(build_requests_url, payload):
 126     try:
 127         response = fetch_json(build_requests_url, payload=json.dumps(payload))
 128         if response['status'] != 'OK':
 129             raise ValueError(response['status'])
 130         return response
 131     except (IOError, ValueError) as error:
 132         print >> sys.stderr, 'Failed to update or fetch build requests at %s: %s' % (build_requests_url, str(error))
 133     return {}
 134
 135
 136 def find_stale_request_updates(configurations, open_requests, requests_on_buildbot):
 137     request_updates = {}
 138     for request in open_requests:
 139         request_id = int(request['id'])
 140         should_be_on_buildbot = request['status'] in ('scheduled', 'running')
 141         if should_be_on_buildbot and request_id not in requests_on_buildbot:
 142             config = config_for_request(configurations, request)
 143             if config:
 144                 request_updates[request_id] = {'status': 'failed', 'url': config['url']}
 145     return request_updates
 146
 147
 148 def organize_root_sets_by_id_and_repository_names(root_sets, roots):
 149     result = {}
 150     root_by_id = {}
 151     for root in roots:
 152         root_by_id[root['id']] = root
 153
 154     for root_set in root_sets:
 155         roots_by_repository = {}
 156         for root_id in root_set['roots']:
 157             root = root_by_id[root_id]
 158             roots_by_repository[root['repository']] = root
 159         result[root_set['id']] = roots_by_repository
 160
 161     return result
 162
 163
 164 def schedule_request(config, request, root_sets):
 165     roots = root_sets[request['rootSet']]
 166     payload = {}
 167     for property_name, property_value in config['arguments'].iteritems():
 168         if not isinstance(property_value, dict):
 169             payload[property_name] = property_value
 170         elif 'root' in property_value:
 171             repository_name = property_value['root']
 172             if repository_name in roots:
 173                 payload[property_name] = roots[repository_name]['revision']
 174         elif 'rootsExcluding' in property_value:
 175             excluded_roots = property_value['rootsExcluding']
 176             filtered_roots = {}
 177             for root_name in roots:
 178                 if root_name not in excluded_roots:
 179                     filtered_roots[root_name] = roots[root_name]
 180             payload[property_name] = json.dumps(filtered_roots)
 181         else:
 182             print >> sys.stderr, "Failed to process an argument %s: %s" % (property_name, property_value)
 183             return
 184     payload[config['buildRequestArgument']] = request['id']
 185
 186     try:
 187         urllib2.urlopen(urllib2.Request(config['url'] + 'force'), urllib.urlencode(payload))
 188         config['scheduledRequests'].add(request['id'])
 189     except (IOError, ValueError) as error:
 190         print >> sys.stderr, "Failed to fetch pending builds for %s: %s" % (config['builder'], str(error))
 191
 192
 193 def config_for_request(configurations, request):
 194     for config in configurations:
 195         if config['platform'] == request['platform'] and config['test'] == request['test']:
 196             return config
 197     return None
 198
 199
 200 def fetch_json(url, payload=None):
 201     request = urllib2.Request(url)
 202     response = urllib2.urlopen(request, payload).read()
 203     try:
 204         return json.loads(response)
 205     except ValueError as error:
 206         raise ValueError(str(error) + '\n' + response)
 207
 208
 209 def property_value_from_build(build, name):
 210     for prop in build.get('properties', []):
 211         if prop[0] == name:
 212             return prop[1]
 213     return None
 214
 215
 216 def request_id_from_build(config, build):
 217     job_id = property_value_from_build(build, config['buildRequestArgument'])
 218     return int(job_id) if job_id and job_id.isdigit() else None
 219
 220
 221 if __name__ == "__main__":
 222     main()