Websites/perf.webkit.org/tools/sync-with-buildbot.py

   1 #!/usr/bin/python
   2
   3 import argparse
   4 import base64
   5 import copy
   6 import json
   7 import sys
   8 import time
   9 import urllib
  10 import urllib2
  11
  12 from util import load_server_config
  13
  14
  15 def main():
  16     parser = argparse.ArgumentParser()
  17     parser.add_argument('--triggerable', required=True, help='The name of the triggerable to process. e.g. build-webkit')
  18     parser.add_argument('--buildbot-url', required=True, help='URL for a buildbot builder; e.g. "https://build.webkit.org/"')
  19     parser.add_argument('--builder-config-json', required=True, help='The path to a JSON file that specifies which test and platform will be posted to which builder. '
  20         'The JSON should contain an array of dictionaries with keys "platform", "test", and "builder" '
  21         'with the platform name (e.g. mountainlion), the test path (e.g. ["Parser", "html5-full-render"]), and the builder name (e.g. Apple MountainLion Release (Perf)) as values.')
  22     parser.add_argument('--server-config-json', required=True, help='The path to a JSON file that specifies the perf dashboard.')
  23
  24     parser.add_argument('--lookback-count', type=int, default=10, help='The number of builds to look back when finding in-progress builds on the buildbot')
  25     parser.add_argument('--seconds-to-sleep', type=float, default=120, help='The seconds to sleep between iterations')
  26     args = parser.parse_args()
  27
  28     configurations = load_config(args.builder_config_json, args.buildbot_url.strip('/'))
  29
  30     request_updates = {}
  31     while True:
  32         server_config = load_server_config(args.server_config_json)
  33         request_updates.update(find_request_updates(configurations, args.lookback_count))
  34         if request_updates:
  35             print 'Updating the build requests %s...' % ', '.join(map(str, request_updates.keys()))
  36         else:
  37             print 'No updates...'
  38
  39         payload = {
  40             'buildRequestUpdates': request_updates,
  41             'slaveName': server_config['slave']['name'],
  42             'slavePassword': server_config['slave']['password']}
  43
  44         build_requests_url = server_config['server']['url'] + '/api/build-requests/' + args.triggerable
  45         response = update_and_fetch_build_requests(build_requests_url, payload)
  46         open_requests = response.get('buildRequests', [])
  47
  48         root_sets = organize_root_sets_by_id_and_repository_names(response.get('rootSets', {}), response.get('roots', []))
  49
  50         for request in filter(lambda request: request['status'] == 'pending', open_requests):
  51             config = config_for_request(configurations, request)
  52             if not config:
  53                 print >> sys.stderr, "Failed to find the configuration for request %s: %s" % (str(request['id']), json.dumps(request))
  54                 continue
  55             if config and len(config['scheduledRequests']) < 1:
  56                 print "Scheduling the build request %s..." % str(request['id'])
  57                 schedule_request(config, request, root_sets)
  58
  59         request_updates = find_stale_request_updates(configurations, open_requests, request_updates.keys())
  60         if request_updates:
  61             print "Found stale build requests %s..." % ', '.join(map(str, request_updates.keys()))
  62
  63         time.sleep(args.seconds_to_sleep)
  64
  65
  66 def load_config(config_json_path, buildbot_url):
  67     with open(config_json_path) as config_json:
  68         configurations = json.load(config_json)
  69
  70     for config in configurations:
  71         escaped_builder_name = urllib.quote(config['builder'])
  72         config['url'] = '%s/builders/%s/' % (buildbot_url, escaped_builder_name)
  73         config['jsonURL'] = '%s/json/builders/%s/' % (buildbot_url, escaped_builder_name)
  74         config['scheduledRequests'] = set()
  75
  76     return configurations
  77
  78
  79 def find_request_updates(configurations, lookback_count):
  80     request_updates = {}
  81
  82     for config in configurations:
  83         try:
  84             pending_builds = fetch_json(config['jsonURL'] + 'pendingBuilds')
  85             scheduled_requests = filter(None, [request_id_from_build(config, build) for build in pending_builds])
  86             for request_id in scheduled_requests:
  87                 request_updates[request_id] = {'status': 'scheduled', 'url': config['url']}
  88             config['scheduledRequests'] = set(scheduled_requests)
  89         except (IOError, ValueError) as error:
  90             print >> sys.stderr, "Failed to fetch pending builds for %s: %s" % (config['builder'], str(error))
  91
  92     for config in configurations:
  93         for i in range(1, lookback_count + 1):
  94             build_error = None
  95             build_index = -i
  96             try:
  97                 build = fetch_json(config['jsonURL'] + 'builds/%d' % build_index)
  98                 request_id = request_id_from_build(config, build)
  99                 if not request_id:
 100                     continue
 101
 102                 in_progress = build.get('currentStep')
 103                 if in_progress:
 104                     request_updates[request_id] = {'status': 'running', 'url': config['url']}
 105                     config['scheduledRequests'].discard(request_id)
 106                 else:
 107                     url = config['url'] + 'builds/' + str(build['number'])
 108                     request_updates[request_id] = {'status': 'failedIfNotCompleted', 'url': url}
 109             except urllib2.HTTPError as error:
 110                 if error.code == 404:
 111                     break
 112                 else:
 113                     build_error = error
 114             except ValueError as error:
 115                 build_error = error
 116             if build_error:
 117                 print >> sys.stderr, "Failed to fetch build %d for %s: %s" % (build_index, config['builder'], str(build_error))
 118
 119     return request_updates
 120
 121
 122 def update_and_fetch_build_requests(build_requests_url, payload):
 123     try:
 124         response = fetch_json(build_requests_url, payload=json.dumps(payload))
 125         if response['status'] != 'OK':
 126             raise ValueError(response['status'])
 127         return response
 128     except (IOError, ValueError) as error:
 129         print >> sys.stderr, 'Failed to update or fetch build requests at %s: %s' % (build_requests_url, str(error))
 130     return {}
 131
 132
 133 def find_stale_request_updates(configurations, open_requests, requests_on_buildbot):
 134     request_updates = {}
 135     for request in open_requests:
 136         request_id = int(request['id'])
 137         should_be_on_buildbot = request['status'] in ('scheduled', 'running')
 138         if should_be_on_buildbot and request_id not in requests_on_buildbot:
 139             config = config_for_request(configurations, request)
 140             if config:
 141                 request_updates[request_id] = {'status': 'failed', 'url': config['url']}
 142     return request_updates
 143
 144
 145 def organize_root_sets_by_id_and_repository_names(root_sets, roots):
 146     result = {}
 147     root_by_id = {}
 148     for root in roots:
 149         root_by_id[root['id']] = root
 150
 151     for root_set in root_sets:
 152         roots_by_repository = {}
 153         for root_id in root_set['roots']:
 154             root = root_by_id[root_id]
 155             roots_by_repository[root['repository']] = root
 156         result[root_set['id']] = roots_by_repository
 157
 158     return result
 159
 160
 161 def schedule_request(config, request, root_sets):
 162     roots = root_sets[request['rootSet']]
 163     payload = {}
 164     for property_name, property_value in config['arguments'].iteritems():
 165         if not isinstance(property_value, dict):
 166             payload[property_name] = property_value
 167         elif 'root' in property_value:
 168             repository_name = property_value['root']
 169             if repository_name in roots:
 170                 payload[property_name] = roots[repository_name]['revision']
 171         elif 'rootsExcluding' in property_value:
 172             excluded_roots = property_value['rootsExcluding']
 173             filtered_roots = {}
 174             for root_name in roots:
 175                 if root_name not in excluded_roots:
 176                     filtered_roots[root_name] = roots[root_name]
 177             payload[property_name] = json.dumps(filtered_roots)
 178         else:
 179             print >> sys.stderr, "Failed to process an argument %s: %s" % (property_name, property_value)
 180             return
 181     payload[config['buildRequestArgument']] = request['id']
 182
 183     try:
 184         urllib2.urlopen(urllib2.Request(config['url'] + 'force'), urllib.urlencode(payload))
 185         config['scheduledRequests'].add(request['id'])
 186     except (IOError, ValueError) as error:
 187         print >> sys.stderr, "Failed to fetch pending builds for %s: %s" % (config['builder'], str(error))
 188
 189
 190 def config_for_request(configurations, request):
 191     for config in configurations:
 192         if config['platform'] == request['platform'] and config['test'] == request['test']:
 193             return config
 194     return None
 195
 196
 197 def fetch_json(url, payload=None):
 198     request = urllib2.Request(url)
 199     response = urllib2.urlopen(request, payload).read()
 200     try:
 201         return json.loads(response)
 202     except ValueError as error:
 203         raise ValueError(str(error) + '\n' + response)
 204
 205
 206 def property_value_from_build(build, name):
 207     for prop in build.get('properties', []):
 208         if prop[0] == name:
 209             return prop[1]
 210     return None
 211
 212
 213 def request_id_from_build(config, build):
 214     job_id = property_value_from_build(build, config['buildRequestArgument'])
 215     return int(job_id) if job_id and job_id.isdigit() else None
 216
 217
 218 if __name__ == "__main__":
 219     main()