Websites/perf.webkit.org/tools/pull-os-versions.py

   1 #!/usr/bin/python
   2
   3 import argparse
   4 import json
   5 import operator
   6 import re
   7 import sys
   8 import subprocess
   9 import time
  10 import urllib
  11 import urllib2
  12
  13 from datetime import datetime
  14 from xml.dom.minidom import parseString as parseXmlString
  15 from util import submit_commits
  16 from util import text_content
  17 from util import load_server_config
  18
  19
  20 def main(argv):
  21     parser = argparse.ArgumentParser()
  22     parser.add_argument('--os-config-json', required=True, help='The path to a JSON that specifies how to fetch OS build information')
  23     parser.add_argument('--server-config-json', required=True, help='The path to a JSON file that specifies the perf dashboard')
  24     parser.add_argument('--seconds-to-sleep', type=float, default=43200, help='The seconds to sleep between iterations')
  25     args = parser.parse_args()
  26
  27     with open(args.os_config_json) as os_config_json:
  28         os_config_list = json.load(os_config_json)
  29
  30     fetchers = [OSBuildFetcher(os_config) for os_config in os_config_list]
  31
  32     while True:
  33         server_config = load_server_config(args.server_config_json)
  34         for fetcher in fetchers:
  35             fetcher.fetch_and_report_new_builds(server_config)
  36         print "Sleeping for %d seconds" % args.seconds_to_sleep
  37         time.sleep(args.seconds_to_sleep)
  38
  39
  40 # FIXME: Move other static functions into this class.
  41 class OSBuildFetcher:
  42     def __init__(self, os_config):
  43         self._os_config = os_config
  44         self._reported_revisions = set()
  45
  46     def _fetch_available_builds(self):
  47         config = self._os_config
  48         repository_name = self._os_config['name']
  49
  50         if 'customCommands' in config:
  51             available_builds = []
  52             for command in config['customCommands']:
  53                 print "Executing", ' '.join(command['command'])
  54                 available_builds += available_builds_from_command(repository_name, command['command'], command['linesToIgnore'])
  55         else:
  56             url = config['buildSourceURL']
  57             print "Fetching available builds from", url
  58             available_builds = fetch_available_builds(repository_name, url, config['trainVersionMap'])
  59         return available_builds
  60
  61     def fetch_and_report_new_builds(self, server_config):
  62         available_builds = self._fetch_available_builds()
  63         reported_revisions = self._reported_revisions
  64         print 'Found %d builds' % len(available_builds)
  65
  66         available_builds = filter(lambda commit: commit['revision'] not in reported_revisions, available_builds)
  67         self._assign_order(available_builds)
  68
  69         print "Submitting %d builds" % len(available_builds)
  70         submit_commits(available_builds, server_config['server']['url'], server_config['slave']['name'], server_config['slave']['password'])
  71         reported_revisions |= set(map(lambda commit: commit['revision'], available_builds))
  72
  73     @staticmethod
  74     def _assign_order(builds):
  75         build_name_regex = re.compile(r'(?P<major>\d+)(?P<kind>\w)(?P<minor>\d+)(?P<variant>\w*)')
  76         for commit in builds:
  77             match = build_name_regex.search(commit['revision'])
  78             major = int(match.group('major'))
  79             kind = ord(match.group('kind').upper()) - ord('A')
  80             minor = int(match.group('minor'))
  81             variant = ord(match.group('variant').upper()) - ord('A') + 1 if match.group('variant') else 0
  82             # These fake times won't conflict with real commit time since even 99Z9999z is still in Feb 1973
  83             fake_time = datetime.utcfromtimestamp((major * 100 + kind) * 10000 + minor + float(variant) / 100)
  84             commit['order'] = fake_time.isoformat()
  85
  86
  87 def available_builds_from_command(repository_name, command, lines_to_ignore):
  88     try:
  89         output = subprocess.check_output(command, stderr=subprocess.STDOUT)
  90     except subprocess.CalledProcessError as error:
  91         print "Failed:", str(error)
  92
  93     regex = re.compile(lines_to_ignore)
  94     return [{'repository': repository_name, 'revision': line} for line in output.split('\n') if not regex.match(line)]
  95
  96
  97 def fetch_available_builds(repository_name, url, train_version_map):
  98     output = urllib2.urlopen(url).read()
  99     try:
 100         xml = parseXmlString(output)
 101     except Exception, error:
 102         raise Exception(error, output)
 103     available_builds = []
 104     for image in xml.getElementsByTagName('baseASR'):
 105         id = text_content(image.getElementsByTagName('id')[0])
 106         train = text_content(image.getElementsByTagName('train')[0])
 107         build = text_content(image.getElementsByTagName('build')[0])
 108         if train not in train_version_map:
 109             continue
 110         available_builds.append({
 111             'repository': repository_name,
 112             'revision': train_version_map[train] + ' ' + build})
 113
 114     return available_builds
 115
 116
 117 if __name__ == "__main__":
 118     main(sys.argv)