Websites/perf.webkit.org/tools/pull-svn.py

   1 #!/usr/bin/python
   2
   3 import json
   4 import re
   5 import subprocess
   6 import sys
   7 import time
   8 import urllib2
   9
  10 from xml.dom.minidom import parseString as parseXmlString
  11
  12
  13 def main(argv):
  14     if len(argv) < 7:
  15         sys.exit('Usage: pull-svn <repository-name> <repository-URL> <dashboard-URL> <slave-name> <slave-password> <seconds-to-sleep> [<account-to-name-helper>]')
  16
  17     repository_name = argv[1]
  18     repository_url = argv[2]
  19     dashboard_url = argv[3]
  20     slave_name = argv[4]
  21     slave_password = argv[5]
  22     seconds_to_sleep = float(argv[6])
  23     account_to_name_helper = argv[7] if len(argv) > 7 else None
  24
  25     print "Submitting revision logs for %s at %s to %s" % (repository_name, repository_url, dashboard_url)
  26
  27     revision_to_fetch = determine_first_revision_to_fetch(dashboard_url, repository_name)
  28     print "Start fetching commits at r%d" % revision_to_fetch
  29
  30     pending_commits_to_send = []
  31
  32     while True:
  33         commit = fetch_commit_and_resolve_author(repository_name, repository_url, account_to_name_helper, revision_to_fetch)
  34
  35         if commit:
  36             print "Fetched r%d." % revision_to_fetch
  37             pending_commits_to_send += [commit]
  38             revision_to_fetch += 1
  39         else:
  40             print "Revision %d not found" % revision_to_fetch
  41
  42         if not commit or len(pending_commits_to_send) >= 10:
  43             if pending_commits_to_send:
  44                 print "Submitting the above commits to %s..." % dashboard_url
  45                 submit_commits(pending_commits_to_send, dashboard_url, slave_name, slave_password)
  46                 print "Successfully submitted."
  47             pending_commits_to_send = []
  48             time.sleep(seconds_to_sleep)
  49
  50
  51 def determine_first_revision_to_fetch(dashboard_url, repository_name):
  52     try:
  53         last_reported_revision = fetch_revision_from_dasbhoard(dashboard_url, repository_name, 'last-reported')
  54     except Exception as error:
  55         sys.exit('Failed to fetch the latest reported commit: ' + str(error))
  56
  57     if last_reported_revision:
  58         return last_reported_revision + 1
  59
  60     # FIXME: This is a problematic if dashboard can get results for revisions older than oldest_revision
  61     # in the future because we never refetch older revisions.
  62     try:
  63         return fetch_revision_from_dasbhoard(dashboard_url, repository_name, 'oldest') or 1
  64     except Exception as error:
  65         sys.exit('Failed to fetch the oldest commit: ' + str(error))
  66
  67
  68 def fetch_revision_from_dasbhoard(dashboard_url, repository_name, filter):
  69     result = urllib2.urlopen(dashboard_url + '/api/commits/' + repository_name + '/' + filter).read()
  70     parsed_result = json.loads(result)
  71     if parsed_result['status'] != 'OK' and parsed_result['status'] != 'RepositoryNotFound':
  72         raise Exception(result)
  73     commits = parsed_result.get('commits')
  74     return int(commits[0]['revision']) if commits else None
  75
  76
  77 def fetch_commit_and_resolve_author(repository_name, repository_url, account_to_name_helper, revision_to_fetch):
  78     try:
  79         commit = fetch_commit(repository_name, repository_url, revision_to_fetch)
  80     except Exception as error:
  81         sys.exit('Failed to fetch the commit %d: %s' % (revision_to_fetch, str(error)))
  82
  83     if not commit:
  84         return None
  85
  86     account = commit['author']['account']
  87     try:
  88         name = resolve_author_name_from_account(account_to_name_helper, account) if account_to_name_helper else None
  89         if name:
  90             commit['author']['name'] = name
  91     except Exception as error:
  92         sys.exit('Failed to resolve the author name from an account %s: %s' % (account, str(error)))
  93
  94     return commit
  95
  96
  97 def fetch_commit(repository_name, repository_url, revision):
  98     args = ['svn', 'log', '--revision', str(revision), '--xml', repository_url]
  99     try:
 100         output = subprocess.check_output(args, stderr=subprocess.STDOUT)
 101     except subprocess.CalledProcessError as error:
 102         if (': No such revision ' + str(revision)) in error.output:
 103             return None
 104         raise error
 105     xml = parseXmlString(output)
 106     time = textContent(xml.getElementsByTagName("date")[0])
 107     author_account = textContent(xml.getElementsByTagName("author")[0])
 108     message = textContent(xml.getElementsByTagName("msg")[0])
 109     return {
 110         'repository': repository_name,
 111         'revision': revision,
 112         'time': time,
 113         'author': {'account': author_account},
 114         'message': message,
 115     }
 116
 117
 118 def textContent(element):
 119     text = ''
 120     for child in element.childNodes:
 121         if child.nodeType == child.TEXT_NODE:
 122             text += child.data
 123         else:
 124             text += textContent(child)
 125     return text
 126
 127
 128 name_account_compound_regex = re.compile(r'^\s*(?P<name>(\".+\"|[^<]+?))\s*\<(?P<account>.+)\>\s*$')
 129
 130
 131 def resolve_author_name_from_account(helper, account):
 132     output = subprocess.check_output(helper + ' ' + account, shell=True)
 133     match = name_account_compound_regex.match(output)
 134     if match:
 135         return match.group('name').strip('"')
 136     return output.strip()
 137
 138
 139 def submit_commits(commits, dashboard_url, slave_name, slave_password):
 140     try:
 141         payload = json.dumps({
 142             'slaveName': slave_name,
 143             'slavePassword': slave_password,
 144             'commits': commits,
 145         })
 146         request = urllib2.Request(dashboard_url + '/api/report-commits')
 147         request.add_header('Content-Type', 'application/json')
 148         request.add_header('Content-Length', len(payload))
 149
 150         output = urllib2.urlopen(request, payload).read()
 151         try:
 152             result = json.loads(output)
 153         except Exception, error:
 154             raise Exception(error, output)
 155
 156         if result.get('status') != 'OK':
 157             raise Exception(result)
 158     except Exception as error:
 159         sys.exit('Failed to submit commits: %s' % str(error))
 160
 161
 162 if __name__ == "__main__":
 163     main(sys.argv)