+2014-12-18 Ryosuke Niwa <rniwa@webkit.org>
+
+ New perf dashboard should not duplicate author information in each commit
+ https://bugs.webkit.org/show_bug.cgi?id=139756
+
+ Reviewed by Darin Adler.
+
+ Instead of each commit having author name and email, make it reference a newly added committers table.
+ Also replace "email" by "account" since some repositories don't use emails as account names.
+
+ This improves the keyword search performance in commits.php since LIKE now runs on committers table,
+ which only contains as many rows as there are accounts in each repository, instead of commits table
+ which contains every commit ever happened in each repository.
+
+ To migrate an existing database into match the new schema, run:
+
+ BEGIN;
+
+ INSERT INTO committers (committer_repository, committer_name, committer_email)
+ (SELECT DISTINCT commit_repository, commit_author_name, commit_author_email
+ FROM commits WHERE commit_author_email IS NOT NULL);
+
+ ALTER TABLE commits ADD COLUMN commit_committer integer REFERENCES committers ON DELETE CASCADE;
+
+ UPDATE commits SET commit_committer = committer_id FROM committers
+ WHERE commit_repository = committer_repository AND commit_author_email = committer_email;
+
+ ALTER TABLE commits DROP COLUMN commit_author_name CASCADE;
+ ALTER TABLE commits DROP COLUMN commit_author_email CASCADE;
+
+ COMMIT;
+
+ * init-database.sql: Added committers table, and replaced author columns in commits table by a foreign
+ reference to committers. Also added the missing drop table statements.
+
+ * public/api/commits.php:
+ (main): Fetch the corresponding committers row for a single commit. Also wrap a single commit by
+ an array here instead of doing it in format_commit.
+ (fetch_commits_between): Updated queries to JOIN commits with committers.
+ (format_commit): Takes both commit and committers rows. Also don't wrap the result in an array as that
+ is now done in main.
+
+ * public/api/report-commits.php:
+ (main): Store the reported committer information or update the existing entry if there is one.
+
+ * tests/admin-regenerate-manifest.js: Fixed tests.
+
+ * tests/api-report-commits.js: Ditto. Also added a test for updating an existing committer entry.
+
+ * tools/pull-svn.py: Renamed email to account.
+ (main):
+ (fetch_commit_and_resolve_author):
+ (fetch_commit):
+ (resolve_author_name_from_account):
+ (resolve_author_name_from_email): Deleted.
+
2014-12-17 Ryosuke Niwa <rniwa@webkit.org>
Unreviewed build fix.
DROP TYPE test_configuration_type CASCADE;
DROP TABLE aggregators CASCADE;
DROP TABLE builds CASCADE;
+DROP TABLE committers CASCADE;
DROP TABLE commits CASCADE;
DROP TABLE build_commits CASCADE;
DROP TABLE builders CASCADE;
DROP TABLE reports CASCADE;
DROP TABLE tracker_repositories CASCADE;
DROP TABLE bug_trackers CASCADE;
+DROP TABLE analysis_tasks CASCADE;
+DROP TABLE bugs CASCADE;
+DROP TABLE analysis_test_groups CASCADE;
+DROP TABLE root_sets CASCADE;
+DROP TABLE build_requests CASCADE;
+
CREATE TABLE platforms (
platform_id serial PRIMARY KEY,
CONSTRAINT builder_build_time_tuple_must_be_unique UNIQUE(build_builder, build_number, build_time));
CREATE INDEX build_builder_index ON builds(build_builder);
+CREATE TABLE committers (
+ committer_id serial PRIMARY KEY,
+ committer_repository integer NOT NULL REFERENCES repositories ON DELETE CASCADE,
+ committer_account varchar(320) NOT NULL,
+ committer_name varchar(128),
+ CONSTRAINT committer_in_repository_must_be_unique UNIQUE(committer_repository, committer_account));
+CREATE INDEX committer_account_index ON committers(committer_account);
+CREATE INDEX committer_name_index ON committers(committer_name);
+
CREATE TABLE commits (
commit_id serial PRIMARY KEY,
commit_repository integer NOT NULL REFERENCES repositories ON DELETE CASCADE,
commit_revision varchar(64) NOT NULL,
commit_parent integer REFERENCES commits ON DELETE CASCADE,
commit_time timestamp,
- commit_author_name varchar(128),
- commit_author_email varchar(320),
+ commit_committer integer REFERENCES committers ON DELETE CASCADE,
commit_message text,
commit_reported boolean NOT NULL DEFAULT FALSE,
CONSTRAINT commit_in_repository_must_be_unique UNIQUE(commit_repository, commit_revision));
CREATE INDEX commit_time_index ON commits(commit_time);
-CREATE INDEX commit_author_name_index ON commits(commit_author_name);
-CREATE INDEX commit_author_email_index ON commits(commit_author_email);
CREATE TABLE build_commits (
commit_build integer NOT NULL REFERENCES builds ON DELETE CASCADE,
$commits = fetch_commits_between($db, $repository_id, $matches[1], $matches[2]);
}
- exit_with_success(array('commits' => $single_commit ? format_commit($single_commit) : $commits));
+ if ($single_commit) {
+ $committer = $db->select_first_row('committers', 'committer', array('id' => $single_commit['commit_committer']));
+ exit_with_success(array('commits' => array(format_commit($single_commit, $committer))));
+ }
+
+ exit_with_success(array('commits' => $commits));
}
function commit_from_revision($db, $repository_id, $revision) {
commit_revision as "revision",
commit_parent as "parent",
commit_time as "time",
- commit_author_name as "authorName",
- commit_author_email as "authorEmail",
+ committer_name as "authorName",
+ committer_account as "authorEmail",
commit_message as "message"
- FROM commits WHERE commit_repository = $1 AND commit_reported = true';
+ FROM commits JOIN committers ON commit_committer = committer_id
+ WHERE commit_repository = $1 AND commit_reported = true';
$values = array($repository_id);
if ($first && $second) {
}
if ($keyword) {
- array_push($values, '%' . str_replace(array('\\', '_', '@'), array('\\\\', '\\_', '\\%'), $keyword) . '%');
- $index = '$' . count($values);
- $statements .= " AND (commit_author_name LIKE $index OR commit_author_email LIKE $index";
+ array_push($values, '%' . str_replace(array('\\', '_', '%'), array('\\\\', '\\_', '\\%'), $keyword) . '%');
+ $keyword_index = '$' . count($values);
array_push($values, ltrim($keyword, 'r'));
- $statements .= ' OR commit_revision = $' . count($values) . ')';
+ $revision_index = '$' . count($values);
+ $statements .= "
+ AND ((committer_name LIKE $keyword_index OR committer_account LIKE $keyword_index) OR commit_revision = $revision_index)";
}
$commits = $db->query_and_fetch_all($statements . ' ORDER BY commit_time', $values);
return $commits;
}
-function format_commit($commit_row) {
- return array(array(
+function format_commit($commit_row, $committer_row) {
+ return array(
'id' => $commit_row['commit_id'],
'revision' => $commit_row['commit_revision'],
'parent' => $commit_row['commit_parent'],
'time' => $commit_row['commit_time'],
- 'authorName' => $commit_row['commit_author_name'],
- 'authorEmail' => $commit_row['commit_author_email'],
+ 'authorName' => $committer_row ? $committer_row['committer_name'] : null,
+ 'authorEmail' => $committer_row ? $committer_row['committer_account'] : null,
'message' => $commit_row['commit_message']
- ));
+ );
}
main(array_key_exists('PATH_INFO', $_SERVER) ? explode('/', trim($_SERVER['PATH_INFO'], '/')) : array());
exit_with_error('FailedToInsertRepository', array('commit' => $commit_info));
}
+ $account = array_get($commit_info['author'], 'account');
+ $committer_query = array('repository' => $repository_id, 'account' => $account);
+ $committer_data = $committer_query;
+ $name = array_get($commit_info['author'], 'name');
+ if ($name)
+ $committer_data['name'] = $name;
+ $committer_id = $db->update_or_insert_row('committers', 'committer', $committer_query, $committer_data);
+ if (!$committer_id) {
+ $db->rollback_transaction();
+ exit_with_error('FailedToInsertCommitter', array('committer' => $committer_data));
+ }
+
$parent_revision = array_get($commit_info, 'parent');
$parent_id = NULL;
if ($parent_revision) {
'revision' => $commit_info['revision'],
'parent' => $parent_id,
'time' => $commit_info['time'],
- 'author_name' => array_get($commit_info['author'], 'name'),
- 'author_email' => array_get($commit_info['author'], 'email'),
+ 'committer' => $committer_id,
'message' => $commit_info['message'],
'reported' => true,
);
httpGet('/data/manifest', function (response) {
assert.equal(response.statusCode, 200);
var manifest = JSON.parse(response.responseText);
+ delete manifest.defaultDashboard;
assert.deepEqual(manifest, {
all: [],
bugTrackers: [],
httpGet('/data/manifest', function (response) {
assert.equal(response.statusCode, 200);
var manifest = JSON.parse(response.responseText);
- assert.deepEqual(manifest['bugTrackers'], { 'Bugzilla': { newBugUrl: 'bugs.webkit.org', repositories: null } });
+ assert.deepEqual(manifest['bugTrackers'],
+ {1: {name: 'Bugzilla', bugUrl: null, newBugUrl: 'bugs.webkit.org', repositories: null}});
notifyDone();
});
});
'WebKit': { url: 'trac.webkit.org', blameUrl: null, hasReportedCommits: false },
'Chromium': { url: null, blameUrl: 'SomeBlameURL', hasReportedCommits: false }
});
- assert.deepEqual(manifest['bugTrackers']['Bugzilla'], { newBugUrl: null, repositories: ['WebKit'] });
- assert.deepEqual(manifest['bugTrackers']['Issue Tracker'], { newBugUrl: null, repositories: ['WebKit', 'Chromium'] });
+ assert.deepEqual(manifest['bugTrackers'][3], {name: 'Bugzilla', bugUrl: null, newBugUrl: null, repositories: ['WebKit']});
+ assert.deepEqual(manifest['bugTrackers'][4], {name: 'Issue Tracker', bugUrl: null, newBugUrl: null, repositories: ['WebKit', 'Chromium']});
notifyDone();
});
});
"repository": "WebKit",
"revision": "141977",
"time": "2013-02-06T08:55:20.9Z",
- "author": {"name": "Commit Queue", "email": "commit-queue@webkit.org"},
+ "author": {"name": "Commit Queue", "account": "commit-queue@webkit.org"},
"message": "some message",
}
],
"repository": "WebKit",
"revision": "_141977",
"time": "2013-02-06T08:55:20.9Z",
- "author": {"name": "Commit Queue", "email": "commit-queue@webkit.org"},
+ "author": {"name": "Commit Queue", "account": "commit-queue@webkit.org"},
"message": "some message",
}
],
"repository": "WebKit",
"revision": "141977",
"time": "2013-02-06T08:55:20.9Z",
- "author": {"name": "Commit Queue", "email": "commit-queue@webkit.org"},
+ "author": {"name": "Commit Queue", "account": "commit-queue@webkit.org"},
"message": "some message",
},
{
"parent": "141977",
"revision": "141978",
"time": "2013-02-06T09:54:56.0Z",
- "author": {"name": "Mikhail Pozdnyakov", "email": "mikhail.pozdnyakov@intel.com"},
+ "author": {"name": "Mikhail Pozdnyakov", "account": "mikhail.pozdnyakov@intel.com"},
"message": "another message",
}
]
postJSON('/api/report-commits/', subversionCommit, function (response) {
assert.equal(response.statusCode, 200);
assert.equal(JSON.parse(response.responseText)['status'], 'OK');
- queryAndFetchAll('SELECT * FROM commits', [], function (rows) {
+ queryAndFetchAll('SELECT * FROM commits JOIN committers ON commit_committer = committer_id', [], function (rows) {
assert.equal(rows.length, 1);
var reportedData = subversionCommit.commits[0];
assert.equal(rows[0]['commit_revision'], reportedData['revision']);
assert.equal(rows[0]['commit_time'].toString(), new Date('2013-02-06 08:55:20.9').toString());
- assert.equal(rows[0]['commit_author_name'], reportedData['author']['name']);
- assert.equal(rows[0]['commit_author_email'], reportedData['author']['email']);
+ assert.equal(rows[0]['committer_name'], reportedData['author']['name']);
+ assert.equal(rows[0]['committer_account'], reportedData['author']['account']);
assert.equal(rows[0]['commit_message'], reportedData['message']);
notifyDone();
});
postJSON('/api/report-commits/', subversionTwoCommits, function (response) {
assert.equal(response.statusCode, 200);
assert.equal(JSON.parse(response.responseText)['status'], 'OK');
- queryAndFetchAll('SELECT * FROM commits ORDER BY commit_time', [], function (rows) {
+ queryAndFetchAll('SELECT * FROM commits JOIN committers ON commit_committer = committer_id ORDER BY commit_time', [], function (rows) {
assert.equal(rows.length, 2);
var reportedData = subversionTwoCommits.commits[0];
assert.equal(rows[0]['commit_revision'], reportedData['revision']);
assert.equal(rows[0]['commit_time'].toString(), new Date('2013-02-06 08:55:20.9').toString());
- assert.equal(rows[0]['commit_author_name'], reportedData['author']['name']);
- assert.equal(rows[0]['commit_author_email'], reportedData['author']['email']);
+ assert.equal(rows[0]['committer_name'], reportedData['author']['name']);
+ assert.equal(rows[0]['committer_account'], reportedData['author']['account']);
assert.equal(rows[0]['commit_message'], reportedData['message']);
var reportedData = subversionTwoCommits.commits[1];
assert.equal(rows[1]['commit_revision'], reportedData['revision']);
assert.equal(rows[1]['commit_time'].toString(), new Date('2013-02-06 09:54:56.0').toString());
- assert.equal(rows[1]['commit_author_name'], reportedData['author']['name']);
- assert.equal(rows[1]['commit_author_email'], reportedData['author']['email']);
+ assert.equal(rows[1]['committer_name'], reportedData['author']['name']);
+ assert.equal(rows[1]['committer_account'], reportedData['author']['account']);
assert.equal(rows[1]['commit_message'], reportedData['message']);
notifyDone();
});
queryAndFetchAll('INSERT INTO commits (commit_repository, commit_revision, commit_time) VALUES ($1, $2, $3) RETURNING *',
[repositoryId, reportedData['revision'], reportedData['time']], function (existingCommits) {
var commitId = existingCommits[0]['commit_id'];
- assert.equal(existingCommits[0]['commit_author_name'], null);
- assert.equal(existingCommits[0]['commit_author_email'], null);
assert.equal(existingCommits[0]['commit_message'], null);
addBuilder(subversionCommit, function () {
postJSON('/api/report-commits/', subversionCommit, function (response) {
assert.equal(response.statusCode, 200);
assert.equal(JSON.parse(response.responseText)['status'], 'OK');
- queryAndFetchAll('SELECT * FROM commits', [], function (rows) {
+ queryAndFetchAll('SELECT * FROM commits JOIN committers ON commit_committer = committer_id', [], function (rows) {
assert.equal(rows.length, 1);
var reportedData = subversionCommit.commits[0];
- assert.equal(rows[0]['commit_author_name'], reportedData['author']['name']);
- assert.equal(rows[0]['commit_author_email'], reportedData['author']['email']);
+ assert.equal(rows[0]['committer_name'], reportedData['author']['name']);
+ assert.equal(rows[0]['committer_account'], reportedData['author']['account']);
assert.equal(rows[0]['commit_message'], reportedData['message']);
notifyDone();
});
postJSON('/api/report-commits/', subversionCommit, function (response) {
assert.equal(response.statusCode, 200);
assert.equal(JSON.parse(response.responseText)['status'], 'OK');
- queryAndFetchAll('SELECT * FROM commits ORDER BY commit_time', [], function (rows) {
+ queryAndFetchAll('SELECT * FROM commits LEFT OUTER JOIN committers ON commit_committer = committer_id ORDER BY commit_time', [], function (rows) {
assert.equal(rows.length, 2);
- assert.equal(rows[0]['commit_author_name'], reportedData['author']['name']);
- assert.equal(rows[0]['commit_author_email'], reportedData['author']['email']);
+ assert.equal(rows[0]['committer_name'], reportedData['author']['name']);
+ assert.equal(rows[0]['committer_account'], reportedData['author']['account']);
assert.equal(rows[0]['commit_message'], reportedData['message']);
- assert.equal(rows[1]['commit_author_name'], null);
- assert.equal(rows[1]['commit_author_email'], null);
+ assert.equal(rows[1]['committer_name'], null);
+ assert.equal(rows[1]['committer_account'], null);
assert.equal(rows[1]['commit_message'], null);
notifyDone();
});
});
});
+ it("should update an existing committer if there is one", function () {
+ queryAndFetchAll('INSERT INTO repositories (repository_id, repository_name) VALUES (1, \'WebKit\')', [], function () {
+ var author = subversionCommit.commits[0]['author'];
+ queryAndFetchAll('INSERT INTO committers (committer_repository, committer_account) VALUES (1, $1)', [author['account']], function () {
+ addBuilder(subversionCommit, function () {
+ postJSON('/api/report-commits/', subversionCommit, function (response) {
+ assert.equal(response.statusCode, 200);
+ assert.equal(JSON.parse(response.responseText)['status'], 'OK');
+ queryAndFetchAll('SELECT * FROM committers', [], function (rows) {
+ assert.equal(rows.length, 1);
+ assert.equal(rows[0]['committer_name'], author['name']);
+ assert.equal(rows[0]['committer_account'], author['account']);
+ notifyDone();
+ });
+ });
+ });
+ });
+ });
+ });
+
});
def main(argv):
if len(argv) < 7:
- sys.exit('Usage: pull-svn <repository-name> <repository-URL> <dashboard-URL> <builder-name> <builder-password> <seconds-to-sleep> [<email-to-name-helper>]')
+ sys.exit('Usage: pull-svn <repository-name> <repository-URL> <dashboard-URL> <builder-name> <builder-password> <seconds-to-sleep> [<account-to-name-helper>]')
repository_name = argv[1]
repository_url = argv[2]
builder_name = argv[4]
builder_password = argv[5]
seconds_to_sleep = float(argv[6])
- email_to_name_helper = argv[7] if len(argv) > 7 else None
+ account_to_name_helper = argv[7] if len(argv) > 7 else None
print "Submitting revision logs for %s at %s to %s" % (repository_name, repository_url, dashboard_url)
pending_commits_to_send = []
while True:
- commit = fetch_commit_and_resolve_author(repository_name, repository_url, email_to_name_helper, revision_to_fetch)
+ commit = fetch_commit_and_resolve_author(repository_name, repository_url, account_to_name_helper, revision_to_fetch)
if commit:
print "Fetched r%d." % revision_to_fetch
return int(commits[0]['revision']) if commits else None
-def fetch_commit_and_resolve_author(repository_name, repository_url, email_to_name_helper, revision_to_fetch):
+def fetch_commit_and_resolve_author(repository_name, repository_url, account_to_name_helper, revision_to_fetch):
try:
commit = fetch_commit(repository_name, repository_url, revision_to_fetch)
except Exception as error:
if not commit:
return None
- email = commit['author']['email']
+ account = commit['author']['account']
try:
- name = resolve_author_name_from_email(email_to_name_helper, email) if email_to_name_helper else None
+ name = resolve_author_name_from_account(account_to_name_helper, account) if account_to_name_helper else None
if name:
commit['author']['name'] = name
except Exception as error:
- sys.exit('Failed to resolve the author name from an email %s: %s' % (email, str(error)))
+ sys.exit('Failed to resolve the author name from an account %s: %s' % (account, str(error)))
return commit
raise error
xml = parseXmlString(output)
time = textContent(xml.getElementsByTagName("date")[0])
- author_email = textContent(xml.getElementsByTagName("author")[0])
+ author_account = textContent(xml.getElementsByTagName("author")[0])
message = textContent(xml.getElementsByTagName("msg")[0])
return {
'repository': repository_name,
'revision': revision,
'time': time,
- 'author': {'email': author_email},
+ 'author': {'account': author_account},
'message': message,
}
return text
-name_email_compound_regex = re.compile(r'^\s*(?P<name>(\".+\"|[^<]+?))\s*\<(?P<email>.+)\>\s*$')
+name_account_compound_regex = re.compile(r'^\s*(?P<name>(\".+\"|[^<]+?))\s*\<(?P<account>.+)\>\s*$')
-def resolve_author_name_from_email(helper, email):
- output = subprocess.check_output(helper + ' ' + email, shell=True)
- match = name_email_compound_regex.match(output)
+def resolve_author_name_from_account(helper, account):
+ output = subprocess.check_output(helper + ' ' + account, shell=True)
+ match = name_account_compound_regex.match(output)
if match:
return match.group('name').strip('"')
return output.strip()
git://git.webkit.org / WebKit-https.git / commitdiff