OSBuildFetcher should respect maxRevision while finding OS builds to report.
[WebKit.git] / Websites / perf.webkit.org / tools / js / os-build-fetcher.js
index 2256e2e28b431e1edb4b3541123d88743d1894c3..051c29fcb7b86336bf4172d55f7a4986d1714fe1 100644 (file)
@@ -2,22 +2,47 @@
 
 let assert = require('assert');
 
+function mapInSerialPromiseChain(list, callback)
+{
+    const results = [];
+    return list.reduce((chainedPromise, item) => {
+        return chainedPromise.then(() => callback(item)).then((result) => results.push(result));
+    }, Promise.resolve()).then(() => results);
+}
+
 class OSBuildFetcher {
 
     constructor(osConfig, remoteAPI, slaveAuth, subprocess, logger)
     {
         this._osConfig = osConfig;
-        this._reportedRevisions = new Set();
         this._logger = logger;
         this._slaveAuth = slaveAuth;
         this._remoteAPI = remoteAPI;
         this._subprocess = subprocess;
+        this._maxSubmitCount = osConfig['maxSubmitCount'] || 20;
+    }
+
+    static fetchAndReportAllInOrder(fetcherList)
+    {
+        return mapInSerialPromiseChain(fetcherList, (fetcher) => fetcher.fetchAndReportNewBuilds());
     }
 
     fetchAndReportNewBuilds()
     {
-        return this._fetchAvailableBuilds().then((results) =>{
-            return this._submitCommits(results);
+        return this._fetchAvailableBuilds().then((results) => {
+            this._logger.log(`Submitting ${results.length} builds for ${this._osConfig['name']}`);
+            if (results.length == 0)
+                return this._submitCommits(results);
+
+            const splittedResults = [];
+            for (let startIndex = 0; startIndex < results.length; startIndex += this._maxSubmitCount)
+                splittedResults.push(results.slice(startIndex, startIndex + this._maxSubmitCount));
+
+            return mapInSerialPromiseChain(splittedResults, this._submitCommits.bind(this)).then((responses) => {
+                assert(responses.every((response) => response['status'] == 'OK'));
+                assert(responses.length > 0);
+                return responses[0];
+            });
         });
     }
 
@@ -27,24 +52,29 @@ class OSBuildFetcher {
         const repositoryName = config['name'];
         let customCommands = config['customCommands'];
 
-        return Promise.all(customCommands.map((command) => {
+        return mapInSerialPromiseChain(customCommands, (command) => {
             assert(command['minRevision']);
             assert(command['maxRevision']);
             const minRevisionOrder = this._computeOrder(command['minRevision']);
             const maxRevisionOrder = this._computeOrder(command['maxRevision']);
 
-            let fetchCommitsPromise = this._remoteAPI.getJSONWithStatus(`/api/commits/${escape(repositoryName)}/last-reported?from=${minRevisionOrder}&to=${maxRevisionOrder}`).then((result) => {
-                const minOrder = result['commits'].length == 1 ? parseInt(result['commits'][0]['order']) : 0;
-                return this._commitsForAvailableBuilds(repositoryName, command['command'], command['linesToIgnore'], minOrder);
-            })
+            const url = `/api/commits/${escape(repositoryName)}/last-reported?from=${minRevisionOrder}&to=${maxRevisionOrder}`;
 
-            if ('subCommitCommand' in command)
-                fetchCommitsPromise = fetchCommitsPromise.then((commits) => this._addSubCommitsForBuild(commits, command['subCommitCommand']));
+            return this._remoteAPI.getJSONWithStatus(url).then((result) => {
+                const minOrder = result['commits'].length == 1 ? parseInt(result['commits'][0]['order']) + 1 : minRevisionOrder;
+                return this._commitsForAvailableBuilds(repositoryName, command['command'], command['linesToIgnore'], minOrder, maxRevisionOrder);
+            }).then((commits) => {
+                const label = 'name' in command ? `"${command['name']}"` : `"${command['minRevision']}" to "${command['maxRevision']}"`;
+                this._logger.log(`Found ${commits.length} builds for ${label}`);
 
-            return fetchCommitsPromise;
-        })).then(results => {
-            return Array.prototype.concat.apply([], results);
-        });
+                if ('ownedCommitCommand' in command) {
+                    this._logger.log(`Resolving ownedCommits for ${label}`);
+                    return this._addOwnedCommitsForBuild(commits, command['ownedCommitCommand']);
+                }
+
+                return commits;
+            });
+        }).then((results) => [].concat(...results));
     }
 
     _computeOrder(revision)
@@ -59,33 +89,34 @@ class OSBuildFetcher {
         return ((major * 100 + kind) * 10000 + minor) * 100 + variant;
     }
 
-    _commitsForAvailableBuilds(repository, command, linesToIgnore, minOrder)
+    _commitsForAvailableBuilds(repository, command, linesToIgnore, minOrder, maxOrder)
     {
         return this._subprocess.execute(command).then((output) => {
             let lines = output.split('\n');
             if (linesToIgnore){
                 const regex = new RegExp(linesToIgnore);
-                lines = lines.filter(function(line) {return !regex.exec(line);});
+                lines = lines.filter((line) => !regex.exec(line));
             }
-            return lines.map(revision => ({repository, revision, 'order': this._computeOrder(revision)}))
-                .filter(commit => commit['order'] > minOrder);
+            return lines.map((revision) => ({repository, revision, 'order': this._computeOrder(revision)}))
+                .filter((commit) => commit['order'] >= minOrder && commit['order'] <= maxOrder);
         });
     }
 
-    _addSubCommitsForBuild(commits, command)
+    _addOwnedCommitsForBuild(commits, command)
     {
-        return commits.reduce((promise, commit) => {
-            return promise.then(() => {
-                return this._subprocess.execute(command.concat(commit['revision']));
-            }).then((subCommitOutput) => {
-                const subCommits = JSON.parse(subCommitOutput);
-                for (let repositoryName in subCommits) {
-                    const subCommit = subCommits[repositoryName];
-                    assert(subCommit['revision']);
+        return mapInSerialPromiseChain(commits, (commit) => {
+            return this._subprocess.execute(command.concat(commit['revision'])).then((ownedCommitOutput) => {
+                const ownedCommits = JSON.parse(ownedCommitOutput);
+                this._logger.log(`Got ${Object.keys((ownedCommits)).length} owned commits for "${commit['revision']}"`);
+                for (let repositoryName in ownedCommits) {
+                    const ownedCommit = ownedCommits[repositoryName];
+                    assert.deepEqual(Object.keys(ownedCommit), ['revision']);
+                    assert(typeof(ownedCommit['revision']) == 'string');
                 }
-                commit['subCommits'] = subCommits;
+                commit['ownedCommits'] = ownedCommits;
+                return commit;
             });
-        }, Promise.resolve()).then(() => commits);
+        });
     }
 
     _submitCommits(commits)
@@ -94,5 +125,6 @@ class OSBuildFetcher {
         return this._remoteAPI.postJSONWithStatus('/api/report-commits/', commitsToReport);
     }
 }
+
 if (typeof module != 'undefined')
     module.exports.OSBuildFetcher = OSBuildFetcher;