Updating commit in OSBuildFetcher should respect revision range in config.
[WebKit-https.git] / Websites / perf.webkit.org / tools / js / os-build-fetcher.js
1 'use strict';
2
3 let assert = require('assert');
4
5 function mapInSerialPromiseChain(list, callback)
6 {
7     const results = [];
8     return list.reduce((chainedPromise, item) => {
9         return chainedPromise.then(() => callback(item)).then((result) => results.push(result));
10     }, Promise.resolve()).then(() => results);
11 }
12
13 class OSBuildFetcher {
14
15     constructor(osConfig, remoteAPI, slaveAuth, subprocess, logger)
16     {
17         this._osConfig = osConfig;
18         this._logger = logger;
19         this._slaveAuth = slaveAuth;
20         this._remoteAPI = remoteAPI;
21         this._subprocess = subprocess;
22         this._maxSubmitCount = osConfig['maxSubmitCount'] || 20;
23     }
24
25     static async fetchReportAndUpdateCommits(fetcherList)
26     {
27         for (const fetcher of fetcherList)
28             await fetcher.fetchReportAndUpdateBuilds();
29     }
30
31     async fetchReportAndUpdateBuilds()
32     {
33         const {newCommitsToReport, commitsToUpdate} = await this._fetchAvailableBuilds();
34
35         this._logger.log(`Submitting ${newCommitsToReport.length} builds for ${this._osConfig['name']}`);
36         await this._reportCommits(newCommitsToReport, true);
37
38         this._logger.log(`Updating ${commitsToUpdate.length} builds for ${this._osConfig['name']}`);
39         await this._reportCommits(commitsToUpdate, false);
40     }
41
42     async _fetchAvailableBuilds()
43     {
44         const config = this._osConfig;
45         const repositoryName = config['name'];
46         const newCommitsToReport = [];
47         const commitsToUpdate = [];
48         const  customCommands = config['customCommands'];
49
50         for (const command of customCommands) {
51             assert(command['minRevision']);
52             assert(command['maxRevision']);
53             const minRevisionOrder = this._computeOrder(command['minRevision']);
54             const maxRevisionOrder = this._computeOrder(command['maxRevision']);
55
56             const url = `/api/commits/${escape(repositoryName)}/last-reported?from=${minRevisionOrder}&to=${maxRevisionOrder}`;
57             const result = await this._remoteAPI.getJSONWithStatus(url);
58             const minOrder = result['commits'].length == 1 ? parseInt(result['commits'][0]['order']) + 1 : minRevisionOrder;
59
60             const commitInfo = await this._commitsForAvailableBuilds(command['command'], command['linesToIgnore']);
61             const commits = this._commitsWithinRange(commitInfo.allRevisions, repositoryName, minOrder, maxRevisionOrder);
62
63             const label = 'name' in command ? `"${command['name']}"` : `"${command['minRevision']}" to "${command['maxRevision']}"`;
64             this._logger.log(`Found ${commits.length} builds for ${label}`);
65             if ('ownedCommitCommand' in command) {
66                 this._logger.log(`Resolving ownedCommits for ${label}`);
67                 await this._addOwnedCommitsForBuild(commits, command['ownedCommitCommand']);
68             }
69             newCommitsToReport.push(...commits);
70
71             for (const [revision, testability] of Object.entries(commitInfo.commitsWithTestability)) {
72                 const order = this._computeOrder(revision);
73                 if (order > maxRevisionOrder || order < minRevisionOrder)
74                     continue;
75                 commitsToUpdate.push({repository: repositoryName, revision, testability});
76             }
77         }
78         return {newCommitsToReport, commitsToUpdate};
79     }
80
81     _computeOrder(revision)
82     {
83         const buildNameRegex = /(\d+)([a-zA-Z])(\d+)([a-zA-Z]*)$/;
84         const match = buildNameRegex.exec(revision);
85         assert(match);
86         const major = parseInt(match[1]);
87         const kind = match[2].toUpperCase().charCodeAt(0) - "A".charCodeAt(0);
88         const minor = parseInt(match[3]);
89         const variant = match[4] ? match[4].toUpperCase().charCodeAt(0) - "A".charCodeAt(0) + 1 : 0;
90         return ((major * 100 + kind) * 10000 + minor) * 100 + variant;
91     }
92
93     async _commitsForAvailableBuilds(command, linesToIgnore)
94     {
95         const output = await this._subprocess.execute(command);
96         try {
97             return JSON.parse(output);
98         } catch (error) {
99             if (!(error instanceof SyntaxError))
100                 throw error;
101             let lines = output.split('\n');
102             if (linesToIgnore) {
103                 const regex = new RegExp(linesToIgnore);
104                 lines = lines.filter((line) => !regex.exec(line));
105             }
106             return {allRevisions: lines, commitsWithTestability: {}};
107         }
108     }
109
110     _commitsWithinRange(revisions, repository, minOrder, maxOrder)
111     {
112         return revisions.map((revision) => ({repository, revision, 'order': this._computeOrder(revision)}))
113             .filter((commit) => commit['order'] >= minOrder && commit['order'] <= maxOrder);
114     }
115
116     _addOwnedCommitsForBuild(commits, command)
117     {
118         return mapInSerialPromiseChain(commits, (commit) => {
119             return this._subprocess.execute(command.concat(commit['revision'])).then((ownedCommitOutput) => {
120                 const ownedCommits = JSON.parse(ownedCommitOutput);
121                 this._logger.log(`Got ${Object.keys((ownedCommits)).length} owned commits for "${commit['revision']}"`);
122                 for (let repositoryName in ownedCommits) {
123                     const ownedCommit = ownedCommits[repositoryName];
124                     assert.deepEqual(Object.keys(ownedCommit), ['revision']);
125                     assert(typeof(ownedCommit['revision']) == 'string');
126                 }
127                 commit['ownedCommits'] = ownedCommits;
128                 return commit;
129             });
130         });
131     }
132
133     async _reportCommits(commitsToPost, insert)
134     {
135         if (!commitsToPost.length)
136             return;
137
138         for(let commitsToSubmit = commitsToPost.splice(0, this._maxSubmitCount); commitsToSubmit.length; commitsToSubmit = commitsToPost.splice(0, this._maxSubmitCount)) {
139             const report = {"slaveName": this._slaveAuth['name'], "slavePassword": this._slaveAuth['password'], 'commits': commitsToSubmit, insert};
140             const response = await this._remoteAPI.postJSONWithStatus('/api/report-commits/', report);
141             assert(response['status'] === 'OK');
142         }
143     }
144 }
145
146 if (typeof module != 'undefined')
147     module.exports.OSBuildFetcher = OSBuildFetcher;