Prune unused uploaded files when the file quota is reached
[WebKit-https.git] / Websites / perf.webkit.org / server-tests / privileged-api-upload-file-tests.js
1 'use strict';
2
3 require('../tools/js/v3-models.js');
4
5 const assert = require('assert');
6 global.FormData = require('form-data');
7
8 const MockData = require('./resources/mock-data.js');
9 const TestServer = require('./resources/test-server.js');
10 const TemporaryFile = require('./resources/temporary-file.js').TemporaryFile;
11 const prepareServerTest = require('./resources/common-operations.js').prepareServerTest;
12
13 describe('/privileged-api/upload-file', function () {
14     prepareServerTest(this);
15     TemporaryFile.inject();
16
17     it('should return "NotFileSpecified" when newFile not is specified', () => {
18         return PrivilegedAPI.sendRequest('upload-file', {}, {useFormData: true}).then(() => {
19             assert(false, 'should never be reached');
20         }, (error) => {
21             assert.equal(error, 'NoFileSpecified');
22         });
23     });
24
25     it('should return "FileSizeLimitExceeded" when the file is too big', () => {
26         return TemporaryFile.makeTemporaryFileOfSizeInMB('some.dat', TestServer.testConfig().uploadFileLimitInMB + 1).then((stream) => {
27             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true}).then(() => {
28                 assert(false, 'should never be reached');
29             }, (error) => {
30                 assert.equal(error, 'FileSizeLimitExceeded');
31             });
32         });
33     });
34
35     it('should upload a file when the filesize is smaller than the limit', () => {
36         const db = TestServer.database();
37         const limitInMB = TestServer.testConfig().uploadFileLimitInMB;
38         let uploadedFile;
39         return TemporaryFile.makeTemporaryFileOfSizeInMB('some.dat', limitInMB).then((stream) => {
40             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
41         }).then((response) => {
42             uploadedFile = response['uploadedFile'];
43             return db.selectAll('uploaded_files', 'id');
44         }).then((rows) => {
45             assert.equal(rows.length, 1);
46             assert.equal(rows[0].id, uploadedFile.id);
47             assert.equal(rows[0].size, limitInMB * 1024 * 1024);
48             assert.equal(rows[0].size, uploadedFile.size);
49             assert.equal(rows[0].filename, 'some.dat');
50             assert.equal(rows[0].filename, uploadedFile.filename);
51             assert.equal(rows[0].extension, '.dat');
52             assert.equal(rows[0].sha256, '5256ec18f11624025905d057d6befb03d77b243511ac5f77ed5e0221ce6d84b5');
53             assert.equal(rows[0].sha256, uploadedFile.sha256);
54         });
55     });
56
57     it('should not create a duplicate files when the identical files are uploaded', () => {
58         const db = TestServer.database();
59         const limitInMB = TestServer.testConfig().uploadFileLimitInMB;
60         let uploadedFile1;
61         let uploadedFile2;
62         return TemporaryFile.makeTemporaryFileOfSizeInMB('some.dat', limitInMB).then((stream) => {
63             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
64         }).then((response) => {
65             uploadedFile1 = response['uploadedFile'];
66             return TemporaryFile.makeTemporaryFileOfSizeInMB('other.dat', limitInMB);
67         }).then((stream) => {
68             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
69         }).then((response) => {
70             uploadedFile2 = response['uploadedFile'];
71             return db.selectAll('uploaded_files', 'id');
72         }).then((rows) => {
73             assert.deepEqual(uploadedFile1, uploadedFile2);
74             assert.equal(rows.length, 1);
75             assert.equal(rows[0].id, uploadedFile1.id);
76             assert.equal(rows[0].size, limitInMB * 1024 * 1024);
77             assert.equal(rows[0].size, uploadedFile1.size);
78             assert.equal(rows[0].filename, 'some.dat');
79             assert.equal(rows[0].filename, uploadedFile1.filename);
80             assert.equal(rows[0].extension, '.dat');
81             assert.equal(rows[0].sha256, '5256ec18f11624025905d057d6befb03d77b243511ac5f77ed5e0221ce6d84b5');
82             assert.equal(rows[0].sha256, uploadedFile1.sha256);
83         });
84     });
85
86     it('should not create a duplicate files when the identical files are uploaded', () => {
87         const db = TestServer.database();
88         const limitInMB = TestServer.testConfig().uploadFileLimitInMB;
89         let uploadedFile1;
90         let uploadedFile2;
91         return TemporaryFile.makeTemporaryFileOfSizeInMB('some.dat', limitInMB).then((stream) => {
92             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
93         }).then((response) => {
94             uploadedFile1 = response['uploadedFile'];
95             return TemporaryFile.makeTemporaryFileOfSizeInMB('other.dat', limitInMB);
96         }).then((stream) => {
97             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
98         }).then((response) => {
99             uploadedFile2 = response['uploadedFile'];
100             return db.selectAll('uploaded_files', 'id');
101         }).then((rows) => {
102             assert.deepEqual(uploadedFile1, uploadedFile2);
103             assert.equal(rows.length, 1);
104             assert.equal(rows[0].id, uploadedFile1.id);
105             assert.equal(rows[0].size, limitInMB * 1024 * 1024);
106             assert.equal(rows[0].size, uploadedFile1.size);
107             assert.equal(rows[0].filename, 'some.dat');
108             assert.equal(rows[0].filename, uploadedFile1.filename);
109             assert.equal(rows[0].extension, '.dat');
110             assert.equal(rows[0].sha256, '5256ec18f11624025905d057d6befb03d77b243511ac5f77ed5e0221ce6d84b5');
111             assert.equal(rows[0].sha256, uploadedFile1.sha256);
112         });
113     });
114
115     it('should re-upload the file when the previously uploaded file had been deleted', () => {
116         const db = TestServer.database();
117         const limitInMB = TestServer.testConfig().uploadFileLimitInMB;
118         let uploadedFile1;
119         let uploadedFile2;
120         return TemporaryFile.makeTemporaryFileOfSizeInMB('some.dat', limitInMB).then((stream) => {
121             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
122         }).then((response) => {
123             uploadedFile1 = response['uploadedFile'];
124             return db.query(`UPDATE uploaded_files SET file_deleted_at = now() at time zone 'utc'`);
125         }).then(() => {
126             return TemporaryFile.makeTemporaryFileOfSizeInMB('other.dat', limitInMB);
127         }).then((stream) => {
128             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
129         }).then((response) => {
130             uploadedFile2 = response['uploadedFile'];
131             return db.selectAll('uploaded_files', 'id');
132         }).then((rows) => {
133             assert.notEqual(uploadedFile1.id, uploadedFile2.id);
134             assert.equal(rows.length, 2);
135             assert.equal(rows[0].id, uploadedFile1.id);
136             assert.equal(rows[1].id, uploadedFile2.id);
137
138             assert.equal(rows[0].filename, 'some.dat');
139             assert.equal(rows[0].filename, uploadedFile1.filename);
140             assert.equal(rows[1].filename, 'other.dat');
141             assert.equal(rows[1].filename, uploadedFile2.filename);
142
143             assert.equal(rows[0].size, limitInMB * 1024 * 1024);
144             assert.equal(rows[0].size, uploadedFile1.size);
145             assert.equal(rows[0].size, uploadedFile2.size);
146             assert.equal(rows[0].size, rows[1].size);
147             assert.equal(rows[0].sha256, '5256ec18f11624025905d057d6befb03d77b243511ac5f77ed5e0221ce6d84b5');
148             assert.equal(rows[0].sha256, uploadedFile1.sha256);
149             assert.equal(rows[0].sha256, uploadedFile2.sha256);
150             assert.equal(rows[0].sha256, rows[1].sha256);
151             assert.equal(rows[0].extension, '.dat');
152             assert.equal(rows[1].extension, '.dat');
153         });
154     });
155
156     it('should pick up at most two file extensions', () => {
157         const db = TestServer.database();
158         const limitInMB = TestServer.testConfig().uploadFileLimitInMB;
159         return TemporaryFile.makeTemporaryFileOfSizeInMB('some.other.tar.gz', limitInMB).then((stream) => {
160             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
161         }).then(() => {
162             return db.selectAll('uploaded_files', 'id');
163         }).then((rows) => {
164             assert.equal(rows.length, 1);
165             assert.equal(rows[0].size, limitInMB * 1024 * 1024);
166             assert.equal(rows[0].mime, 'application/octet-stream');
167             assert.equal(rows[0].filename, 'some.other.tar.gz');
168             assert.equal(rows[0].extension, '.tar.gz');
169             assert.equal(rows[0].sha256, '5256ec18f11624025905d057d6befb03d77b243511ac5f77ed5e0221ce6d84b5');
170         });
171     });
172
173     it('should delete an old file when uploading the file would result in the quota being exceeded', () => {
174         const db = TestServer.database();
175         const limitInMB = TestServer.testConfig().uploadFileLimitInMB;
176         return TemporaryFile.makeTemporaryFileOfSizeInMB('some.dat', limitInMB, 'a').then((stream) => {
177             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
178         }).then(() => {
179             return TemporaryFile.makeTemporaryFileOfSizeInMB('other.dat', limitInMB, 'b');
180         }).then((stream) => {
181             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
182         }).then(() => {
183             return TemporaryFile.makeTemporaryFileOfSizeInMB('another.dat', limitInMB, 'c');
184         }).then((stream) => {
185             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
186         }).then(() => {
187             return db.selectAll('uploaded_files', 'id');
188         }).then((rows) => {
189             assert.equal(rows.length, 3);
190             assert.equal(rows[0].filename, 'some.dat');
191             assert.notEqual(rows[0].deleted_at, null);
192             assert.equal(rows[1].filename, 'other.dat');
193             assert.equal(rows[1].deleted_at, null);
194             assert.equal(rows[2].filename, 'another.dat');
195             assert.equal(rows[2].deleted_at, null);
196         })
197     });
198
199     it('should return "FileSizeQuotaExceeded" when there is no file to delete', () => {
200         const db = TestServer.database();
201         const limitInMB = TestServer.testConfig().uploadFileLimitInMB;
202         let fileA;
203         return MockData.addMockData(db).then(() => {
204             return TemporaryFile.makeTemporaryFileOfSizeInMB('some.patch', limitInMB, 'a');
205         }).then((stream) => {
206             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
207         }).then((result) => {
208             fileA = result.uploadedFile;
209             return TemporaryFile.makeTemporaryFileOfSizeInMB('other.patch', limitInMB, 'b');
210         }).then((stream) => {
211             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
212         }).then((result) => {
213             const fileB = result.uploadedFile;
214             return Promise.all([
215                 db.query('UPDATE commit_set_items SET commitset_patch_file = $1 WHERE commitset_set = 402 AND commitset_commit = 87832', [fileA.id]),
216                 db.query('UPDATE commit_set_items SET commitset_patch_file = $1 WHERE commitset_set = 402 AND commitset_commit = 96336', [fileB.id])
217             ]);
218         }).then(() => {
219             return TemporaryFile.makeTemporaryFileOfSizeInMB('other.dat', limitInMB, 'c');
220         }).then((stream) => {
221             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true}).then(() => {
222                 assert(false, 'should never be reached');
223             }, (error) => {
224                 assert.equal(error, 'FileSizeQuotaExceeded');
225             });
226         });
227     });
228
229     it('should delete old patches that belong to finished build requests', () => {
230         const db = TestServer.database();
231         const limitInMB = TestServer.testConfig().uploadFileLimitInMB;
232         let fileA;
233         return MockData.addMockData(db, ['completed', 'completed', 'failed', 'canceled']).then(() => {
234             return TemporaryFile.makeTemporaryFileOfSizeInMB('some.patch', limitInMB, 'a');
235         }).then((stream) => {
236             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
237         }).then((result) => {
238             fileA = result.uploadedFile;
239             return TemporaryFile.makeTemporaryFileOfSizeInMB('other.patch', limitInMB, 'b');
240         }).then((stream) => {
241             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
242         }).then((result) => {
243             const fileB = result.uploadedFile;
244             return Promise.all([
245                 db.query('UPDATE commit_set_items SET commitset_patch_file = $1 WHERE commitset_set = 402 AND commitset_commit = 87832', [fileA.id]),
246                 db.query('UPDATE commit_set_items SET commitset_patch_file = $1 WHERE commitset_set = 402 AND commitset_commit = 96336', [fileB.id])
247             ]);
248         }).then(() => {
249             return TemporaryFile.makeTemporaryFileOfSizeInMB('another.dat', limitInMB, 'c');
250         }).then((stream) => {
251             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
252         }).then(() => {
253             return db.selectAll('uploaded_files', 'id');
254         }).then((rows) => {
255             assert.equal(rows.length, 3);
256             assert.equal(rows[0].filename, 'some.patch');
257             assert.notEqual(rows[0].deleted_at, null);
258             assert.equal(rows[1].filename, 'other.patch');
259             assert.equal(rows[1].deleted_at, null);
260             assert.equal(rows[2].filename, 'another.dat');
261             assert.equal(rows[2].deleted_at, null);
262         });
263     });
264
265     it('should delete old build products that belong to finished build requests before deleting patches', () => {
266         const db = TestServer.database();
267         const limitInMB = TestServer.testConfig().uploadFileLimitInMB;
268         let fileA;
269         return MockData.addMockData(db, ['completed', 'completed', 'failed', 'canceled']).then(() => {
270             return TemporaryFile.makeTemporaryFileOfSizeInMB('some.patch', limitInMB, 'a');
271         }).then((stream) => {
272             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
273         }).then((result) => {
274             fileA = result.uploadedFile;
275             return TemporaryFile.makeTemporaryFileOfSizeInMB('root.tar.gz', limitInMB, 'b');
276         }).then((stream) => {
277             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
278         }).then((result) => {
279             const fileB = result.uploadedFile;
280             return db.query(`UPDATE commit_set_items SET (commitset_patch_file, commitset_root_file) = ($1, $2)
281                 WHERE commitset_set = 402 AND commitset_commit = 87832`, [fileA.id, fileB.id]);
282         }).then(() => {
283             return TemporaryFile.makeTemporaryFileOfSizeInMB('another.dat', limitInMB, 'c');
284         }).then((stream) => {
285             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
286         }).then(() => {
287             return db.selectAll('uploaded_files', 'id');
288         }).then((rows) => {
289             assert.equal(rows.length, 3);
290             assert.equal(rows[0].filename, 'some.patch');
291             assert.equal(rows[0].deleted_at, null);
292             assert.equal(rows[1].filename, 'root.tar.gz');
293             assert.notEqual(rows[1].deleted_at, null);
294             assert.equal(rows[2].filename, 'another.dat');
295             assert.equal(rows[2].deleted_at, null);
296         });
297     });
298
299     it('should return "FileSizeQuotaExceeded" when the total quota is exceeded due to files uploaded by other users', () => {
300         const db = TestServer.database();
301         const limitInMB = TestServer.testConfig().uploadFileLimitInMB;
302         let fileA;
303         return MockData.addMockData(db).then(() => {
304             return TemporaryFile.makeTemporaryFileOfSizeInMB('some.dat', limitInMB, 'a');
305         }).then((stream) => {
306             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
307         }).then(() => {
308             return TemporaryFile.makeTemporaryFileOfSizeInMB('other.dat', limitInMB, 'b');
309         }).then((stream) => {
310             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
311         }).then(() => {
312             return db.query('UPDATE uploaded_files SET file_author = $1', ['someUser']);
313         }).then(() => {
314             return TemporaryFile.makeTemporaryFileOfSizeInMB('another.dat', limitInMB, 'c');
315         }).then((stream) => {
316             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true});
317         }).then(() => {
318             return db.query('UPDATE uploaded_files SET file_author = $1 WHERE file_author IS NULL', ['anotherUser']);
319         }).then(() => {
320             return TemporaryFile.makeTemporaryFileOfSizeInMB('other.dat', limitInMB, 'c');
321         }).then((stream) => {
322             return PrivilegedAPI.sendRequest('upload-file', {newFile: stream}, {useFormData: true}).then(() => {
323                 assert(false, 'should never be reached');
324             }, (error) => {
325                 assert.equal(error, 'FileSizeQuotaExceeded');
326             });
327         });
328     });
329 });