Prune unused uploaded files when the file quota is reached
[WebKit-https.git] / Websites / perf.webkit.org / public / include / uploaded-file-helpers.php
1 <?php
2
3 define('MEGABYTES', 1024 * 1024);
4
5 function format_uploaded_file($file_row)
6 {
7     return array(
8         'id' => $file_row['file_id'],
9         'size' => $file_row['file_size'],
10         'createdAt' => Database::to_js_time($file_row['file_created_at']),
11         'mime' => $file_row['file_mime'],
12         'filename' => $file_row['file_filename'],
13         'extension' => $file_row['file_extension'],
14         'author' => $file_row['file_author'],
15         'sha256' => $file_row['file_sha256']);
16 }
17
18 function uploaded_file_path_for_row($file_row)
19 {
20     return config_path('uploadDirectory', $file_row['file_id'] . $file_row['file_extension']);
21 }
22
23 function validate_uploaded_file($field_name)
24 {
25     if (array_get($_SERVER, 'CONTENT_LENGTH') && empty($_POST) && empty($_FILES))
26         exit_with_error('FileSizeLimitExceeded');
27
28     if (!is_dir(config_path('uploadDirectory', '')))
29         exit_with_error('NotSupported');
30
31     $input_file = array_get($_FILES, $field_name);
32     if (!$input_file)
33         exit_with_error('NoFileSpecified');
34
35     if ($input_file['error'] == UPLOAD_ERR_INI_SIZE || $input_file['error'] == UPLOAD_ERR_FORM_SIZE)
36         exit_with_error('FileSizeLimitExceeded');
37
38     if ($input_file['error'] != UPLOAD_ERR_OK)
39         exit_with_error('FailedToUploadFile', array('name' => $input_file['name'], 'error' => $input_file['error']));
40
41     if (config('uploadFileLimitInMB') * MEGABYTES < $input_file['size'])
42         exit_with_error('FileSizeLimitExceeded');
43
44     return $input_file;
45 }
46
47 function query_file_usage_for_user($db, $user)
48 {
49     if ($user)
50         $count_result = $db->query_and_fetch_all('SELECT sum(file_size) as "sum" FROM uploaded_files WHERE file_deleted_at IS NULL AND file_author = $1', array($user));
51     else
52         $count_result = $db->query_and_fetch_all('SELECT sum(file_size) as "sum" FROM uploaded_files WHERE file_deleted_at IS NULL AND file_author IS NULL');
53     if (!$count_result)
54         exit_with_error('FailedToQueryDiskUsagePerUser');
55     return intval($count_result[0]["sum"]);
56 }
57
58 function query_total_file_usage($db)
59 {
60     $count_result = $db->query_and_fetch_all('SELECT sum(file_size) as "sum" FROM uploaded_files WHERE file_deleted_at IS NULL');
61     if (!$count_result)
62         exit_with_error('FailedToQueryTotalDiskUsage');
63     return intval($count_result[0]["sum"]);
64 }
65
66 function create_uploaded_file_from_form_data($input_file)
67 {
68     $file_sha256 = hash_file('sha256', $input_file['tmp_name']);
69     if (!$file_sha256)
70         exit_with_error('FailedToComputeSHA256');
71
72     $matches = array();
73     $file_extension = null;
74     if (preg_match('/(\.[a-zA-Z0-9]{1,5}){1,2}$/', $input_file['name'], $matches)) {
75         $file_extension = $matches[0];
76         assert(strlen($file_extension) <= 16);
77     }
78
79     return array(
80         'author' => remote_user_name(),
81         'filename' => $input_file['name'],
82         'extension' => $file_extension,
83         'mime' => $input_file['type'], // Sanitize MIME types.
84         'size' => $input_file['size'],
85         'sha256' => $file_sha256
86     );
87 }
88
89 function upload_file_in_transaction($db, $input_file, $remote_user, $additional_work = NULL)
90 {
91     $new_file_size = $input_file['size'];
92     if (config('uploadUserQuotaInMB') * MEGABYTES - query_file_usage_for_user($db, $remote_user) < $new_file_size
93         || config('uploadTotalQuotaInMB') * MEGABYTES - query_total_file_usage($db) < $new_file_size) {
94         // Instead of <quota> - <used> - <new file size>, just ask for <new file size>
95         // since finding files to delete is an expensive operation.
96         if (!prune_old_files($db, $new_file_size, $remote_user))
97             exit_with_error('FileSizeQuotaExceeded');
98     }
99
100     $uploaded_file = create_uploaded_file_from_form_data($input_file);
101
102     $db->begin_transaction();
103     $file_row = $db->select_or_insert_row('uploaded_files', 'file',
104         array('sha256' => $uploaded_file['sha256'], 'deleted_at' => null), $uploaded_file, '*');
105     if (!$file_row)
106         exit_with_error('FailedToInsertFileData');
107
108     // A concurrent session may have inserted another file.
109     if (config('uploadUserQuotaInMB') * MEGABYTES < query_file_usage_for_user($db, $remote_user)
110         || config('uploadTotalQuotaInMB') * MEGABYTES < query_total_file_usage($db)) {
111         $db->rollback_transaction();
112         exit_with_error('FileSizeQuotaExceeded');
113     }
114
115     if ($additional_work) {
116         $error = $additional_work($db, $file_row);
117         if ($error) {
118             $db->rollback_transaction();
119             exit_with_error($error['status'], $error);
120         }
121     }
122
123     $new_path = uploaded_file_path_for_row($file_row);
124     if (!move_uploaded_file($input_file['tmp_name'], $new_path)) {
125         $db->rollback_transaction();
126         exit_with_error('FailedToMoveUploadedFile');
127     }
128
129     $db->commit_transaction();
130
131     return format_uploaded_file($file_row);
132 }
133
134 function delete_file($db, $file_row)
135 {
136     $db->begin_transaction();
137
138     if (!$db->query_and_get_affected_rows("UPDATE uploaded_files SET file_deleted_at = CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
139         WHERE file_id = $1", array($file_row['file_id']))) {
140         $db->rollback_transaction();
141         return FALSE;
142     }
143
144     $file_path = uploaded_file_path_for_row($file_row);
145     // The file may have been deleted by a concurrent session by the time we get here.
146     if (file_exists($file_path) && !unlink($file_path)) {
147         $db->rollback_transaction();
148         return FALSE;
149     }
150
151     $db->commit_transaction();
152     return TRUE;
153 }
154
155 function prune_old_files($db, $size_needed, $remote_user)
156 {
157     $user_filter = $remote_user ? 'AND file_author = $1' : 'AND file_author IS NULL';
158     $params = $remote_user ? array($remote_user) : array();
159
160     // 1. Delete old build products created for a patch not associated with any pending or in-progress builds.
161     $build_product_query = $db->query("SELECT file_id, file_extension, file_size FROM uploaded_files, commit_set_items
162         WHERE file_id = commitset_root_file AND commitset_patch_file IS NOT NULL AND file_deleted_at IS NULL
163             AND NOT EXISTS (SELECT request_id FROM build_requests WHERE request_commit_set = commitset_set AND request_status <= 'running')
164             $user_filter
165         ORDER BY file_created_at LIMIT 10", $params);
166     if (!$build_product_query)
167         return FALSE;
168     while ($row = $db->fetch_next_row($build_product_query)) {
169         if (!$row || !delete_file($db, $row))
170             return FALSE;
171         $size_needed -= $row['file_size'];
172         if ($size_needed <= 0)
173             return TRUE;
174     }
175
176     // 2. Delete any uploaded file not associated with any pending or in-progress builds.
177     $unused_file_query = $db->query("SELECT file_id, file_extension, file_size FROM uploaded_files
178         WHERE NOT EXISTS (SELECT request_id FROM build_requests, commit_set_items
179             WHERE (commitset_root_file = file_id OR commitset_patch_file = file_id)
180                 AND request_commit_set = commitset_set AND request_status <= 'running')
181             $user_filter
182         ORDER BY file_created_at LIMIT 10", $params);
183     if (!$unused_file_query)
184         return FALSE;
185     while ($row = $db->fetch_next_row($unused_file_query)) {
186         if (!$row || !delete_file($db, $row))
187             return FALSE;
188         $size_needed -= $row['file_size'];
189         if ($size_needed <= 0)
190             return TRUE;
191     }
192     return FALSE;
193 }
194
195 ?>