52e35a831b86d4861277fb7557795d48cdc632d5
[WebKit-https.git] / Websites / perf.webkit.org / public / include / report-processor.php
1 <?php
2
3 require_once('../include/json-header.php');
4
5 class ReportProcessor {
6     private $db;
7     private $name_to_aggregator;
8     private $report_id;
9     private $runs;
10
11     function __construct($db) {
12         $this->db = $db;
13         $this->name_to_aggregator = array();
14         $aggregator_table = $db->fetch_table('aggregators');
15         if ($aggregator_table) {
16             foreach ($aggregator_table as $aggregator_row) {
17                 $this->name_to_aggregator[$aggregator_row['aggregator_name']] = $aggregator_row;
18             }
19         }
20     }
21
22     private function exit_with_error($message, $details = NULL) {
23         if (!$this->report_id) {
24             $details['failureStored'] = FALSE;
25             exit_with_error($message, $details);
26         }
27
28         $details['failureStored'] = $this->db->query_and_get_affected_rows(
29             'UPDATE reports SET report_failure = $1, report_failure_details = $2 WHERE report_id = $3',
30             array($message, $details ? json_encode($details) : NULL, $this->report_id)) == 1;
31         exit_with_error($message, $details);
32     }
33
34     function process($report, $existing_report_id = NULL) {
35         $this->report_id = $existing_report_id;
36         $this->runs = NULL;
37
38         array_key_exists('builderName', $report) or $this->exit_with_error('MissingBuilderName');
39         array_key_exists('buildTime', $report) or $this->exit_with_error('MissingBuildTime');
40
41         $builder_info = array('name' => $report['builderName']);
42         $slave_name = array_get($report, 'slaveName', NULL);
43         $slave_id = NULL;
44         if (!$existing_report_id) {
45             $hash = NULL;
46             if ($slave_name && array_key_exists('slavePassword', $report)) {
47                 $hash = hash('sha256', $report['slavePassword']);
48                 $slave = $this->db->select_first_row('build_slaves', 'slave', array('name' => $slave_name, 'password_hash' => $hash));
49                 if ($slave)
50                     $slave_id = $slave['slave_id'];
51             } else if (array_key_exists('builderPassword', $report))
52                 $hash = hash('sha256', $report['builderPassword']);
53
54             if (!$hash)
55                 $this->exit_with_error('BuilderNotFound');
56             if (!$slave_id)
57                 $builder_info['password_hash'] = $hash;
58         }
59
60         if (array_key_exists('builderPassword', $report))
61             unset($report['builderPassword']);
62         if (array_key_exists('slavePassword', $report))
63             unset($report['slavePassword']);
64
65         $builder_id = NULL;
66         if ($slave_id)
67             $builder_id = $this->db->select_or_insert_row('builders', 'builder', $builder_info);
68         else {
69             $builder = $this->db->select_first_row('builders', 'builder', $builder_info);
70             if (!$builder)
71                 $this->exit_with_error('BuilderNotFound', array('name' => $builder_info['name']));
72             $builder_id = $builder['builder_id'];
73             if ($slave_name)
74                 $slave_id = $this->db->select_or_insert_row('build_slaves', 'slave', array('name' => $slave_name));
75         }
76
77         $build_data = $this->construct_build_data($report, $builder_id, $slave_id);
78         if (!$existing_report_id)
79             $this->store_report($report, $build_data);
80
81         $this->runs = new TestRunsGenerator($this->db, $this->name_to_aggregator, $this->report_id);
82         $this->recursively_ensure_tests($report['tests']);
83
84         $this->runs->aggregate();
85         $this->runs->compute_caches();
86
87         $platform_id = $this->db->select_or_insert_row('platforms', 'platform', array('name' => $report['platform']));
88         if (!$platform_id)
89             $this->exit_with_error('FailedToInsertPlatform', array('name' => $report['platform']));
90
91         // FIXME: Deprecate and unsupport "jobId".
92         $build_id = $this->resolve_build_id($build_data, array_get($report, 'revisions', array()),
93             array_get($report, 'jobId') or array_get($report, 'buildRequest'));
94
95         $this->runs->commit($platform_id, $build_id);
96     }
97
98     private function construct_build_data($report, $builder_id, $slave_id) {
99         array_key_exists('buildNumber', $report) or $this->exit_with_error('MissingBuildNumber');
100         array_key_exists('buildTime', $report) or $this->exit_with_error('MissingBuildTime');
101
102         return array('builder' => $builder_id, 'slave' => $slave_id, 'number' => $report['buildNumber'], 'time' => $report['buildTime']);
103     }
104
105     private function store_report($report, $build_data) {
106         assert(!$this->report_id);
107         $this->report_id = $this->db->insert_row('reports', 'report', array(
108             'builder' => $build_data['builder'],
109             'slave' => $build_data['slave'],
110             'build_number' => $build_data['number'],
111             'content' => json_encode($report)));
112         if (!$this->report_id)
113             $this->exit_with_error('FailedToStoreRunReport');
114     }
115
116     private function resolve_build_id($build_data, $revisions, $build_request_id) {
117         // FIXME: This code has a race condition. See <rdar://problem/15876303>.
118         $results = $this->db->query_and_fetch_all("SELECT build_id, build_slave FROM builds
119             WHERE build_builder = $1 AND build_number = $2 AND build_time <= $3 AND build_time + interval '1 day' > $3",
120             array($build_data['builder'], $build_data['number'], $build_data['time']));
121         if ($results) {
122             $first_result = $results[0];
123             if ($first_result['build_slave'] != $build_data['slave'])
124                 $this->exit_with_error('MismatchingBuildSlave', array('storedBuild' => $results, 'reportedBuildData' => $build_data));
125             $build_id = $first_result['build_id'];
126         } else
127             $build_id = $this->db->insert_row('builds', 'build', $build_data);
128         if (!$build_id)
129             $this->exit_with_error('FailedToInsertBuild', $build_data);
130
131         if ($build_request_id) {
132             if ($db->update_row('build_requests', 'request', array('id' => $build_request_id), array('status' => 'completed', 'build' => $build_id))
133                 != $build_request_id)
134                 $this->exit_with_error('FailedToUpdateBuildRequest', array('buildRequest' => $build_request_id, 'build' => $build_id));
135         }
136
137
138         foreach ($revisions as $repository_name => $revision_data) {
139             $repository_id = $this->db->select_or_insert_row('repositories', 'repository', array('name' => $repository_name));
140             if (!$repository_id)
141                 $this->exit_with_error('FailedToInsertRepository', array('name' => $repository_name));
142
143             $commit_data = array('repository' => $repository_id, 'revision' => $revision_data['revision'], 'time' => array_get($revision_data, 'timestamp'));
144
145             $mismatching_commit = $this->db->query_and_fetch_all('SELECT * FROM build_commits, commits
146                 WHERE build_commit = commit_id AND commit_build = $1 AND commit_repository = $2 AND commit_revision != $3 LIMIT 1',
147                 array($build_id, $repository_id, $revision_data['revision']));
148             if ($mismatching_commit)
149                 $this->exit_with_error('MismatchingCommitRevision', array('build' => $build_id, 'existing' => $mismatching_commit, 'new' => $commit_data));
150
151             $commit_row = $this->db->select_or_insert_row('commits', 'commit',
152                 array('repository' => $repository_id, 'revision' => $revision_data['revision']), $commit_data, '*');
153             if (!$commit_row)
154                 $this->exit_with_error('FailedToRecordCommit', $commit_data);
155             if (abs($commit_row['commit_time'] - $commit_data['time']) > 1.0)
156                 $this->exit_with_error('MismatchingCommitTime', array('existing' => $commit_row, 'new' => $commit_data));
157
158             if (!$this->db->select_or_insert_row('build_commits', null,
159                 array('commit_build' => $build_id, 'build_commit' => $commit_row['commit_id']), null, '*'))
160                 $this->exit_with_error('FailedToRelateCommitToBuild', array('commit' => $commit_row, 'build' => $build_id));
161         }
162
163         return $build_id;
164     }
165
166     private function recursively_ensure_tests($tests, $parent_id = NULL, $level = 0) {
167         foreach ($tests as $test_name => $test) {
168             $test_id = $this->db->select_or_insert_row('tests', 'test', $parent_id ? array('name' => $test_name, 'parent' => $parent_id) : array('name' => $test_name),
169                 array('name' => $test_name, 'parent' => $parent_id, 'url' => array_get($test, 'url')));
170             if (!$test_id)
171                 $this->exit_with_error('FailedToAddTest', array('name' => $test_name, 'parent' => $parent_id));
172
173             if (array_key_exists('tests', $test))
174                 $this->recursively_ensure_tests($test['tests'], $test_id, $level + 1);
175
176             foreach (array_get($test, 'metrics', array()) as $metric_name => $aggregators_or_config_types) {
177                 $aggregators = $this->aggregator_list_if_exists($aggregators_or_config_types);
178                 if ($aggregators) {
179                     foreach ($aggregators as $aggregator_name)
180                         $this->runs->add_aggregated_metric($parent_id, $test_id, $test_name, $metric_name, $aggregator_name, $level);
181                 } else {
182                     $metric_id = $this->db->select_or_insert_row('test_metrics', 'metric', array('name' => $metric_name, 'test' => $test_id));
183                     if (!$metric_id)
184                         $this->exit_with_error('FailedToAddMetric', array('name' => $metric_name, 'test' => $test_id));
185
186                     foreach ($aggregators_or_config_types as $config_type => $values) {
187                         // Some tests submit groups of iterations; e.g. [[1, 2, 3, 4], [5, 6, 7, 8]]
188                         // Convert other tests to this format to simplify the computation later.
189                         if (gettype($values) !== 'array')
190                             $values = array($values);
191                         if (gettype($values[0]) !== 'array')
192                             $values = array($values);
193                         $this->runs->add_values_to_commit($metric_id, $config_type, $values);
194                         $this->runs->add_values_for_aggregation($parent_id, $test_name, $metric_name, $config_type, $values);
195                     }
196                 }
197             }
198         }
199     }
200
201     private function aggregator_list_if_exists($aggregators_or_config_types) {
202         if (array_key_exists(0, $aggregators_or_config_types))
203             return $aggregators_or_config_types;
204         else if (array_get($aggregators_or_config_types, 'aggregators'))
205             return $aggregators_or_config_types['aggregators'];
206         return NULL;
207     }
208 };
209
210 class TestRunsGenerator {
211     private $db;
212     private $name_to_aggregator;
213     private $report_id;
214     private $metrics_to_aggregate;
215     private $parent_to_values;
216     private $values_to_commit;
217
218     function __construct($db, $name_to_aggregator, $report_id) {
219         $this->db = $db;
220         $this->name_to_aggregator = $name_to_aggregator or array();
221         $this->report_id = $report_id;
222         $this->metrics_to_aggregate = array();
223         $this->parent_to_values = array();
224         $this->values_to_commit = array();
225     }
226
227     private function exit_with_error($message, $details = NULL) {
228         $details['failureStored'] = $this->db->query_and_get_affected_rows(
229             'UPDATE reports SET report_failure = $1, report_failure_details = $2 WHERE report_id = $3',
230             array($message, $details ? json_encode($details) : NULL, $this->report_id)) == 1;
231         exit_with_error($message, $details);
232     }
233
234     function add_aggregated_metric($parent_id, $test_id, $test_name, $metric_name, $aggregator_name, $level) {
235         array_key_exists($aggregator_name, $this->name_to_aggregator) or $this->exit_with_error('AggregatorNotFound', array('name' => $aggregator_name));
236
237         $metric_id = $this->db->select_or_insert_row('test_metrics', 'metric', array('name' => $metric_name,
238             'test' => $test_id, 'aggregator' => $this->name_to_aggregator[$aggregator_name]['aggregator_id']));
239         if (!$metric_id)
240             $this->exit_with_error('FailedToAddAggregatedMetric', array('name' => $metric_name, 'test' => $test_id, 'aggregator' => $aggregator_name));
241
242         array_push($this->metrics_to_aggregate, array(
243             'test_id' => $test_id,
244             'parent_id' => $parent_id,
245             'metric_id' => $metric_id,
246             'test_name' => $test_name,
247             'metric_name' => $metric_name,
248             'aggregator' => $aggregator_name,
249             'aggregator_definition' => $this->name_to_aggregator[$aggregator_name]['aggregator_definition'],
250             'level' => $level));
251     }
252
253     function add_values_for_aggregation($parent_id, $test_name, $metric_name, $config_type, $values, $aggregator = NULL) {
254         $value_list = &array_ensure_item_has_array(array_ensure_item_has_array(array_ensure_item_has_array(array_ensure_item_has_array(
255             $this->parent_to_values, strval($parent_id)), $metric_name), $config_type), $test_name);
256         array_push($value_list, array('aggregator' => $aggregator, 'values' => $values));
257     }
258
259     function aggregate() {
260         $expressions = array();
261         foreach ($this->metrics_to_aggregate as $test_metric) {
262             $configurations = array_get(array_get($this->parent_to_values, strval($test_metric['test_id']), array()), $test_metric['metric_name']);
263             foreach ($configurations as $config_type => $test_value_list) {
264                 // FIXME: We should preserve the test order. For that, we need a new column in our database.
265                 $values_by_iteration = $this->test_value_list_to_values_by_iterations($test_value_list, $test_metric, $test_metric['aggregator']);
266                 $flattened_aggregated_values = array();
267                 for ($i = 0; $i < count($values_by_iteration['values']); ++$i)
268                     array_push($flattened_aggregated_values, $this->aggregate_values($test_metric['aggregator'], $values_by_iteration['values'][$i]));
269
270                 $grouped_values = array();
271                 foreach ($values_by_iteration['group_sizes'] as $size) {
272                     $new_group = array();
273                     for ($i = 0; $i < $size; ++$i)
274                         array_push($new_group, array_shift($flattened_aggregated_values));
275                     array_push($grouped_values, $new_group);
276                 }
277
278                 $this->add_values_to_commit($test_metric['metric_id'], $config_type, $grouped_values);
279                 $this->add_values_for_aggregation($test_metric['parent_id'], $test_metric['test_name'], $test_metric['metric_name'],
280                     $config_type, $grouped_values, $test_metric['aggregator']);
281             }
282         }
283     }
284
285     private function test_value_list_to_values_by_iterations($test_value_list, $test_metric, $aggregator) {
286         $values_by_iterations = array();
287         $group_sizes = array();
288         $first_test = TRUE;
289         foreach ($test_value_list as $test_name => $aggregators_and_values) {
290             if (count($aggregators_and_values) == 1) // Either the subtest has exactly one aggregator or is raw value (not aggregated)
291                 $values = $aggregators_and_values[0]['values'];
292             else {
293                 $values = NULL;
294                 // Find the values of the subtest aggregated by the same aggregator.
295                 foreach ($aggregators_and_values as $aggregator_and_values) {
296                     if ($aggregator_and_values['aggregator'] == $aggregator) {
297                         $values = $aggregator_and_values['values'];
298                         break;                        
299                     }
300                 }
301                 if (!$values) {
302                     $this->exit_with_error('NoMatchingAggregatedValueInSubtest',
303                         array('parent' => $test_metric['test_id'],
304                         'metric' => $test_metric['metric_name'],
305                         'childTest' => $test_name,
306                         'aggregator' => $aggregator,
307                         'aggregatorAndValues' => $aggregators_and_values));
308                 }
309             }
310
311             for ($group = 0; $group < count($values); ++$group) {
312                 if ($first_test) {
313                     array_push($group_sizes, count($values[$group]));
314                     for ($i = 0; $i < count($values[$group]); ++$i)
315                         array_push($values_by_iterations, array());
316                 }
317
318                 if ($group_sizes[$group] != count($values[$group])) {
319                     $this->exit_with_error('IterationGroupSizeIsInconsistent',
320                         array('parent' => $test_metric['test_id'],
321                         'metric' => $test_metric['metric_name'],
322                         'childTest' => $test_name,
323                         'groupSizes' => $group_sizes,
324                         'group' => $group,
325                         'values' => $values));
326                 }
327             }
328             $first_test = FALSE;
329
330             if (count($values) != count($group_sizes)) {
331                 // FIXME: We should support bootstrapping or just computing the mean in this case.
332                 $this->exit_with_error('IterationGroupCountIsInconsistent', array('parent' => $test_metric['test_id'],
333                     'metric' => $test_metric['metric_name'], 'childTest' => $name_and_values['name'],
334                     'valuesByIterations' => $values_by_iterations, 'values' => $values));
335             }
336
337             $flattened_iteration_index = 0;
338             for ($group = 0; $group < count($values); ++$group) {
339                 for ($i = 0; $i < count($values[$group]); ++$i) {
340                     $run_iteration_value = $values[$group][$i];
341                     if (!is_numeric($run_iteration_value)) {
342                         $this->exit_with_error('NonNumeralIterationValueForAggregation', array('parent' => $test_metric['test_id'],
343                             'metric' => $test_metric['metric_name'], 'childTest' => $name_and_values['name'],
344                             'valuesByIterations' => $values_by_iterations, 'values' => $values, 'index' => $i));
345                     }
346                     array_push($values_by_iterations[$flattened_iteration_index], $run_iteration_value);
347                     $flattened_iteration_index++;
348                 }
349             }
350         }
351
352         if (!$values_by_iterations)
353             $this->exit_with_error('NoIterationToAggregation', array('parent' => $test_metric['test_id'], 'metric' => $test_metric['metric_name']));
354
355         return array('values' => $values_by_iterations, 'group_sizes' => $group_sizes);
356     }
357
358     private function aggregate_values($aggregator, $values) {
359         switch ($aggregator) {
360         case 'Arithmetic':
361             return array_sum($values) / count($values);
362         case 'Geometric':
363             return pow(array_product($values), 1 / count($values));
364         case 'Harmonic':
365             return count($values) / array_sum(array_map(function ($x) { return 1 / $x; }, $values));
366         case 'Total':
367             return array_sum($values);
368         case 'SquareSum':
369             return array_sum(array_map(function ($x) { return $x * $x; }, $values));
370         default:
371             $this->exit_with_error('UnknownAggregator', array('aggregator' => $aggregator));
372         }
373         return NULL;
374     }
375
376     function compute_caches() {
377         $expressions = array();
378         $size = count($this->values_to_commit);
379         for ($i = 0; $i < $size; ++$i) {
380             $flattened_value = array();
381             foreach ($this->values_to_commit[$i]['values'] as $group) {
382                 for ($j = 0; $j < count($group); ++$j) {
383                     $iteration_value = $group[$j];
384                     if (gettype($iteration_value) === 'array') { // [relative time, value]
385                         if (count($iteration_value) != 2) {
386                             // FIXME: Also report test and metric.
387                             $this->exit_with_error('InvalidIterationValueFormat', array('values' => $this->values_to_commit[$i]['values']));
388                         }
389                         $iteration_value = $iteration_value[1];
390                     }
391                     array_push($flattened_value, $iteration_value);                    
392                 }
393             }
394             $this->values_to_commit[$i]['mean'] = $this->aggregate_values('Arithmetic', $flattened_value);
395             $this->values_to_commit[$i]['sum'] = $this->aggregate_values('Total', $flattened_value);
396             $this->values_to_commit[$i]['square_sum'] = $this->aggregate_values('SquareSum', $flattened_value);
397         }
398     }
399
400     function add_values_to_commit($metric_id, $config_type, $values) {
401         array_push($this->values_to_commit, array('metric_id' => $metric_id, 'config_type' => $config_type, 'values' => $values));
402     }
403
404     function commit($platform_id, $build_id) {
405         $this->db->begin_transaction() or $this->exit_with_error('FailedToBeginTransaction');
406
407         foreach ($this->values_to_commit as $item) {
408             $config_data = array('metric' => $item['metric_id'], 'type' => $item['config_type'], 'platform' => $platform_id);
409             $config_id = $this->db->select_or_insert_row('test_configurations', 'config', $config_data);
410             if (!$config_id)
411                 $this->rollback_with_error('FailedToObtainConfiguration', $config_data);
412
413             $values = $item['values'];
414             $total_count = 0;
415             for ($group = 0; $group < count($values); ++$group)
416                 $total_count += count($values[$group]);
417             $run_data = array('config' => $config_id, 'build' => $build_id, 'iteration_count_cache' => $total_count,
418                 'mean_cache' => $item['mean'], 'sum_cache' => $item['sum'], 'square_sum_cache' => $item['square_sum']);
419             $run_id = $this->db->insert_row('test_runs', 'run', $run_data);
420             if (!$run_id)
421                 $this->rollback_with_error('FailedToInsertRun', array('metric' => $item['metric_id'], 'param' => $run_data));
422
423             $flattened_order = 0;
424             for ($group = 0; $group < count($values); ++$group) {
425                 for ($i = 0; $i < count($values[$group]); ++$i) {
426                     $iteration_value = $values[$group][$i];
427                     $relative_time = NULL;
428                     if (gettype($iteration_value) === 'array') {
429                         assert(count($iteration_value) == 2); // compute_caches checks this condition.
430                         $relative_time = $iteration_value[0];
431                         $iteration_value = $iteration_value[1];
432                     }
433                     $param = array('run' => $run_id, 'order' => $flattened_order, 'value' => $iteration_value,
434                         'group' => count($values) == 1 ? NULL : $group, 'relative_time' => $relative_time);
435                     $this->db->insert_row('run_iterations', 'iteration', $param, NULL)
436                         or $this->rollback_with_error('FailedToInsertIteration', array('config' => $config_id, 'build' => $build_id, 'param' => $param));
437                     $flattened_order++;
438                 }
439             }
440         }
441
442         $this->db->query_and_get_affected_rows("UPDATE reports SET report_committed_at = CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
443             WHERE report_id = $1", array($this->report_id));
444
445         $this->db->commit_transaction() or $this->exit_with_error('FailedToCommitTransaction');
446     }
447
448     private function rollback_with_error($message, $details) {
449         $this->db->rollback_transaction();
450         $this->exit_with_error($message, $details);
451     }
452 };
453
454 ?>