Websites/perf.webkit.org/public/include/report-processor.php

   1 <?php
   2
   3 require_once('../include/json-header.php');
   4
   5 class ReportProcessor {
   6     private $db;
   7     private $name_to_aggregator;
   8     private $report_id;
   9     private $runs;
  10
  11     function __construct($db) {
  12         $this->db = $db;
  13         $this->name_to_aggregator = array();
  14         $aggregator_table = $db->fetch_table('aggregators');
  15         if ($aggregator_table) {
  16             foreach ($aggregator_table as $aggregator_row) {
  17                 $this->name_to_aggregator[$aggregator_row['aggregator_name']] = $aggregator_row;
  18             }
  19         }
  20     }
  21
  22     private function exit_with_error($message, $details = NULL) {
  23         if (!$this->report_id) {
  24             $details['failureStored'] = FALSE;
  25             exit_with_error($message, $details);
  26         }
  27
  28         $details['failureStored'] = $this->db->query_and_get_affected_rows(
  29             'UPDATE reports SET report_failure = $1, report_failure_details = $2 WHERE report_id = $3',
  30             array($message, $details ? json_encode($details) : NULL, $this->report_id)) == 1;
  31         exit_with_error($message, $details);
  32     }
  33
  34     function process($report, $existing_report_id = NULL) {
  35         $this->report_id = $existing_report_id;
  36         $this->runs = NULL;
  37
  38         array_key_exists('builderName', $report) or $this->exit_with_error('MissingBuilderName');
  39         array_key_exists('buildTime', $report) or $this->exit_with_error('MissingBuildTime');
  40
  41         $builder_info = array('name' => $report['builderName']);
  42         if (!$existing_report_id)
  43             $builder_info['password_hash'] = hash('sha256', $report['builderPassword']);
  44         if (array_key_exists('builderPassword', $report))
  45             unset($report['builderPassword']);
  46
  47         $matched_builder = $this->db->select_first_row('builders', 'builder', $builder_info);
  48         if (!$matched_builder)
  49             $this->exit_with_error('BuilderNotFound', array('name' => $builder_info['name']));
  50
  51         $build_data = $this->construct_build_data($report, $matched_builder);
  52         if (!$existing_report_id)
  53             $this->store_report($report, $build_data);
  54
  55         $this->runs = new TestRunsGenerator($this->db, $this->name_to_aggregator, $this->report_id);
  56         $this->recursively_ensure_tests($report['tests']);
  57
  58         $this->runs->aggregate();
  59         $this->runs->compute_caches();
  60
  61         $platform_id = $this->db->select_or_insert_row('platforms', 'platform', array('name' => $report['platform']));
  62         if (!$platform_id)
  63             $this->exit_with_error('FailedToInsertPlatform', array('name' => $report['platform']));
  64
  65         $build_id = $this->resolve_build_id($build_data, array_get($report, 'revisions', array()));
  66
  67         $this->runs->commit($platform_id, $build_id);
  68     }
  69
  70     private function construct_build_data($report, $builder) {
  71         array_key_exists('buildNumber', $report) or $this->exit_with_error('MissingBuildNumber');
  72         array_key_exists('buildTime', $report) or $this->exit_with_error('MissingBuildTime');
  73
  74         return array('builder' => $builder['builder_id'], 'number' => $report['buildNumber'], 'time' => $report['buildTime']);
  75     }
  76
  77     private function store_report($report, $build_data) {
  78         assert(!$this->report_id);
  79         $this->report_id = $this->db->insert_row('reports', 'report', array('builder' => $build_data['builder'], 'build_number' => $build_data['number'],
  80             'content' => json_encode($report)));
  81         if (!$this->report_id)
  82             $this->exit_with_error('FailedToStoreRunReport');
  83     }
  84
  85     private function resolve_build_id($build_data, $revisions) {
  86         // FIXME: This code has a race condition. See <rdar://problem/15876303>.
  87         $results = $this->db->query_and_fetch_all("SELECT build_id FROM builds WHERE build_builder = $1 AND build_number = $2 AND build_time <= $3 AND build_time + interval '1 day' > $3",
  88             array($build_data['builder'], $build_data['number'], $build_data['time']));
  89         if ($results)
  90             $build_id = $results[0]['build_id'];
  91         else
  92             $build_id = $this->db->insert_row('builds', 'build', $build_data);
  93         if (!$build_id)
  94             $this->exit_with_error('FailedToInsertBuild', $build_data);
  95
  96         foreach ($revisions as $repository_name => $revision_data) {
  97             $repository_id = $this->db->select_or_insert_row('repositories', 'repository', array('name' => $repository_name));
  98             if (!$repository_id)
  99                 $this->exit_with_error('FailedToInsertRepository', array('name' => $repository_name));
 100
 101             $commit_data = array('repository' => $repository_id, 'revision' => $revision_data['revision'], 'time' => array_get($revision_data, 'timestamp'));
 102
 103             $mismatching_commit = $this->db->query_and_fetch_all('SELECT * FROM build_commits, commits
 104                 WHERE build_commit = commit_id AND commit_build = $1 AND commit_repository = $2 AND commit_revision != $3 LIMIT 1',
 105                 array($build_id, $repository_id, $revision_data['revision']));
 106             if ($mismatching_commit)
 107                 $this->exit_with_error('MismatchingCommitRevision', array('build' => $build_id, 'existing' => $mismatching_commit, 'new' => $commit_data));
 108
 109             $commit_row = $this->db->select_or_insert_row('commits', 'commit',
 110                 array('repository' => $repository_id, 'revision' => $revision_data['revision']), $commit_data, '*');
 111             if (!$commit_row)
 112                 $this->exit_with_error('FailedToRecordCommit', $commit_data);
 113             if (abs($commit_row['commit_time'] - $commit_data['time']) > 1.0)
 114                 $this->exit_with_error('MismatchingCommitTime', array('existing' => $commit_row, 'new' => $commit_data));
 115
 116             if (!$this->db->select_or_insert_row('build_commits', null,
 117                 array('commit_build' => $build_id, 'build_commit' => $commit_row['commit_id']), null, '*'))
 118                 $this->exit_with_error('FailedToRelateCommitToBuild', array('commit' => $commit_row, 'build' => $build_id));
 119         }
 120
 121         return $build_id;
 122     }
 123
 124     private function recursively_ensure_tests($tests, $parent_id = NULL, $level = 0) {
 125         foreach ($tests as $test_name => $test) {
 126             $test_id = $this->db->select_or_insert_row('tests', 'test', $parent_id ? array('name' => $test_name, 'parent' => $parent_id) : array('name' => $test_name),
 127                 array('name' => $test_name, 'parent' => $parent_id, 'url' => array_get($test, 'url')));
 128             if (!$test_id)
 129                 $this->exit_with_error('FailedToAddTest', array('name' => $test_name, 'parent' => $parent_id));
 130
 131             if (array_key_exists('tests', $test))
 132                 $this->recursively_ensure_tests($test['tests'], $test_id, $level + 1);
 133
 134             foreach (array_get($test, 'metrics', array()) as $metric_name => $aggregators_or_config_types) {
 135                 $aggregators = $this->aggregator_list_if_exists($aggregators_or_config_types);
 136                 if ($aggregators) {
 137                     foreach ($aggregators as $aggregator_name)
 138                         $this->runs->add_aggregated_metric($parent_id, $test_id, $test_name, $metric_name, $aggregator_name, $level);
 139                 } else {
 140                     $metric_id = $this->db->select_or_insert_row('test_metrics', 'metric', array('name' => $metric_name, 'test' => $test_id));
 141                     if (!$metric_id)
 142                         $this->exit_with_error('FailedToAddMetric', array('name' => $metric_name, 'test' => $test_id));
 143
 144                     foreach ($aggregators_or_config_types as $config_type => $values) {
 145                         // Some tests submit groups of iterations; e.g. [[1, 2, 3, 4], [5, 6, 7, 8]]
 146                         // Convert other tests to this format to simplify the computation later.
 147                         if (gettype($values) !== 'array')
 148                             $values = array($values);
 149                         if (gettype($values[0]) !== 'array')
 150                             $values = array($values);
 151                         $this->runs->add_values_to_commit($metric_id, $config_type, $values);
 152                         $this->runs->add_values_for_aggregation($parent_id, $test_name, $metric_name, $config_type, $values);
 153                     }
 154                 }
 155             }
 156         }
 157     }
 158
 159     private function aggregator_list_if_exists($aggregators_or_config_types) {
 160         if (array_key_exists(0, $aggregators_or_config_types))
 161             return $aggregators_or_config_types;
 162         else if (array_get($aggregators_or_config_types, 'aggregators'))
 163             return $aggregators_or_config_types['aggregators'];
 164         return NULL;
 165     }
 166 };
 167
 168 class TestRunsGenerator {
 169     private $db;
 170     private $name_to_aggregator;
 171     private $report_id;
 172     private $metrics_to_aggregate;
 173     private $parent_to_values;
 174     private $values_to_commit;
 175
 176     function __construct($db, $name_to_aggregator, $report_id) {
 177         $this->db = $db;
 178         $this->name_to_aggregator = $name_to_aggregator or array();
 179         $this->report_id = $report_id;
 180         $this->metrics_to_aggregate = array();
 181         $this->parent_to_values = array();
 182         $this->values_to_commit = array();
 183     }
 184
 185     private function exit_with_error($message, $details = NULL) {
 186         $details['failureStored'] = $this->db->query_and_get_affected_rows(
 187             'UPDATE reports SET report_failure = $1, report_failure_details = $2 WHERE report_id = $3',
 188             array($message, $details ? json_encode($details) : NULL, $this->report_id)) == 1;
 189         exit_with_error($message, $details);
 190     }
 191
 192     function add_aggregated_metric($parent_id, $test_id, $test_name, $metric_name, $aggregator_name, $level) {
 193         array_key_exists($aggregator_name, $this->name_to_aggregator) or $this->exit_with_error('AggregatorNotFound', array('name' => $aggregator_name));
 194
 195         $metric_id = $this->db->select_or_insert_row('test_metrics', 'metric', array('name' => $metric_name,
 196             'test' => $test_id, 'aggregator' => $this->name_to_aggregator[$aggregator_name]['aggregator_id']));
 197         if (!$metric_id)
 198             $this->exit_with_error('FailedToAddAggregatedMetric', array('name' => $metric_name, 'test' => $test_id, 'aggregator' => $aggregator_name));
 199
 200         array_push($this->metrics_to_aggregate, array(
 201             'test_id' => $test_id,
 202             'parent_id' => $parent_id,
 203             'metric_id' => $metric_id,
 204             'test_name' => $test_name,
 205             'metric_name' => $metric_name,
 206             'aggregator' => $aggregator_name,
 207             'aggregator_definition' => $this->name_to_aggregator[$aggregator_name]['aggregator_definition'],
 208             'level' => $level));
 209     }
 210
 211     function add_values_for_aggregation($parent_id, $test_name, $metric_name, $config_type, $values, $aggregator = NULL) {
 212         $value_list = &array_ensure_item_has_array(array_ensure_item_has_array(array_ensure_item_has_array(array_ensure_item_has_array(
 213             $this->parent_to_values, strval($parent_id)), $metric_name), $config_type), $test_name);
 214         array_push($value_list, array('aggregator' => $aggregator, 'values' => $values));
 215     }
 216
 217     function aggregate() {
 218         $expressions = array();
 219         foreach ($this->metrics_to_aggregate as $test_metric) {
 220             $configurations = array_get(array_get($this->parent_to_values, strval($test_metric['test_id']), array()), $test_metric['metric_name']);
 221             foreach ($configurations as $config_type => $test_value_list) {
 222                 // FIXME: We should preserve the test order. For that, we need a new column in our database.
 223                 $values_by_iteration = $this->test_value_list_to_values_by_iterations($test_value_list, $test_metric, $test_metric['aggregator']);
 224                 $flattened_aggregated_values = array();
 225                 for ($i = 0; $i < count($values_by_iteration['values']); ++$i)
 226                     array_push($flattened_aggregated_values, $this->aggregate_values($test_metric['aggregator'], $values_by_iteration['values'][$i]));
 227
 228                 $grouped_values = array();
 229                 foreach ($values_by_iteration['group_sizes'] as $size) {
 230                     $new_group = array();
 231                     for ($i = 0; $i < $size; ++$i)
 232                         array_push($new_group, array_shift($flattened_aggregated_values));
 233                     array_push($grouped_values, $new_group);
 234                 }
 235
 236                 $this->add_values_to_commit($test_metric['metric_id'], $config_type, $grouped_values);
 237                 $this->add_values_for_aggregation($test_metric['parent_id'], $test_metric['test_name'], $test_metric['metric_name'],
 238                     $config_type, $grouped_values, $test_metric['aggregator']);
 239             }
 240         }
 241     }
 242
 243     private function test_value_list_to_values_by_iterations($test_value_list, $test_metric, $aggregator) {
 244         $values_by_iterations = array();
 245         $group_sizes = array();
 246         $first_test = TRUE;
 247         foreach ($test_value_list as $test_name => $aggregators_and_values) {
 248             if (count($aggregators_and_values) == 1) // Either the subtest has exactly one aggregator or is raw value (not aggregated)
 249                 $values = $aggregators_and_values[0]['values'];
 250             else {
 251                 $values = NULL;
 252                 // Find the values of the subtest aggregated by the same aggregator.
 253                 foreach ($aggregators_and_values as $aggregator_and_values) {
 254                     if ($aggregator_and_values['aggregator'] == $aggregator) {
 255                         $values = $aggregator_and_values['values'];
 256                         break;
 257                     }
 258                 }
 259                 if (!$values) {
 260                     $this->exit_with_error('NoMatchingAggregatedValueInSubtest',
 261                         array('parent' => $test_metric['test_id'],
 262                         'metric' => $test_metric['metric_name'],
 263                         'childTest' => $test_name,
 264                         'aggregator' => $aggregator,
 265                         'aggregatorAndValues' => $aggregators_and_values));
 266                 }
 267             }
 268
 269             for ($group = 0; $group < count($values); ++$group) {
 270                 if ($first_test) {
 271                     array_push($group_sizes, count($values[$group]));
 272                     for ($i = 0; $i < count($values[$group]); ++$i)
 273                         array_push($values_by_iterations, array());
 274                 }
 275
 276                 if ($group_sizes[$group] != count($values[$group])) {
 277                     $this->exit_with_error('IterationGroupSizeIsInconsistent',
 278                         array('parent' => $test_metric['test_id'],
 279                         'metric' => $test_metric['metric_name'],
 280                         'childTest' => $test_name,
 281                         'groupSizes' => $group_sizes,
 282                         'group' => $group,
 283                         'values' => $values));
 284                 }
 285             }
 286             $first_test = FALSE;
 287
 288             if (count($values) != count($group_sizes)) {
 289                 // FIXME: We should support bootstrapping or just computing the mean in this case.
 290                 $this->exit_with_error('IterationGroupCountIsInconsistent', array('parent' => $test_metric['test_id'],
 291                     'metric' => $test_metric['metric_name'], 'childTest' => $name_and_values['name'],
 292                     'valuesByIterations' => $values_by_iterations, 'values' => $values));
 293             }
 294
 295             $flattened_iteration_index = 0;
 296             for ($group = 0; $group < count($values); ++$group) {
 297                 for ($i = 0; $i < count($values[$group]); ++$i) {
 298                     $run_iteration_value = $values[$group][$i];
 299                     if (!is_numeric($run_iteration_value)) {
 300                         $this->exit_with_error('NonNumeralIterationValueForAggregation', array('parent' => $test_metric['test_id'],
 301                             'metric' => $test_metric['metric_name'], 'childTest' => $name_and_values['name'],
 302                             'valuesByIterations' => $values_by_iterations, 'values' => $values, 'index' => $i));
 303                     }
 304                     array_push($values_by_iterations[$flattened_iteration_index], $run_iteration_value);
 305                     $flattened_iteration_index++;
 306                 }
 307             }
 308         }
 309
 310         if (!$values_by_iterations)
 311             $this->exit_with_error('NoIterationToAggregation', array('parent' => $test_metric['test_id'], 'metric' => $test_metric['metric_name']));
 312
 313         return array('values' => $values_by_iterations, 'group_sizes' => $group_sizes);
 314     }
 315
 316     private function aggregate_values($aggregator, $values) {
 317         switch ($aggregator) {
 318         case 'Arithmetic':
 319             return array_sum($values) / count($values);
 320         case 'Geometric':
 321             return pow(array_product($values), 1 / count($values));
 322         case 'Harmonic':
 323             return count($values) / array_sum(array_map(function ($x) { return 1 / $x; }, $values));
 324         case 'Total':
 325             return array_sum($values);
 326         case 'SquareSum':
 327             return array_sum(array_map(function ($x) { return $x * $x; }, $values));
 328         default:
 329             $this->exit_with_error('UnknownAggregator', array('aggregator' => $aggregator));
 330         }
 331         return NULL;
 332     }
 333
 334     function compute_caches() {
 335         $expressions = array();
 336         $size = count($this->values_to_commit);
 337         for ($i = 0; $i < $size; ++$i) {
 338             $flattened_value = array();
 339             foreach ($this->values_to_commit[$i]['values'] as $group) {
 340                 for ($j = 0; $j < count($group); ++$j) {
 341                     $iteration_value = $group[$j];
 342                     if (gettype($iteration_value) === 'array') { // [relative time, value]
 343                         if (count($iteration_value) != 2) {
 344                             // FIXME: Also report test and metric.
 345                             $this->exit_with_error('InvalidIterationValueFormat', array('values' => $this->values_to_commit[$i]['values']));
 346                         }
 347                         $iteration_value = $iteration_value[1];
 348                     }
 349                     array_push($flattened_value, $iteration_value);
 350                 }
 351             }
 352             $this->values_to_commit[$i]['mean'] = $this->aggregate_values('Arithmetic', $flattened_value);
 353             $this->values_to_commit[$i]['sum'] = $this->aggregate_values('Total', $flattened_value);
 354             $this->values_to_commit[$i]['square_sum'] = $this->aggregate_values('SquareSum', $flattened_value);
 355         }
 356     }
 357
 358     function add_values_to_commit($metric_id, $config_type, $values) {
 359         array_push($this->values_to_commit, array('metric_id' => $metric_id, 'config_type' => $config_type, 'values' => $values));
 360     }
 361
 362     function commit($platform_id, $build_id) {
 363         $this->db->begin_transaction() or $this->exit_with_error('FailedToBeginTransaction');
 364
 365         foreach ($this->values_to_commit as $item) {
 366             $config_data = array('metric' => $item['metric_id'], 'type' => $item['config_type'], 'platform' => $platform_id);
 367             $config_id = $this->db->select_or_insert_row('test_configurations', 'config', $config_data);
 368             if (!$config_id)
 369                 $this->rollback_with_error('FailedToObtainConfiguration', $config_data);
 370
 371             $values = $item['values'];
 372             $total_count = 0;
 373             for ($group = 0; $group < count($values); ++$group)
 374                 $total_count += count($values[$group]);
 375             $run_data = array('config' => $config_id, 'build' => $build_id, 'iteration_count_cache' => $total_count,
 376                 'mean_cache' => $item['mean'], 'sum_cache' => $item['sum'], 'square_sum_cache' => $item['square_sum']);
 377             $run_id = $this->db->insert_row('test_runs', 'run', $run_data);
 378             if (!$run_id)
 379                 $this->rollback_with_error('FailedToInsertRun', array('metric' => $item['metric_id'], 'param' => $run_data));
 380
 381             $flattened_order = 0;
 382             for ($group = 0; $group < count($values); ++$group) {
 383                 for ($i = 0; $i < count($values[$group]); ++$i) {
 384                     $iteration_value = $values[$group][$i];
 385                     $relative_time = NULL;
 386                     if (gettype($iteration_value) === 'array') {
 387                         assert(count($iteration_value) == 2); // compute_caches checks this condition.
 388                         $relative_time = $iteration_value[0];
 389                         $iteration_value = $iteration_value[1];
 390                     }
 391                     $param = array('run' => $run_id, 'order' => $flattened_order, 'value' => $iteration_value,
 392                         'group' => count($values) == 1 ? NULL : $group, 'relative_time' => $relative_time);
 393                     $this->db->insert_row('run_iterations', 'iteration', $param, NULL)
 394                         or $this->rollback_with_error('FailedToInsertIteration', array('config' => $config_id, 'build' => $build_id, 'param' => $param));
 395                     $flattened_order++;
 396                 }
 397             }
 398         }
 399
 400         $this->db->query_and_get_affected_rows("UPDATE reports SET report_committed_at = CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
 401             WHERE report_id = $1", array($this->report_id));
 402
 403         $this->db->commit_transaction() or $this->exit_with_error('FailedToCommitTransaction');
 404     }
 405
 406     private function rollback_with_error($message, $details) {
 407         $this->db->rollback_transaction();
 408         $this->exit_with_error($message, $details);
 409     }
 410 };
 411
 412 ?>