Websites/perf.webkit.org/public/include/report-processor.php

   1 <?php
   2
   3 require_once('../include/json-header.php');
   4
   5 class ReportProcessor {
   6     private $db;
   7     private $name_to_aggregator_id;
   8     private $report_id;
   9     private $runs;
  10
  11     function __construct($db) {
  12         $this->db = $db;
  13         $this->name_to_aggregator_id = array();
  14     }
  15
  16     private function exit_with_error($message, $details = NULL) {
  17         if (!$this->report_id) {
  18             $details['failureStored'] = FALSE;
  19             exit_with_error($message, $details);
  20         }
  21
  22         $details['failureStored'] = $this->db->query_and_get_affected_rows(
  23             'UPDATE reports SET report_failure = $1, report_failure_details = $2 WHERE report_id = $3',
  24             array($message, $details ? json_encode($details) : NULL, $this->report_id)) == 1;
  25         exit_with_error($message, $details);
  26     }
  27
  28     function process($report, $existing_report_id = NULL) {
  29         $this->report_id = $existing_report_id;
  30         $this->runs = NULL;
  31
  32         array_key_exists('builderName', $report) or $this->exit_with_error('MissingBuilderName');
  33         array_key_exists('buildTime', $report) or $this->exit_with_error('MissingBuildTime');
  34
  35         $build_data = $this->authenticate_and_construct_build_data($report, $existing_report_id);
  36         if (!$existing_report_id)
  37             $this->store_report($report, $build_data);
  38
  39         $this->ensure_aggregators();
  40
  41         $this->runs = new TestRunsGenerator($this->db, $this->name_to_aggregator_id, $this->report_id);
  42         $this->recursively_ensure_tests($report['tests']);
  43
  44         $this->runs->aggregate();
  45         $this->runs->compute_caches();
  46
  47         $platform_id = $this->db->select_or_insert_row('platforms', 'platform', array('name' => $report['platform']));
  48         if (!$platform_id)
  49             $this->exit_with_error('FailedToInsertPlatform', array('name' => $report['platform']));
  50
  51         // FIXME: Deprecate and unsupport "jobId".
  52         $build_id = $this->resolve_build_id($build_data, array_get($report, 'revisions', array()),
  53             array_get($report, 'jobId', array_get($report, 'buildRequest')));
  54
  55         $this->runs->commit($platform_id, $build_id);
  56     }
  57
  58     private function authenticate_and_construct_build_data(&$report, $existing_report_id) {
  59         $builder_info = array('name' => $report['builderName']);
  60         $slave_name = array_get($report, 'slaveName', NULL);
  61         $slave_id = NULL;
  62         if (!$existing_report_id) {
  63             $hash = NULL;
  64             if ($slave_name && array_key_exists('slavePassword', $report)) {
  65                 $universal_password = config('universalSlavePassword');
  66                 if ($universal_password && $universal_password == $report['slavePassword'])
  67                     $slave = $this->db->select_first_row('build_slaves', 'slave', array('name' => $slave_name));
  68                 else {
  69                     $hash = hash('sha256', $report['slavePassword']);
  70                     $slave = $this->db->select_first_row('build_slaves', 'slave', array('name' => $slave_name, 'password_hash' => $hash));
  71                 }
  72                 if ($slave)
  73                     $slave_id = $slave['slave_id'];
  74             } else if (array_key_exists('builderPassword', $report))
  75                 $hash = hash('sha256', $report['builderPassword']);
  76
  77             if (!$hash)
  78                 $this->exit_with_error('BuilderNotFound');
  79             if (!$slave_id)
  80                 $builder_info['password_hash'] = $hash;
  81         }
  82
  83         if (array_key_exists('builderPassword', $report))
  84             unset($report['builderPassword']);
  85         if (array_key_exists('slavePassword', $report))
  86             unset($report['slavePassword']);
  87
  88         $builder_id = NULL;
  89         if ($slave_id)
  90             $builder_id = $this->db->select_or_insert_row('builders', 'builder', $builder_info);
  91         else {
  92             $builder = $this->db->select_first_row('builders', 'builder', $builder_info);
  93             if (!$builder)
  94                 $this->exit_with_error('BuilderNotFound', array('name' => $builder_info['name']));
  95             $builder_id = $builder['builder_id'];
  96             if ($slave_name)
  97                 $slave_id = $this->db->select_or_insert_row('build_slaves', 'slave', array('name' => $slave_name));
  98         }
  99
 100         return $this->construct_build_data($report, $builder_id, $slave_id);
 101     }
 102
 103     private function construct_build_data(&$report, $builder_id, $slave_id) {
 104         array_key_exists('buildNumber', $report) or $this->exit_with_error('MissingBuildNumber');
 105         array_key_exists('buildTime', $report) or $this->exit_with_error('MissingBuildTime');
 106
 107         return array('builder' => $builder_id, 'slave' => $slave_id, 'number' => $report['buildNumber'], 'time' => $report['buildTime']);
 108     }
 109
 110     private function store_report(&$report, $build_data) {
 111         assert(!$this->report_id);
 112         $this->report_id = $this->db->insert_row('reports', 'report', array(
 113             'builder' => $build_data['builder'],
 114             'slave' => $build_data['slave'],
 115             'build_number' => $build_data['number'],
 116             'content' => json_encode($report)));
 117         if (!$this->report_id)
 118             $this->exit_with_error('FailedToStoreRunReport');
 119     }
 120
 121     private function ensure_aggregators() {
 122         foreach (TestRunsGenerator::$aggregators as $name) {
 123             $id = $this->db->select_or_insert_row('aggregators', 'aggregator', array('name' => $name));
 124             if (!$id)
 125                 $this->exit_with_error('FailedToInsertAggregator', array('aggregator' => $name));
 126             $this->name_to_aggregator_id[$name] = $id;
 127         }
 128     }
 129
 130     private function resolve_build_id(&$build_data, $revisions, $build_request_id) {
 131         // FIXME: This code has a race condition. See <rdar://problem/15876303>.
 132         $results = $this->db->query_and_fetch_all("SELECT build_id, build_slave FROM builds
 133             WHERE build_builder = $1 AND build_number = $2 AND build_time <= $3 AND build_time + interval '1 day' > $3",
 134             array($build_data['builder'], $build_data['number'], $build_data['time']));
 135         if ($results) {
 136             $first_result = $results[0];
 137             if ($first_result['build_slave'] != $build_data['slave'])
 138                 $this->exit_with_error('MismatchingBuildSlave', array('storedBuild' => $results, 'reportedBuildData' => $build_data));
 139             $build_id = $first_result['build_id'];
 140         } else
 141             $build_id = $this->db->insert_row('builds', 'build', $build_data);
 142         if (!$build_id)
 143             $this->exit_with_error('FailedToInsertBuild', $build_data);
 144
 145         if ($build_request_id) {
 146             if ($this->db->update_row('build_requests', 'request', array('id' => $build_request_id), array('status' => 'completed', 'build' => $build_id))
 147                 != $build_request_id)
 148                 $this->exit_with_error('FailedToUpdateBuildRequest', array('buildRequest' => $build_request_id, 'build' => $build_id));
 149         }
 150
 151
 152         foreach ($revisions as $repository_name => $revision_data) {
 153             $repository_id = $this->db->select_or_insert_row('repositories', 'repository', array('name' => $repository_name));
 154             if (!$repository_id)
 155                 $this->exit_with_error('FailedToInsertRepository', array('name' => $repository_name));
 156
 157             $commit_data = array('repository' => $repository_id, 'revision' => $revision_data['revision'], 'time' => array_get($revision_data, 'timestamp'));
 158
 159             $mismatching_commit = $this->db->query_and_fetch_all('SELECT * FROM build_commits, commits
 160                 WHERE build_commit = commit_id AND commit_build = $1 AND commit_repository = $2 AND commit_revision != $3 LIMIT 1',
 161                 array($build_id, $repository_id, $revision_data['revision']));
 162             if ($mismatching_commit)
 163                 $this->exit_with_error('MismatchingCommitRevision', array('build' => $build_id, 'existing' => $mismatching_commit, 'new' => $commit_data));
 164
 165             $commit_row = $this->db->select_or_insert_row('commits', 'commit',
 166                 array('repository' => $repository_id, 'revision' => $revision_data['revision']), $commit_data, '*');
 167             if (!$commit_row)
 168                 $this->exit_with_error('FailedToRecordCommit', $commit_data);
 169             if ($commit_data['time'] && abs($commit_row['commit_time'] - $commit_data['time']) > 1.0)
 170                 $this->exit_with_error('MismatchingCommitTime', array('existing' => $commit_row, 'new' => $commit_data));
 171
 172             if (!$this->db->select_or_insert_row('build_commits', null,
 173                 array('commit_build' => $build_id, 'build_commit' => $commit_row['commit_id']), null, '*'))
 174                 $this->exit_with_error('FailedToRelateCommitToBuild', array('commit' => $commit_row, 'build' => $build_id));
 175         }
 176
 177         return $build_id;
 178     }
 179
 180     private function recursively_ensure_tests(&$tests, $parent_id = NULL, $level = 0) {
 181         foreach ($tests as $test_name => $test) {
 182             $test_id = $this->db->select_or_insert_row('tests', 'test', $parent_id ? array('name' => $test_name, 'parent' => $parent_id) : array('name' => $test_name),
 183                 array('name' => $test_name, 'parent' => $parent_id, 'url' => array_get($test, 'url')));
 184             if (!$test_id)
 185                 $this->exit_with_error('FailedToAddTest', array('name' => $test_name, 'parent' => $parent_id));
 186
 187             if (array_key_exists('tests', $test))
 188                 $this->recursively_ensure_tests($test['tests'], $test_id, $level + 1);
 189
 190             foreach (array_get($test, 'metrics', array()) as $metric_name => $aggregators_or_config_types) {
 191                 $aggregators = $this->aggregator_list_if_exists($aggregators_or_config_types);
 192                 if ($aggregators) {
 193                     foreach ($aggregators as $aggregator_name)
 194                         $this->runs->add_aggregated_metric($parent_id, $test_id, $test_name, $metric_name, $aggregator_name, $level);
 195                 } else {
 196                     $metric_id = $this->db->select_or_insert_row('test_metrics', 'metric', array('name' => $metric_name, 'test' => $test_id));
 197                     if (!$metric_id)
 198                         $this->exit_with_error('FailedToAddMetric', array('name' => $metric_name, 'test' => $test_id));
 199
 200                     foreach ($aggregators_or_config_types as $config_type => $values) {
 201                         // Some tests submit groups of iterations; e.g. [[1, 2, 3, 4], [5, 6, 7, 8]]
 202                         // Convert other tests to this format to simplify the computation later.
 203                         if (gettype($values) !== 'array')
 204                             $values = array($values);
 205                         if (gettype($values[0]) !== 'array')
 206                             $values = array($values);
 207                         $this->runs->add_values_to_commit($metric_id, $config_type, $values);
 208                         $this->runs->add_values_for_aggregation($parent_id, $test_name, $metric_name, $config_type, $values);
 209                     }
 210                 }
 211             }
 212         }
 213     }
 214
 215     private function aggregator_list_if_exists($aggregators_or_config_types) {
 216         if (array_key_exists(0, $aggregators_or_config_types))
 217             return $aggregators_or_config_types;
 218         else if (array_get($aggregators_or_config_types, 'aggregators'))
 219             return $aggregators_or_config_types['aggregators'];
 220         return NULL;
 221     }
 222 };
 223
 224 class TestRunsGenerator {
 225     private $db;
 226     private $name_to_aggregator_id;
 227     private $report_id;
 228     private $metrics_to_aggregate;
 229     private $parent_to_values;
 230     private $values_to_commit;
 231
 232     function __construct($db, $name_to_aggregator_id, $report_id) {
 233         $this->db = $db;
 234         $this->name_to_aggregator_id = $name_to_aggregator_id;
 235         $this->report_id = $report_id;
 236         $this->metrics_to_aggregate = array();
 237         $this->parent_to_values = array();
 238         $this->values_to_commit = array();
 239     }
 240
 241     private function exit_with_error($message, $details = NULL) {
 242         $details['failureStored'] = $this->db->query_and_get_affected_rows(
 243             'UPDATE reports SET report_failure = $1, report_failure_details = $2 WHERE report_id = $3',
 244             array($message, $details ? json_encode($details) : NULL, $this->report_id)) == 1;
 245         exit_with_error($message, $details);
 246     }
 247
 248     function add_aggregated_metric($parent_id, $test_id, $test_name, $metric_name, $aggregator_name, $level) {
 249         array_key_exists($aggregator_name, $this->name_to_aggregator_id)
 250             or $this->exit_with_error('AggregatorNotFound', array('name' => $aggregator_name));
 251
 252         $metric_id = $this->db->select_or_insert_row('test_metrics', 'metric', array('name' => $metric_name,
 253             'test' => $test_id, 'aggregator' => $this->name_to_aggregator_id[$aggregator_name]));
 254         if (!$metric_id)
 255             $this->exit_with_error('FailedToAddAggregatedMetric', array('name' => $metric_name, 'test' => $test_id, 'aggregator' => $aggregator_name));
 256
 257         array_push($this->metrics_to_aggregate, array(
 258             'test_id' => $test_id,
 259             'parent_id' => $parent_id,
 260             'metric_id' => $metric_id,
 261             'test_name' => $test_name,
 262             'metric_name' => $metric_name,
 263             'aggregator' => $aggregator_name,
 264             'level' => $level));
 265     }
 266
 267     function add_values_for_aggregation($parent_id, $test_name, $metric_name, $config_type, $values, $aggregator = NULL) {
 268         $value_list = &array_ensure_item_has_array(array_ensure_item_has_array(array_ensure_item_has_array(array_ensure_item_has_array(
 269             $this->parent_to_values, strval($parent_id)), $metric_name), $config_type), $test_name);
 270         array_push($value_list, array('aggregator' => $aggregator, 'values' => $values));
 271     }
 272
 273     function aggregate() {
 274         $expressions = array();
 275         foreach ($this->metrics_to_aggregate as $test_metric) {
 276             $configurations = array_get(array_get($this->parent_to_values, strval($test_metric['test_id']), array()), $test_metric['metric_name']);
 277             foreach ($configurations as $config_type => $test_value_list) {
 278                 // FIXME: We should preserve the test order. For that, we need a new column in our database.
 279                 $values_by_iteration = $this->test_value_list_to_values_by_iterations($test_value_list, $test_metric, $test_metric['aggregator']);
 280                 $flattened_aggregated_values = array();
 281                 for ($i = 0; $i < count($values_by_iteration['values']); ++$i)
 282                     array_push($flattened_aggregated_values, $this->aggregate_values($test_metric['aggregator'], $values_by_iteration['values'][$i]));
 283
 284                 $grouped_values = array();
 285                 foreach ($values_by_iteration['group_sizes'] as $size) {
 286                     $new_group = array();
 287                     for ($i = 0; $i < $size; ++$i)
 288                         array_push($new_group, array_shift($flattened_aggregated_values));
 289                     array_push($grouped_values, $new_group);
 290                 }
 291
 292                 $this->add_values_to_commit($test_metric['metric_id'], $config_type, $grouped_values);
 293                 $this->add_values_for_aggregation($test_metric['parent_id'], $test_metric['test_name'], $test_metric['metric_name'],
 294                     $config_type, $grouped_values, $test_metric['aggregator']);
 295             }
 296         }
 297     }
 298
 299     private function test_value_list_to_values_by_iterations($test_value_list, $test_metric, $aggregator) {
 300         $values_by_iterations = array();
 301         $group_sizes = array();
 302         $first_test = TRUE;
 303         foreach ($test_value_list as $test_name => $aggregators_and_values) {
 304             if (count($aggregators_and_values) == 1) // Either the subtest has exactly one aggregator or is raw value (not aggregated)
 305                 $values = $aggregators_and_values[0]['values'];
 306             else {
 307                 $values = NULL;
 308                 // Find the values of the subtest aggregated by the same aggregator.
 309                 foreach ($aggregators_and_values as $aggregator_and_values) {
 310                     if ($aggregator_and_values['aggregator'] == $aggregator) {
 311                         $values = $aggregator_and_values['values'];
 312                         break;
 313                     }
 314                 }
 315                 if (!$values) {
 316                     $this->exit_with_error('NoMatchingAggregatedValueInSubtest',
 317                         array('parent' => $test_metric['test_id'],
 318                         'metric' => $test_metric['metric_name'],
 319                         'childTest' => $test_name,
 320                         'aggregator' => $aggregator,
 321                         'aggregatorAndValues' => $aggregators_and_values));
 322                 }
 323             }
 324
 325             for ($group = 0; $group < count($values); ++$group) {
 326                 if ($first_test) {
 327                     array_push($group_sizes, count($values[$group]));
 328                     for ($i = 0; $i < count($values[$group]); ++$i)
 329                         array_push($values_by_iterations, array());
 330                 }
 331
 332                 if ($group_sizes[$group] != count($values[$group])) {
 333                     $this->exit_with_error('IterationGroupSizeIsInconsistent',
 334                         array('parent' => $test_metric['test_id'],
 335                         'metric' => $test_metric['metric_name'],
 336                         'childTest' => $test_name,
 337                         'groupSizes' => $group_sizes,
 338                         'group' => $group,
 339                         'values' => $values));
 340                 }
 341             }
 342             $first_test = FALSE;
 343
 344             if (count($values) != count($group_sizes)) {
 345                 // FIXME: We should support bootstrapping or just computing the mean in this case.
 346                 $this->exit_with_error('IterationGroupCountIsInconsistent', array('parent' => $test_metric['test_id'],
 347                     'metric' => $test_metric['metric_name'], 'childTest' => $test_name,
 348                     'valuesByIterations' => $values_by_iterations, 'values' => $values));
 349             }
 350
 351             $flattened_iteration_index = 0;
 352             for ($group = 0; $group < count($values); ++$group) {
 353                 for ($i = 0; $i < count($values[$group]); ++$i) {
 354                     $run_iteration_value = $values[$group][$i];
 355                     if (!is_numeric($run_iteration_value)) {
 356                         $this->exit_with_error('NonNumeralIterationValueForAggregation', array('parent' => $test_metric['test_id'],
 357                             'metric' => $test_metric['metric_name'], 'childTest' => $test_name,
 358                             'valuesByIterations' => $values_by_iterations, 'values' => $values, 'index' => $i));
 359                     }
 360                     array_push($values_by_iterations[$flattened_iteration_index], $run_iteration_value);
 361                     $flattened_iteration_index++;
 362                 }
 363             }
 364         }
 365
 366         if (!$values_by_iterations)
 367             $this->exit_with_error('NoIterationToAggregation', array('parent' => $test_metric['test_id'], 'metric' => $test_metric['metric_name']));
 368
 369         return array('values' => $values_by_iterations, 'group_sizes' => $group_sizes);
 370     }
 371
 372     static public $aggregators = array('Arithmetic', 'Geometric', 'Harmonic', 'Total');
 373
 374     private function aggregate_values($aggregator, $values) {
 375         switch ($aggregator) {
 376         case 'Arithmetic':
 377             return array_sum($values) / count($values);
 378         case 'Geometric':
 379             return exp(array_sum(array_map(function ($x) { return log($x); }, $values)) / count($values));
 380         case 'Harmonic':
 381             return count($values) / array_sum(array_map(function ($x) { return 1 / $x; }, $values));
 382         case 'Total':
 383             return array_sum($values);
 384         case 'SquareSum': # This aggregator is only used internally to compute run_square_sum_cache in test_runs table.
 385             return array_sum(array_map(function ($x) { return $x * $x; }, $values));
 386         default:
 387             $this->exit_with_error('UnknownAggregator', array('aggregator' => $aggregator));
 388         }
 389         return NULL;
 390     }
 391
 392     function compute_caches() {
 393         $expressions = array();
 394         $size = count($this->values_to_commit);
 395         for ($i = 0; $i < $size; ++$i) {
 396             $flattened_value = array();
 397             foreach ($this->values_to_commit[$i]['values'] as $group) {
 398                 for ($j = 0; $j < count($group); ++$j) {
 399                     $iteration_value = $group[$j];
 400                     if (gettype($iteration_value) === 'array') { // [relative time, value]
 401                         if (count($iteration_value) != 2) {
 402                             // FIXME: Also report test and metric.
 403                             $this->exit_with_error('InvalidIterationValueFormat', array('values' => $this->values_to_commit[$i]['values']));
 404                         }
 405                         $iteration_value = $iteration_value[1];
 406                     }
 407                     array_push($flattened_value, $iteration_value);
 408                 }
 409             }
 410             $this->values_to_commit[$i]['mean'] = $this->aggregate_values('Arithmetic', $flattened_value);
 411             $this->values_to_commit[$i]['sum'] = $this->aggregate_values('Total', $flattened_value);
 412             $this->values_to_commit[$i]['square_sum'] = $this->aggregate_values('SquareSum', $flattened_value);
 413         }
 414     }
 415
 416     function add_values_to_commit($metric_id, $config_type, $values) {
 417         array_push($this->values_to_commit, array('metric_id' => $metric_id, 'config_type' => $config_type, 'values' => $values));
 418     }
 419
 420     function commit($platform_id, $build_id) {
 421         $this->db->begin_transaction() or $this->exit_with_error('FailedToBeginTransaction');
 422
 423         foreach ($this->values_to_commit as $item) {
 424             $config_data = array('metric' => $item['metric_id'], 'type' => $item['config_type'], 'platform' => $platform_id);
 425             $config_id = $this->db->select_or_insert_row('test_configurations', 'config', $config_data);
 426             if (!$config_id)
 427                 $this->rollback_with_error('FailedToObtainConfiguration', $config_data);
 428
 429             $values = $item['values'];
 430             $total_count = 0;
 431             for ($group = 0; $group < count($values); ++$group)
 432                 $total_count += count($values[$group]);
 433             $run_data = array('config' => $config_id, 'build' => $build_id, 'iteration_count_cache' => $total_count,
 434                 'mean_cache' => $item['mean'], 'sum_cache' => $item['sum'], 'square_sum_cache' => $item['square_sum']);
 435             $run_id = $this->db->insert_row('test_runs', 'run', $run_data);
 436             if (!$run_id)
 437                 $this->rollback_with_error('FailedToInsertRun', array('metric' => $item['metric_id'], 'param' => $run_data));
 438
 439             $flattened_order = 0;
 440             for ($group = 0; $group < count($values); ++$group) {
 441                 for ($i = 0; $i < count($values[$group]); ++$i) {
 442                     $iteration_value = $values[$group][$i];
 443                     $relative_time = NULL;
 444                     if (gettype($iteration_value) === 'array') {
 445                         assert(count($iteration_value) == 2); // compute_caches checks this condition.
 446                         $relative_time = $iteration_value[0];
 447                         $iteration_value = $iteration_value[1];
 448                     }
 449                     $param = array('run' => $run_id, 'order' => $flattened_order, 'value' => $iteration_value,
 450                         'group' => count($values) == 1 ? NULL : $group, 'relative_time' => $relative_time);
 451                     $this->db->insert_row('run_iterations', 'iteration', $param, NULL)
 452                         or $this->rollback_with_error('FailedToInsertIteration', array('config' => $config_id, 'build' => $build_id, 'param' => $param));
 453                     $flattened_order++;
 454                 }
 455             }
 456         }
 457
 458         $this->db->query_and_get_affected_rows("UPDATE reports
 459             SET (report_committed_at, report_build) = (CURRENT_TIMESTAMP AT TIME ZONE 'UTC', $2)
 460             WHERE report_id = $1", array($this->report_id, $build_id));
 461
 462         $this->db->commit_transaction() or $this->exit_with_error('FailedToCommitTransaction');
 463     }
 464
 465     private function rollback_with_error($message, $details) {
 466         $this->db->rollback_transaction();
 467         $this->exit_with_error($message, $details);
 468     }
 469 };
 470
 471 ?>