ManifestGenerator shouldn't need more than 1GB of memory or run for 30 seconds
[WebKit-https.git] / Websites / perf.webkit.org / public / include / manifest-generator.php
index 3a71edf..a641508 100644 (file)
@@ -9,6 +9,7 @@ class ManifestGenerator {
 
     function __construct($db) {
         $this->db = $db;
+        $this->elapsed_time = NULL;
     }
 
     function generate() {
@@ -28,7 +29,6 @@ class ManifestGenerator {
         $tests = (object)$this->tests();
         $metrics = (object)$this->metrics();
         $platforms = (object)$this->platforms($platform_table, false);
-        $dashboard = (object)$this->platforms($platform_table, true);
         $repositories = (object)$this->repositories($repositories_table, $repositories_with_commit);
 
         $this->manifest = array(
@@ -36,7 +36,7 @@ class ManifestGenerator {
             'tests' => &$tests,
             'metrics' => &$metrics,
             'all' => &$platforms,
-            'dashboard' => &$dashboard,
+            'dashboard' => (object)array(), // Only used by v1 UI.
             'repositories' => &$repositories,
             'builders' => (object)$this->builders(),
             'bugTrackers' => (object)$this->bug_trackers($repositories_table),
@@ -44,9 +44,10 @@ class ManifestGenerator {
             'dashboards' => (object)config('dashboards'),
             'summaryPages' => config('summaryPages'),
             'fileUploadSizeLimit' => config('uploadFileLimitInMB', 0) * 1024 * 1024,
+            'testAgeToleranceInHours' => config('testAgeToleranceInHours'),
         );
 
-        $this->manifest['elapsedTime'] = (microtime(true) - $start_time) * 1000;
+        $this->elapsed_time = (microtime(true) - $start_time) * 1000;
 
         return TRUE;
     }
@@ -54,7 +55,7 @@ class ManifestGenerator {
     function manifest() { return $this->manifest; }
 
     function store() {
-        return generate_data_file('manifest.json', json_encode($this->manifest));
+        return generate_json_data_with_elapsed_time_if_needed('manifest.json', $this->manifest, $this->elapsed_time);
     }
 
     private function tests() {
@@ -62,11 +63,11 @@ class ManifestGenerator {
         $tests_table = $this->db->fetch_table('tests');
         if (!$tests_table)
             return $tests;
-        foreach ($tests_table as $test_row) {
+        foreach ($tests_table as &$test_row) {
             $tests[$test_row['test_id']] = array(
                 'name' => $test_row['test_name'],
                 'url' => $test_row['test_url'],
-                'parentId' => $test_row['test_parent'],
+                'parentId' => $test_row['test_parent'] ? intval($test_row['test_parent']) : NULL,
             );
         }
         return $tests;
@@ -77,53 +78,64 @@ class ManifestGenerator {
         $metrics_table = $this->db->query_and_fetch_all('SELECT * FROM test_metrics LEFT JOIN aggregators ON metric_aggregator = aggregator_id');
         if (!$metrics_table)
             return $metrics;
-        foreach ($metrics_table as $row) {
+        foreach ($metrics_table as &$row) {
             $metrics[$row['metric_id']] = array(
                 'name' => $row['metric_name'],
-                'test' => $row['metric_test'],
+                'test' => intval($row['metric_test']),
                 'aggregator' => $row['aggregator_name']);
         }
         return $metrics;
     }
 
-    private function platforms($platform_table, $is_dashboard) {
-        $metrics = $this->db->query_and_fetch_all('SELECT config_metric AS metric_id, config_platform AS platform_id,
-            extract(epoch from max(config_runs_last_modified) at time zone \'utc\') * 1000 AS last_modified, bool_or(config_is_in_dashboard) AS in_dashboard
-            FROM test_configurations GROUP BY config_metric, config_platform ORDER BY config_platform');
+    private function platforms(&$platform_table, $is_dashboard) {
+        $config_query = $this->db->query('SELECT config_platform, config_metric,
+            extract(epoch from config_runs_last_modified at time zone \'utc\') * 1000 AS last_modified
+            FROM test_configurations');
 
         $platform_metrics = array();
 
-        if ($metrics) {
+        if ($config_query) {
             $current_platform_entry = null;
-            foreach ($metrics as $metric_row) {
-                if ($is_dashboard && !Database::is_true($metric_row['in_dashboard']))
+            $last_modified_map = array();
+            while (1) {
+                $config_row = $this->db->fetch_next_row($config_query);
+                if (!$config_row)
+                    break;
+
+                $platform_id = $config_row['config_platform'];
+                $metric_id = $config_row['config_metric'];
+                $last_modified = intval($config_row['last_modified']);
+
+                $key = $platform_id . '-' . $metric_id;
+                if (array_key_exists($key, $last_modified_map)) {
+                    $last_modified_map[$key] = max($last_modified_map[$key], $last_modified);
                     continue;
-
-                $platform_id = $metric_row['platform_id'];
-                if (!$current_platform_entry || $current_platform_entry['id'] != $platform_id) {
-                    $current_platform_entry = &array_ensure_item_has_array($platform_metrics, $platform_id);
-                    $current_platform_entry['id'] = $platform_id;
-                    array_ensure_item_has_array($current_platform_entry, 'metrics');
-                    array_ensure_item_has_array($current_platform_entry, 'last_modified');
                 }
+                $last_modified_map[$key] = $last_modified;
 
-                array_push($current_platform_entry['metrics'], $metric_row['metric_id']);
-                array_push($current_platform_entry['last_modified'], intval($metric_row['last_modified']));
+                $current_platform_entry = &array_ensure_item_has_array($platform_metrics, $platform_id);
+                array_ensure_item_has_array($current_platform_entry, 'metrics');
+                array_push($current_platform_entry['metrics'], intval($metric_id));
             }
         }
         $configurations = array();
 
         $platforms = array();
         if ($platform_table) {
-            foreach ($platform_table as $platform_row) {
+            foreach ($platform_table as &$platform_row) {
                 if (Database::is_true($platform_row['platform_hidden']))
                     continue;
                 $id = $platform_row['platform_id'];
                 if (array_key_exists($id, $platform_metrics)) {
+                    $metrics = &$platform_metrics[$id]['metrics'];
+                    $last_modified = array();
+                    foreach ($metrics as $metric_id)
+                        array_push($last_modified, $last_modified_map[$id . '-' . $metric_id]);
                     $platforms[$id] = array(
-                        'name' => $platform_row['platform_name'],
-                        'metrics' => $platform_metrics[$id]['metrics'],
-                        'lastModified' => $platform_metrics[$id]['last_modified']);
+                        'name' => &$platform_row['platform_name'],
+                        'metrics' => &$metrics,
+                        'lastModified' => &$last_modified
+                    );
                 }
             }
         }
@@ -134,7 +146,7 @@ class ManifestGenerator {
         $repositories = array();
         if (!$repositories_table)
             return $repositories;
-        foreach ($repositories_table as $row) {
+        foreach ($repositories_table as &$row) {
             $repositories[$row['repository_id']] = array(
                 'name' => $row['repository_name'],
                 'url' => $row['repository_url'],
@@ -151,7 +163,7 @@ class ManifestGenerator {
         if (!$builders_table)
             return array();
         $builders = array();
-        foreach ($builders_table as $row)
+        foreach ($builders_table as &$row)
             $builders[$row['builder_id']] = array('name' => $row['builder_name'], 'buildUrl' => $row['builder_build_url']);
 
         return $builders;
@@ -170,7 +182,7 @@ class ManifestGenerator {
         $bug_trackers = array();
         $bug_trackers_table = $this->db->fetch_table('bug_trackers');
         if ($bug_trackers_table) {
-            foreach ($bug_trackers_table as $row) {
+            foreach ($bug_trackers_table as &$row) {
                 $bug_trackers[$row['tracker_id']] = array(
                     'name' => $row['tracker_name'],
                     'bugUrl' => $row['tracker_bug_url'],
@@ -189,33 +201,69 @@ class ManifestGenerator {
 
     static function fetch_triggerables($db, $query)
     {
-        $triggerables = $db->fetch_table('build_triggerables');
+        $triggerables = $db->select_rows('build_triggerables', 'triggerable', $query);
         if (!$triggerables)
             return array();
 
         $id_to_triggerable = array();
-        foreach ($triggerables as $row) {
+        $triggerable_id_to_repository_set = array();
+        foreach ($triggerables as &$row) {
             $id = $row['triggerable_id'];
             $id_to_triggerable[$id] = array(
-                'id' => $id,
                 'name' => $row['triggerable_name'],
+                'isDisabled' => Database::is_true($row['triggerable_disabled']),
                 'acceptedRepositories' => array(),
+                'repositoryGroups' => array(),
                 'configurations' => array());
+            $triggerable_id_to_repository_set[$id] = array();
         }
 
-        $repository_map = $db->fetch_table('triggerable_repositories');
-        if ($repository_map) {
-            foreach ($repository_map as $row) {
-                $triggerable = &$id_to_triggerable[$row['trigrepo_triggerable']];
-                array_push($triggerable['acceptedRepositories'], $row['trigrepo_repository']);
+        $repository_groups = $db->fetch_table('triggerable_repository_groups', 'repositorygroup_name');
+        $group_repositories = $db->fetch_table('triggerable_repositories');
+        if ($repository_groups && $group_repositories) {
+            $repository_set_by_group = array();
+            foreach ($group_repositories as &$repository_row) {
+                $group_id = $repository_row['trigrepo_group'];
+                array_ensure_item_has_array($repository_set_by_group, $group_id);
+                array_push($repository_set_by_group[$group_id], array(
+                    'repository' => intval($repository_row['trigrepo_repository']),
+                    'acceptsPatch' => Database::is_true($repository_row['trigrepo_accepts_patch'])));
+            }
+            foreach ($repository_groups as &$group_row) {
+                $triggerable_id = $group_row['repositorygroup_triggerable'];
+                if (!array_key_exists($triggerable_id, $id_to_triggerable))
+                    continue;
+                $triggerable = &$id_to_triggerable[$triggerable_id];
+                $group_id = $group_row['repositorygroup_id'];
+                $repository_list = array_get($repository_set_by_group, $group_id, array());
+                array_push($triggerable['repositoryGroups'], array(
+                    'id' => intval($group_row['repositorygroup_id']),
+                    'name' => $group_row['repositorygroup_name'],
+                    'description' => $group_row['repositorygroup_description'],
+                    'hidden' => Database::is_true($group_row['repositorygroup_hidden']),
+                    'acceptsCustomRoots' => Database::is_true($group_row['repositorygroup_accepts_roots']),
+                    'repositories' => $repository_list));
+                // V2 UI compatibility.
+                foreach ($repository_list as &$repository_data) {
+                    $repository_id = $repository_data['repository'];
+                    $set = &$triggerable_id_to_repository_set[$triggerable_id];
+                    if (array_key_exists($repository_id, $set))
+                        continue;
+                    $set[$repository_id] = true;
+                    array_push($triggerable['acceptedRepositories'], $repository_id);
+                }
+
             }
         }
 
         $configuration_map = $db->fetch_table('triggerable_configurations');
         if ($configuration_map) {
-            foreach ($configuration_map as $row) {
-                $triggerable = &$id_to_triggerable[$row['trigconfig_triggerable']];
-                array_push($triggerable['configurations'], array($row['trigconfig_test'], $row['trigconfig_platform']));
+            foreach ($configuration_map as &$row) {
+                $triggerable_id = $row['trigconfig_triggerable'];
+                if (!array_key_exists($triggerable_id, $id_to_triggerable))
+                    continue;
+                $triggerable = &$id_to_triggerable[$triggerable_id];
+                array_push($triggerable['configurations'], array(intval($row['trigconfig_test']), intval($row['trigconfig_platform'])));
             }
         }