2010-08-17 Victor Wang <victorw@chromium.org>
authorvictorw@chromium.org <victorw@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 17 Aug 2010 23:56:24 +0000 (23:56 +0000)
committervictorw@chromium.org <victorw@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 17 Aug 2010 23:56:24 +0000 (23:56 +0000)
        Reviewed by Ojan Vafai.

        Update json results generator to have incremental json including
        results for tests that pass in current run but failed before.

        https://bugs.webkit.org/show_bug.cgi?id=44119

        * Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py:
        * Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py:
        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@65574 268f45cc-cd09-0410-ab3c-d52691b4dbfc

WebKitTools/ChangeLog
WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py
WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py
WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

index 0bc80e5..3c8d878 100644 (file)
@@ -1,3 +1,16 @@
+2010-08-17  Victor Wang  <victorw@chromium.org>
+
+        Reviewed by Ojan Vafai.
+
+        Update json results generator to have incremental json including
+        results for tests that pass in current run but failed before.
+
+        https://bugs.webkit.org/show_bug.cgi?id=44119
+
+        * Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py:
+        * Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py:
+        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+
 2010-08-17  Dirk Pranke  <dpranke@chromium.org>
 
         Reviewed by Eric Seidel.
index 6c36c93..c6c3066 100644 (file)
@@ -57,7 +57,7 @@ class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGeneratorBase
     def __init__(self, port, builder_name, build_name, build_number,
         results_file_base_path, builder_base_url,
         test_timings, expectations, result_summary, all_tests,
-        generate_incremental_results=False):
+        generate_incremental_results=False, test_results_server=None):
         """Modifies the results.json file. Grabs it off the archive directory
         if it is not found locally.
 
@@ -68,7 +68,7 @@ class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGeneratorBase
         super(JSONLayoutResultsGenerator, self).__init__(
             builder_name, build_name, build_number, results_file_base_path,
             builder_base_url, {}, port.test_repository_paths(),
-            generate_incremental_results)
+            generate_incremental_results, test_results_server)
 
         self._port = port
         self._expectations = expectations
index e746bc0..8106ecb 100644 (file)
@@ -84,10 +84,14 @@ class JSONResultsGeneratorBase(object):
     RESULTS_FILENAME = "results.json"
     INCREMENTAL_RESULTS_FILENAME = "incremental_results.json"
 
+    URL_FOR_TEST_LIST_JSON = \
+        "http://%s/testfile?builder=%s&name=%s&testlistjson=1"
+
     def __init__(self, builder_name, build_name, build_number,
         results_file_base_path, builder_base_url,
         test_results_map, svn_repositories=None,
-        generate_incremental_results=False):
+        generate_incremental_results=False,
+        test_results_server=None):
         """Modifies the results.json file. Grabs it off the archive directory
         if it is not found locally.
 
@@ -103,6 +107,9 @@ class JSONResultsGeneratorBase(object):
           svn_repositories: A (json_field_name, svn_path) pair for SVN
               repositories that tests rely on.  The SVN revision will be
               included in the JSON with the given json_field_name.
+          generate_incremental_results: If true, generate incremental json file
+              from current run results.
+          test_results_server: server that hosts test results json.
         """
         self._builder_name = builder_name
         self._build_name = build_name
@@ -121,6 +128,8 @@ class JSONResultsGeneratorBase(object):
         if not self._svn_repositories:
             self._svn_repositories = {}
 
+        self._test_results_server = test_results_server
+
         self._json = None
         self._archived_results = None
 
@@ -144,25 +153,24 @@ class JSONResultsGeneratorBase(object):
 
     def get_json(self, incremental=False):
         """Gets the results for the results.json file."""
-        if incremental:
-            results_json = {}
-        else:
+        results_json = {}
+        if not incremental:
             if self._json:
                 return self._json
 
-            if not self._archived_results:
-                self._archived_results, error = \
-                    self._get_archived_json_results()
-                if error:
-                    # If there was an error don't write a results.json
-                    # file at all as it would lose all the information on the
-                    # bot.
-                    _log.error("Archive directory is inaccessible. Not "
-                               "modifying or clobbering the results.json "
-                               "file: " + str(error))
-                    return None
+            if self._archived_results:
+                results_json = self._archived_results
 
-            results_json = self._archived_results
+        if not results_json:
+            results_json, error = self._get_archived_json_results(incremental)
+            if error:
+                # If there was an error don't write a results.json
+                # file at all as it would lose all the information on the
+                # bot.
+                _log.error("Archive directory is inaccessible. Not "
+                           "modifying or clobbering the results.json "
+                           "file: " + str(error))
+                return None
 
         builder_name = self._builder_name
         if results_json and builder_name not in results_json:
@@ -186,7 +194,7 @@ class JSONResultsGeneratorBase(object):
         all_failing_tests = self._get_failed_test_names()
         all_failing_tests.update(tests.iterkeys())
         for test in all_failing_tests:
-            self._insert_test_time_and_result(test, tests)
+            self._insert_test_time_and_result(test, tests, incremental)
 
         return results_json
 
@@ -253,24 +261,40 @@ class JSONResultsGeneratorBase(object):
                 return ""
         return ""
 
-    def _get_archived_json_results(self):
+    def _get_archived_json_results(self, for_incremental=False):
         """Reads old results JSON file if it exists.
         Returns (archived_results, error) tuple where error is None if results
         were successfully read.
+
+        if for_incremental is True, download JSON file that only contains test
+        name list from test-results server. This is for generating incremental
+        JSON so the file generated has info for tests that failed before but
+        pass or are skipped from current run.
         """
         results_json = {}
         old_results = None
         error = None
 
-        if os.path.exists(self._results_file_path):
+        if os.path.exists(self._results_file_path) and not for_incremental:
             with codecs.open(self._results_file_path, "r", "utf-8") as file:
                 old_results = file.read()
-        elif self._builder_base_url:
-            # Check if we have the archived JSON file on the buildbot server.
-            results_file_url = (self._builder_base_url +
-                self._build_name + "/" + self.RESULTS_FILENAME)
-            _log.error("Local results.json file does not exist. Grabbing "
-                       "it off the archive at " + results_file_url)
+        elif self._builder_base_url or for_incremental:
+            if for_incremental:
+                if not self._test_results_server:
+                    # starting from fresh if no test results server specified.
+                    return {}, None
+
+                results_file_url = (self.URL_FOR_TEST_LIST_JSON %
+                    (self._test_results_server,
+                     self._builder_name,
+                     self.RESULTS_FILENAME))
+            else:
+                # Check if we have the archived JSON file on the buildbot
+                # server.
+                results_file_url = (self._builder_base_url +
+                    self._build_name + "/" + self.RESULTS_FILENAME)
+                _log.error("Local results.json file does not exist. Grabbing "
+                           "it off the archive at " + results_file_url)
 
             try:
                 results_file = urllib2.urlopen(results_file_url)
@@ -387,7 +411,7 @@ class JSONResultsGeneratorBase(object):
             int(time.time()),
             self.TIME)
 
-    def _insert_test_time_and_result(self, test_name, tests):
+    def _insert_test_time_and_result(self, test_name, tests, incremental=False):
         """ Insert a test item with its results to the given tests dictionary.
 
         Args:
@@ -401,9 +425,20 @@ class JSONResultsGeneratorBase(object):
             tests[test_name] = self._create_results_and_times_json()
 
         thisTest = tests[test_name]
-        self._insert_item_run_length_encoded(result, thisTest[self.RESULTS])
-        self._insert_item_run_length_encoded(time, thisTest[self.TIMES])
-        self._normalize_results_json(thisTest, test_name, tests)
+        if self.RESULTS in thisTest:
+            self._insert_item_run_length_encoded(result, thisTest[self.RESULTS])
+        else:
+            thisTest[self.RESULTS] = [[1, result]]
+
+        if self.TIMES in thisTest:
+            self._insert_item_run_length_encoded(time, thisTest[self.TIMES])
+        else:
+            thisTest[self.TIMES] = [[1, time]]
+
+        # Don't normalize the incremental results json because we need results
+        # for tests that pass or have no data from current run.
+        if not incremental:
+            self._normalize_results_json(thisTest, test_name, tests)
 
     def _convert_json_to_current_version(self, results_json):
         """If the JSON does not match the current version, converts it to the
index cb453f0..9f1750e 100755 (executable)
@@ -879,7 +879,8 @@ class TestRunner:
             self._options.build_number, self._options.results_directory,
             BUILDER_BASE_URL, individual_test_timings,
             self._expectations, result_summary, self._test_files_list,
-            not self._options.upload_full_results)
+            not self._options.upload_full_results,
+            self._options.test_results_server)
 
         _log.debug("Finished writing JSON files.")