2010-11-24 Ojan Vafai <ojan@chromium.org>
authorojan@chromium.org <ojan@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 24 Nov 2010 22:28:00 +0000 (22:28 +0000)
committerojan@chromium.org <ojan@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 24 Nov 2010 22:28:00 +0000 (22:28 +0000)
        Reviewed by Tony Chang.

        [chromium] improve memory usage for test results server and store fewer runs
        https://bugs.webkit.org/show_bug.cgi?id=50035

        Store fewer runs and don't pass full files around when we only need bools.
        Greatly improves error rates on test-results.appspot.com.

        * Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py:
        * TestResultServer/handlers/testfilehandler.py:
        * TestResultServer/model/dashboardfile.py:
        * TestResultServer/model/jsonresults.py:

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@72699 268f45cc-cd09-0410-ab3c-d52691b4dbfc

WebKitTools/ChangeLog
WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py
WebKitTools/TestResultServer/handlers/testfilehandler.py
WebKitTools/TestResultServer/model/dashboardfile.py
WebKitTools/TestResultServer/model/jsonresults.py

index 7a89f33c99067d2ffd81fc48e259886601bb3ad0..1eb7b3559fe2c5241454ed7e788e1bd290a02bb3 100644 (file)
@@ -1,3 +1,18 @@
+2010-11-24  Ojan Vafai  <ojan@chromium.org>
+
+        Reviewed by Tony Chang.
+
+        [chromium] improve memory usage for test results server and store fewer runs
+        https://bugs.webkit.org/show_bug.cgi?id=50035
+
+        Store fewer runs and don't pass full files around when we only need bools.
+        Greatly improves error rates on test-results.appspot.com.
+
+        * Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py:
+        * TestResultServer/handlers/testfilehandler.py:
+        * TestResultServer/model/dashboardfile.py:
+        * TestResultServer/model/jsonresults.py:
+
 2010-11-24  Dirk Pranke  <dpranke@chromium.org>
 
         Reviewed by Tony Chang.
index 3484fcd4c5c685b7911b4b386c370b7fc2e60421..331e33074eb54f7de0366c7e13a5ebb7cd2a2139 100644 (file)
@@ -80,7 +80,7 @@ class TestResult(object):
 class JSONResultsGeneratorBase(object):
     """A JSON results generator for generic tests."""
 
-    MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 1500
+    MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 750
     # Min time (seconds) that will be added to the JSON.
     MIN_TIME = 1
     JSON_PREFIX = "ADD_RESULTS("
index d81789017d3abb72ea608d347e9618df49548480..1e3120613cc5ce6cab1fec54e304a5f0eb3a5bdc 100644 (file)
@@ -198,12 +198,12 @@ class Upload(webapp.RequestHandler):
             if ((incremental and filename == "results.json") or
                 (filename == "incremental_results.json")):
                 # Merge incremental json results.
-                saved_file = JsonResults.update(master, builder, test_type, file.value)
+                update_succeeded = JsonResults.update(master, builder, test_type, file.value)
             else:
-                saved_file = TestFile.update(
+                update_succeeded = TestFile.update(
                     master, builder, test_type, file.filename, file.value)
 
-            if not saved_file:
+            if not update_succeeded:
                 errors.append(
                     "Upload failed, master: %s, builder: %s, test_type: %s, name: %s." %
                     (master, builder, test_type, file.filename))
index 57d3f6fbb15fe567ac7e7449614d510b1dafc9ae..aad6d50947e31e25230647debae7d461d5e3940e 100644 (file)
@@ -85,14 +85,16 @@ class DashboardFile(db.Model):
     def update_file(cls, name):
         data = cls.grab_file_from_svn(name)
         if not data:
-            return None
+            return False
 
         logging.info("Got file from SVN.")
 
         files = cls.get_files(name)
         if not files:
             logging.info("No existing file, added as new file.")
-            return cls.add_file(name, data)
+            if cls.add_file(name, data):
+                return True
+            return False
 
         logging.debug("Updating existing file.")
         file = files[0]
@@ -102,7 +104,7 @@ class DashboardFile(db.Model):
 
         logging.info("Dashboard file replaced, name: %s.", name)
 
-        return file
+        return True
 
     @classmethod
     def delete_file(cls, name):
index 97b277afb5ce05e660e57faec4d553f64affd1cb..f5a0fdedf5e2b1a5b9aba572b602022d84b025eb 100755 (executable)
@@ -45,7 +45,7 @@ JSON_RESULTS_PASS = "P"
 JSON_RESULTS_NO_DATA = "N"
 JSON_RESULTS_MIN_TIME = 1
 JSON_RESULTS_VERSION = 3
-JSON_RESULTS_MAX_BUILDS = 1500
+JSON_RESULTS_MAX_BUILDS = 750
 JSON_RESULTS_MAX_BUILDS_SMALL = 200
 
 
@@ -406,12 +406,10 @@ class JsonResults(object):
             Large TestFile object if update succeeds or
             None on failure.
         """
-        small_file = cls.update_file(master, builder, test_type, incremental, JSON_RESULTS_FILE_SMALL, JSON_RESULTS_MAX_BUILDS_SMALL)
-        large_file = cls.update_file(master, builder, test_type, incremental, JSON_RESULTS_FILE, JSON_RESULTS_MAX_BUILDS)
+        small_file_updated = cls.update_file(master, builder, test_type, incremental, JSON_RESULTS_FILE_SMALL, JSON_RESULTS_MAX_BUILDS_SMALL)
+        large_file_updated = cls.update_file(master, builder, test_type, incremental, JSON_RESULTS_FILE, JSON_RESULTS_MAX_BUILDS)
 
-        if small_file and large_file:
-            return large_file
-        return None
+        return small_file_updated and large_file_updated
 
     @classmethod
     def update_file(cls, master, builder, test_type, incremental, filename, num_runs):
@@ -433,9 +431,9 @@ class JsonResults(object):
             logging.info(
                 "Update failed, master: %s, builder: %s, test_type: %s, name: %s." %
                 (master, builder, test_type, filename))
-            return None
+            return False
 
-        return file
+        return True
 
     @classmethod
     def get_test_list(cls, builder, json_file_data):