testlistjson on the test results server doesn't understand hierarchical results format
authorojan@chromium.org <ojan@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 28 Nov 2011 22:00:45 +0000 (22:00 +0000)
committerojan@chromium.org <ojan@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 28 Nov 2011 22:00:45 +0000 (22:00 +0000)
https://bugs.webkit.org/show_bug.cgi?id=73246

Reviewed by Tony Chang.

* TestResultServer/model/jsonresults.py:
(JsonResults._delete_results_and_times):
Instead of just getting the top-level keys, we now walk the tests tree and
delete the results and times values.

* TestResultServer/model/jsonresults_unittest.py:
Simplify the test harness to take in the same JSON format for the tests
that the actual results.json uses instead of something very similar but
needlessly different. This also allows for testing the broken case of
results and times values being at non-leaf level.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@101280 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/TestResultServer/model/jsonresults.py
Tools/TestResultServer/model/jsonresults_unittest.py

index 21cf86b..8d1e266 100644 (file)
@@ -1,3 +1,21 @@
+2011-11-28  Ojan Vafai  <ojan@chromium.org>
+
+        testlistjson on the test results server doesn't understand hierarchical results format
+        https://bugs.webkit.org/show_bug.cgi?id=73246
+
+        Reviewed by Tony Chang.
+
+        * TestResultServer/model/jsonresults.py:
+        (JsonResults._delete_results_and_times):
+        Instead of just getting the top-level keys, we now walk the tests tree and
+        delete the results and times values.
+
+        * TestResultServer/model/jsonresults_unittest.py:
+        Simplify the test harness to take in the same JSON format for the tests
+        that the actual results.json uses instead of something very similar but
+        needlessly different. This also allows for testing the broken case of
+        results and times values being at non-leaf level.
+
 2011-11-28  Tony Chang  <tony@chromium.org>
 
         ews bots should pass --force to update-webkit-chromium
 2011-11-28  Tony Chang  <tony@chromium.org>
 
         ews bots should pass --force to update-webkit-chromium
index d5dbd7e..d9f2e5a 100755 (executable)
@@ -313,6 +313,14 @@ class JsonResults(object):
         return True
 
     @classmethod
         return True
 
     @classmethod
+    def _delete_results_and_times(cls, tests):
+        for key in tests.keys():
+            if key in (JSON_RESULTS_RESULTS, JSON_RESULTS_TIMES):
+                del tests[key]
+            else:
+                cls._delete_results_and_times(tests[key])
+
+    @classmethod
     def get_test_list(cls, builder, json_file_data):
         logging.debug("Loading test results json...")
         json = cls._load_json(json_file_data)
     def get_test_list(cls, builder, json_file_data):
         logging.debug("Loading test results json...")
         json = cls._load_json(json_file_data)
@@ -325,6 +333,6 @@ class JsonResults(object):
 
         test_list_json = {}
         tests = json[builder][JSON_RESULTS_TESTS]
 
         test_list_json = {}
         tests = json[builder][JSON_RESULTS_TESTS]
-        test_list_json[builder] = {"tests": dict.fromkeys(tests, {})}
-
+        cls._delete_results_and_times(tests)
+        test_list_json[builder] = {"tests": tests}
         return cls._generate_file_data(test_list_json)
         return cls._generate_file_data(test_list_json)
index 99e709a..2bfff2a 100755 (executable)
@@ -33,6 +33,8 @@ except ImportError:
     print "ERROR: Add the TestResultServer, google_appengine and yaml/lib directories to your PYTHONPATH"
     raise
 
     print "ERROR: Add the TestResultServer, google_appengine and yaml/lib directories to your PYTHONPATH"
     raise
 
+from django.utils import simplejson
+
 import unittest
 
 
 import unittest
 
 
@@ -63,10 +65,10 @@ JSON_RESULTS_COUNTS_TEMPLATE = (
     '"X":[TESTDATA],'
     '"Z":[TESTDATA]}')
 
     '"X":[TESTDATA],'
     '"Z":[TESTDATA]}')
 
-JSON_RESULTS_DIRECTORY_TEMPLATE = '"[TESTDATA_DIRECTORY]":{[TESTDATA_DATA]}'
+JSON_RESULTS_DIRECTORY_TEMPLATE = '[[TESTDATA_DIRECTORY]]:{[TESTDATA_DATA]}'
 
 JSON_RESULTS_TESTS_TEMPLATE = (
 
 JSON_RESULTS_TESTS_TEMPLATE = (
-    '"[TESTDATA_TEST_NAME]":{'
+    '[[TESTDATA_TEST_NAME]]:{'
     '"results":[[TESTDATA_TEST_RESULTS]],'
     '"times":[[TESTDATA_TEST_TIMES]]}')
 
     '"results":[[TESTDATA_TEST_RESULTS]],'
     '"times":[[TESTDATA_TEST_TIMES]]}')
 
@@ -115,29 +117,9 @@ class JsonResultsTest(unittest.TestCase):
 
         version = str(test_data["version"]) if "version" in test_data else "4"
         json = json.replace("[VERSION]", version)
 
         version = str(test_data["version"]) if "version" in test_data else "4"
         json = json.replace("[VERSION]", version)
-
-        json_tests = []
-        for (name, test) in sorted(tests.iteritems()):
-            json_tests.append(self._parse_tests_dict(name, test))
-
-        json = json.replace("[TESTDATA_TESTS]", ",".join(json_tests))
-
+        json = json.replace("{[TESTDATA_TESTS]}", simplejson.dumps(tests, separators=(',', ':'), sort_keys=True))
         return json
 
         return json
 
-    def _parse_tests_dict(self, name, test):
-        if "results" in test:
-            test_results = JSON_RESULTS_TESTS_TEMPLATE.replace("[TESTDATA_TEST_NAME]", name)
-            test_results = test_results.replace("[TESTDATA_TEST_RESULTS]", test["results"])
-            test_results = test_results.replace("[TESTDATA_TEST_TIMES]", test["times"])
-            return test_results
-
-        test_results = JSON_RESULTS_DIRECTORY_TEMPLATE.replace("[TESTDATA_DIRECTORY]", name)
-        testdata = []
-        for (child_name, child_test) in sorted(test.iteritems()):
-            testdata.append(self._parse_tests_dict(child_name, child_test))
-        test_results = test_results.replace("[TESTDATA_DATA]", ",".join(testdata))
-        return test_results
-
     def _test_merge(self, aggregated_data, incremental_data, expected_data, max_builds=jsonresults.JSON_RESULTS_MAX_BUILDS):
         aggregated_results = self._make_test_json(aggregated_data)
         incremental_results = self._make_test_json(incremental_data)
     def _test_merge(self, aggregated_data, incremental_data, expected_data, max_builds=jsonresults.JSON_RESULTS_MAX_BUILDS):
         aggregated_results = self._make_test_json(aggregated_data)
         incremental_results = self._make_test_json(incremental_data)
@@ -151,12 +133,7 @@ class JsonResultsTest(unittest.TestCase):
 
     def _test_get_test_list(self, input_data, expected_data):
         input_results = self._make_test_json(input_data)
 
     def _test_get_test_list(self, input_data, expected_data):
         input_results = self._make_test_json(input_data)
-
-        json_tests = []
-        for test in expected_data:
-            json_tests.append("\"" + test + "\":{}")
-
-        expected_results = JSON_RESULTS_TEST_LIST_TEMPLATE.replace("[TESTDATA_TESTS]", ",".join(json_tests))
+        expected_results = JSON_RESULTS_TEST_LIST_TEMPLATE.replace("{[TESTDATA_TESTS]}", simplejson.dumps(expected_data, separators=(',', ':')))
         actual_results = JsonResults.get_test_list(self._builder, input_results)
         self.assertEquals(actual_results, expected_results)
 
         actual_results = JsonResults.get_test_list(self._builder, input_results)
         self.assertEquals(actual_results, expected_results)
 
@@ -167,8 +144,8 @@ class JsonResultsTest(unittest.TestCase):
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[200,\"F\"]",
-                           "times": "[200,0]"}}},
+                           "results": [[200,"F"]],
+                           "times": [[200,0]]}}},
             # Incremental results
             None,
             # Expect no merge happens.
             # Incremental results
             None,
             # Expect no merge happens.
@@ -181,8 +158,8 @@ class JsonResultsTest(unittest.TestCase):
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[200,\"F\"]",
-                           "times": "[200,0]"}}},
+                           "results": [[200,"F"]],
+                           "times": [[200,0]]}}},
             # Incremental results
             {"builds": [],
              "tests": {}},
             # Incremental results
             {"builds": [],
              "tests": {}},
@@ -199,13 +176,13 @@ class JsonResultsTest(unittest.TestCase):
 
             {"builds": ["2", "1"],
              "tests": {"001.html": {
 
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[200,\"F\"]",
-                           "times": "[200,0]"}}},
+                           "results": [[200,"F"]],
+                           "times": [[200,0]]}}},
             # Expected result
             {"builds": ["2", "1"],
              "tests": {"001.html": {
             # Expected result
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[200,\"F\"]",
-                           "times": "[200,0]"}}})
+                           "results": [[200,"F"]],
+                           "times": [[200,0]]}}})
 
     def test_merge_incremental_single_test_single_run_same_result(self):
         # Incremental results has the latest build and same test results for
 
     def test_merge_incremental_single_test_single_run_same_result(self):
         # Incremental results has the latest build and same test results for
@@ -216,18 +193,18 @@ class JsonResultsTest(unittest.TestCase):
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[200,\"F\"]",
-                           "times": "[200,0]"}}},
+                           "results": [[200,"F"]],
+                           "times": [[200,0]]}}},
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
-                           "results": "[1,\"F\"]",
-                           "times": "[1,0]"}}},
+                           "results": [[1,"F"]],
+                           "times": [[1,0]]}}},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
-                           "results": "[201,\"F\"]",
-                           "times": "[201,0]"}}})
+                           "results": [[201,"F"]],
+                           "times": [[201,0]]}}})
 
     def test_merge_single_test_single_run_different_result(self):
         # Incremental results has the latest build but different test results
 
     def test_merge_single_test_single_run_different_result(self):
         # Incremental results has the latest build but different test results
@@ -237,18 +214,18 @@ class JsonResultsTest(unittest.TestCase):
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[200,\"F\"]",
-                           "times": "[200,0]"}}},
+                           "results": [[200,"F"]],
+                           "times": [[200,0]]}}},
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
-                           "results": "[1, \"I\"]",
-                           "times": "[1,1]"}}},
+                           "results": [[1, "I"]],
+                           "times": [[1,1]]}}},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
-                           "results": "[1,\"I\"],[200,\"F\"]",
-                           "times": "[1,1],[200,0]"}}})
+                           "results": [[1,"I"],[200,"F"]],
+                           "times": [[1,1],[200,0]]}}})
 
     def test_merge_single_test_single_run_result_changed(self):
         # Incremental results has the latest build but results which differ from
 
     def test_merge_single_test_single_run_result_changed(self):
         # Incremental results has the latest build but results which differ from
@@ -257,18 +234,18 @@ class JsonResultsTest(unittest.TestCase):
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[200,\"F\"],[10,\"I\"]",
-                           "times": "[200,0],[10,1]"}}},
+                           "results": [[200,"F"],[10,"I"]],
+                           "times": [[200,0],[10,1]]}}},
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
-                           "results": "[1,\"I\"]",
-                           "times": "[1,1]"}}},
+                           "results": [[1,"I"]],
+                           "times": [[1,1]]}}},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
-                           "results": "[1,\"I\"],[200,\"F\"],[10,\"I\"]",
-                           "times": "[1,1],[200,0],[10,1]"}}})
+                           "results": [[1,"I"],[200,"F"],[10,"I"]],
+                           "times": [[1,1],[200,0],[10,1]]}}})
 
     def test_merge_multiple_tests_single_run(self):
         # All tests have incremental updates.
 
     def test_merge_multiple_tests_single_run(self):
         # All tests have incremental updates.
@@ -276,96 +253,96 @@ class JsonResultsTest(unittest.TestCase):
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[200,\"F\"]",
-                           "times": "[200,0]"},
+                           "results": [[200,"F"]],
+                           "times": [[200,0]]},
                        "002.html": {
                        "002.html": {
-                           "results": "[100,\"I\"]",
-                           "times": "[100,1]"}}},
+                           "results": [[100,"I"]],
+                           "times": [[100,1]]}}},
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
-                           "results": "[1,\"F\"]",
-                           "times": "[1,0]"},
+                           "results": [[1,"F"]],
+                           "times": [[1,0]]},
                        "002.html": {
                        "002.html": {
-                           "results": "[1,\"I\"]",
-                           "times": "[1,1]"}}},
+                           "results": [[1,"I"]],
+                           "times": [[1,1]]}}},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
-                           "results": "[201,\"F\"]",
-                           "times": "[201,0]"},
+                           "results": [[201,"F"]],
+                           "times": [[201,0]]},
                        "002.html": {
                        "002.html": {
-                           "results": "[101,\"I\"]",
-                           "times": "[101,1]"}}})
+                           "results": [[101,"I"]],
+                           "times": [[101,1]]}}})
 
     def test_merge_multiple_tests_single_run_one_no_result(self):
         self._test_merge(
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
 
     def test_merge_multiple_tests_single_run_one_no_result(self):
         self._test_merge(
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[200,\"F\"]",
-                           "times": "[200,0]"},
+                           "results": [[200,"F"]],
+                           "times": [[200,0]]},
                        "002.html": {
                        "002.html": {
-                           "results": "[100,\"I\"]",
-                           "times": "[100,1]"}}},
+                           "results": [[100,"I"]],
+                           "times": [[100,1]]}}},
             # Incremental results
             {"builds": ["3"],
              "tests": {"002.html": {
             # Incremental results
             {"builds": ["3"],
              "tests": {"002.html": {
-                           "results": "[1,\"I\"]",
-                           "times": "[1,1]"}}},
+                           "results": [[1,"I"]],
+                           "times": [[1,1]]}}},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
-                           "results": "[1,\"N\"],[200,\"F\"]",
-                           "times": "[201,0]"},
+                           "results": [[1,"N"],[200,"F"]],
+                           "times": [[201,0]]},
                        "002.html": {
                        "002.html": {
-                           "results": "[101,\"I\"]",
-                           "times": "[101,1]"}}})
+                           "results": [[101,"I"]],
+                           "times": [[101,1]]}}})
 
     def test_merge_single_test_multiple_runs(self):
         self._test_merge(
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
 
     def test_merge_single_test_multiple_runs(self):
         self._test_merge(
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[200,\"F\"]",
-                           "times": "[200,0]"}}},
+                           "results": [[200,"F"]],
+                           "times": [[200,0]]}}},
             # Incremental results
             {"builds": ["4", "3"],
              "tests": {"001.html": {
             # Incremental results
             {"builds": ["4", "3"],
              "tests": {"001.html": {
-                           "results": "[2, \"I\"]",
-                           "times": "[2,2]"}}},
+                           "results": [[2, "I"]],
+                           "times": [[2,2]]}}},
             # Expected results
             {"builds": ["4", "3", "2", "1"],
              "tests": {"001.html": {
             # Expected results
             {"builds": ["4", "3", "2", "1"],
              "tests": {"001.html": {
-                           "results": "[2,\"I\"],[200,\"F\"]",
-                           "times": "[2,2],[200,0]"}}})
+                           "results": [[2,"I"],[200,"F"]],
+                           "times": [[2,2],[200,0]]}}})
 
     def test_merge_multiple_tests_multiple_runs(self):
         self._test_merge(
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
 
     def test_merge_multiple_tests_multiple_runs(self):
         self._test_merge(
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[200,\"F\"]",
-                           "times": "[200,0]"},
+                           "results": [[200,"F"]],
+                           "times": [[200,0]]},
                        "002.html": {
                        "002.html": {
-                           "results": "[10,\"Z\"]",
-                           "times": "[10,0]"}}},
+                           "results": [[10,"Z"]],
+                           "times": [[10,0]]}}},
             # Incremental results
             {"builds": ["4", "3"],
              "tests": {"001.html": {
             # Incremental results
             {"builds": ["4", "3"],
              "tests": {"001.html": {
-                           "results": "[2, \"I\"]",
-                           "times": "[2,2]"},
+                           "results": [[2, "I"]],
+                           "times": [[2,2]]},
                        "002.html": {
                        "002.html": {
-                           "results": "[1,\"C\"]",
-                           "times": "[1,1]"}}},
+                           "results": [[1,"C"]],
+                           "times": [[1,1]]}}},
             # Expected results
             {"builds": ["4", "3", "2", "1"],
              "tests": {"001.html": {
             # Expected results
             {"builds": ["4", "3", "2", "1"],
              "tests": {"001.html": {
-                           "results": "[2,\"I\"],[200,\"F\"]",
-                           "times": "[2,2],[200,0]"},
+                           "results": [[2,"I"],[200,"F"]],
+                           "times": [[2,2],[200,0]]},
                        "002.html": {
                        "002.html": {
-                           "results": "[1,\"C\"],[10,\"Z\"]",
-                           "times": "[1,1],[10,0]"}}})
+                           "results": [[1,"C"],[10,"Z"]],
+                           "times": [[1,1],[10,0]]}}})
 
     def test_merge_incremental_result_older_build(self):
         # Test the build in incremental results is older than the most recent
 
     def test_merge_incremental_result_older_build(self):
         # Test the build in incremental results is older than the most recent
@@ -374,18 +351,18 @@ class JsonResultsTest(unittest.TestCase):
             # Aggregated results
             {"builds": ["3", "1"],
              "tests": {"001.html": {
             # Aggregated results
             {"builds": ["3", "1"],
              "tests": {"001.html": {
-                           "results": "[5,\"F\"]",
-                           "times": "[5,0]"}}},
+                           "results": [[5,"F"]],
+                           "times": [[5,0]]}}},
             # Incremental results
             {"builds": ["2"],
              "tests": {"001.html": {
             # Incremental results
             {"builds": ["2"],
              "tests": {"001.html": {
-                           "results": "[1, \"F\"]",
-                           "times": "[1,0]"}}},
+                           "results": [[1, "F"]],
+                           "times": [[1,0]]}}},
             # Expected no merge happens.
             {"builds": ["2", "3", "1"],
              "tests": {"001.html": {
             # Expected no merge happens.
             {"builds": ["2", "3", "1"],
              "tests": {"001.html": {
-                           "results": "[6,\"F\"]",
-                           "times": "[6,0]"}}})
+                           "results": [[6,"F"]],
+                           "times": [[6,0]]}}})
 
     def test_merge_incremental_result_same_build(self):
         # Test the build in incremental results is same as the build in
 
     def test_merge_incremental_result_same_build(self):
         # Test the build in incremental results is same as the build in
@@ -394,18 +371,18 @@ class JsonResultsTest(unittest.TestCase):
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[5,\"F\"]",
-                           "times": "[5,0]"}}},
+                           "results": [[5,"F"]],
+                           "times": [[5,0]]}}},
             # Incremental results
             {"builds": ["3", "2"],
              "tests": {"001.html": {
             # Incremental results
             {"builds": ["3", "2"],
              "tests": {"001.html": {
-                           "results": "[2, \"F\"]",
-                           "times": "[2,0]"}}},
+                           "results": [[2, "F"]],
+                           "times": [[2,0]]}}},
             # Expected no merge happens.
             {"builds": ["3", "2", "2", "1"],
              "tests": {"001.html": {
             # Expected no merge happens.
             {"builds": ["3", "2", "2", "1"],
              "tests": {"001.html": {
-                           "results": "[7,\"F\"]",
-                           "times": "[7,0]"}}})
+                           "results": [[7,"F"]],
+                           "times": [[7,0]]}}})
 
     def test_merge_remove_test_with_no_data(self):
         # Remove test where there is no data in all runs.
 
     def test_merge_remove_test_with_no_data(self):
         # Remove test where there is no data in all runs.
@@ -413,24 +390,24 @@ class JsonResultsTest(unittest.TestCase):
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[200,\"N\"]",
-                           "times": "[200,0]"},
+                           "results": [[200,"N"]],
+                           "times": [[200,0]]},
                        "002.html": {
                        "002.html": {
-                           "results": "[10,\"F\"]",
-                           "times": "[10,0]"}}},
+                           "results": [[10,"F"]],
+                           "times": [[10,0]]}}},
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
-                           "results": "[1,\"N\"]",
-                           "times": "[1,0]"},
+                           "results": [[1,"N"]],
+                           "times": [[1,0]]},
                        "002.html": {
                        "002.html": {
-                           "results": "[1,\"P\"]",
-                           "times": "[1,0]"}}},
+                           "results": [[1,"P"]],
+                           "times": [[1,0]]}}},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"002.html": {
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"002.html": {
-                           "results": "[1,\"P\"],[10,\"F\"]",
-                           "times": "[11,0]"}}})
+                           "results": [[1,"P"],[10,"F"]],
+                           "times": [[11,0]]}}})
 
     def test_merge_remove_test_with_all_pass(self):
         # Remove test where all run pass and max running time < 1 seconds
 
     def test_merge_remove_test_with_all_pass(self):
         # Remove test where all run pass and max running time < 1 seconds
@@ -438,24 +415,24 @@ class JsonResultsTest(unittest.TestCase):
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[200,\"P\"]",
-                           "times": "[200,0]"},
+                           "results": [[200,"P"]],
+                           "times": [[200,0]]},
                        "002.html": {
                        "002.html": {
-                           "results": "[10,\"F\"]",
-                           "times": "[10,0]"}}},
+                           "results": [[10,"F"]],
+                           "times": [[10,0]]}}},
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
-                           "results": "[1,\"P\"]",
-                           "times": "[1,0]"},
+                           "results": [[1,"P"]],
+                           "times": [[1,0]]},
                        "002.html": {
                        "002.html": {
-                           "results": "[1,\"P\"]",
-                           "times": "[1,0]"}}},
+                           "results": [[1,"P"]],
+                           "times": [[1,0]]}}},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"002.html": {
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"002.html": {
-                           "results": "[1,\"P\"],[10,\"F\"]",
-                           "times": "[11,0]"}}})
+                           "results": [[1,"P"],[10,"F"]],
+                           "times": [[11,0]]}}})
 
     def test_merge_keep_test_with_all_pass_but_slow_time(self):
         # Do not remove test where all run pass but max running time >= 1 seconds
 
     def test_merge_keep_test_with_all_pass_but_slow_time(self):
         # Do not remove test where all run pass but max running time >= 1 seconds
@@ -463,91 +440,91 @@ class JsonResultsTest(unittest.TestCase):
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[200,\"P\"]",
-                           "times": "[200,0]"},
+                           "results": [[200,"P"]],
+                           "times": [[200,0]]},
                        "002.html": {
                        "002.html": {
-                           "results": "[10,\"F\"]",
-                           "times": "[10,0]"}}},
+                           "results": [[10,"F"]],
+                           "times": [[10,0]]}}},
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
-                           "results": "[1,\"P\"]",
-                           "times": "[1,1]"},
+                           "results": [[1,"P"]],
+                           "times": [[1,1]]},
                        "002.html": {
                        "002.html": {
-                           "results": "[1,\"P\"]",
-                           "times": "[1,0]"}}},
+                           "results": [[1,"P"]],
+                           "times": [[1,0]]}}},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
-                           "results": "[201,\"P\"]",
-                           "times": "[1,1],[200,0]"},
+                           "results": [[201,"P"]],
+                           "times": [[1,1],[200,0]]},
                        "002.html": {
                        "002.html": {
-                           "results": "[1,\"P\"],[10,\"F\"]",
-                           "times": "[11,0]"}}})
+                           "results": [[1,"P"],[10,"F"]],
+                           "times": [[11,0]]}}})
 
     def test_merge_prune_extra_results(self):
         # Remove items from test results and times that exceed the max number
         # of builds to track.
 
     def test_merge_prune_extra_results(self):
         # Remove items from test results and times that exceed the max number
         # of builds to track.
-        max_builds = str(jsonresults.JSON_RESULTS_MAX_BUILDS)
+        max_builds = jsonresults.JSON_RESULTS_MAX_BUILDS
         self._test_merge(
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
         self._test_merge(
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[" + max_builds + ",\"F\"],[1,\"I\"]",
-                           "times": "[" + max_builds + ",0],[1,1]"}}},
+                           "results": [[max_builds,"F"],[1,"I"]],
+                           "times": [[max_builds,0],[1,1]]}}},
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
-                           "results": "[1,\"T\"]",
-                           "times": "[1,1]"}}},
+                           "results": [[1,"T"]],
+                           "times": [[1,1]]}}},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
-                           "results": "[1,\"T\"],[" + max_builds + ",\"F\"]",
-                           "times": "[1,1],[" + max_builds + ",0]"}}})
+                           "results": [[1,"T"],[max_builds,"F"]],
+                           "times": [[1,1],[max_builds,0]]}}})
 
     def test_merge_prune_extra_results_small(self):
         # Remove items from test results and times that exceed the max number
         # of builds to track, using smaller threshold.
 
     def test_merge_prune_extra_results_small(self):
         # Remove items from test results and times that exceed the max number
         # of builds to track, using smaller threshold.
-        max_builds = str(jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL)
+        max_builds = jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL
         self._test_merge(
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
         self._test_merge(
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[" + max_builds + ",\"F\"],[1,\"I\"]",
-                           "times": "[" + max_builds + ",0],[1,1]"}}},
+                           "results": [[max_builds,"F"],[1,"I"]],
+                           "times": [[max_builds,0],[1,1]]}}},
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
-                           "results": "[1,\"T\"]",
-                           "times": "[1,1]"}}},
+                           "results": [[1,"T"]],
+                           "times": [[1,1]]}}},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
-                           "results": "[1,\"T\"],[" + max_builds + ",\"F\"]",
-                           "times": "[1,1],[" + max_builds + ",0]"}}},
+                           "results": [[1,"T"],[max_builds,"F"]],
+                           "times": [[1,1],[max_builds,0]]}}},
             int(max_builds))
 
     def test_merge_prune_extra_results_with_new_result_of_same_type(self):
         # Test that merging in a new result of the same type as the last result
         # causes old results to fall off.
             int(max_builds))
 
     def test_merge_prune_extra_results_with_new_result_of_same_type(self):
         # Test that merging in a new result of the same type as the last result
         # causes old results to fall off.
-        max_builds = str(jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL)
+        max_builds = jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL
         self._test_merge(
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
         self._test_merge(
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"001.html": {
-                           "results": "[" + max_builds + ",\"F\"],[1,\"N\"]",
-                           "times": "[" + max_builds + ",0],[1,1]"}}},
+                           "results": [[max_builds,"F"],[1,"N"]],
+                           "times": [[max_builds,0],[1,1]]}}},
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
             # Incremental results
             {"builds": ["3"],
              "tests": {"001.html": {
-                           "results": "[1,\"F\"]",
-                           "times": "[1,0]"}}},
+                           "results": [[1,"F"]],
+                           "times": [[1,0]]}}},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"001.html": {
-                           "results": "[" + max_builds + ",\"F\"]",
-                           "times": "[" + max_builds + ",0]"}}},
+                           "results": [[max_builds,"F"]],
+                           "times": [[max_builds,0]]}}},
             int(max_builds))
 
     def test_merge_build_directory_hierarchy_old_version(self):
             int(max_builds))
 
     def test_merge_build_directory_hierarchy_old_version(self):
@@ -555,46 +532,46 @@ class JsonResultsTest(unittest.TestCase):
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"bar/003.html": {
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"bar/003.html": {
-                           "results": "[25,\"F\"]",
-                           "times": "[25,0]"},
+                           "results": [[25,"F"]],
+                           "times": [[25,0]]},
                        "foo/001.html": {
                        "foo/001.html": {
-                           "results": "[50,\"F\"]",
-                           "times": "[50,0]"},
+                           "results": [[50,"F"]],
+                           "times": [[50,0]]},
                        "foo/002.html": {
                        "foo/002.html": {
-                           "results": "[100,\"I\"]",
-                           "times": "[100,0]"}},
+                           "results": [[100,"I"]],
+                           "times": [[100,0]]}},
              "version": 3},
             # Incremental results
             {"builds": ["3"],
              "tests": {"baz": {
                            "004.html": {
              "version": 3},
             # Incremental results
             {"builds": ["3"],
              "tests": {"baz": {
                            "004.html": {
-                               "results": "[1,\"I\"]",
-                               "times": "[1,0]"}},
+                               "results": [[1,"I"]],
+                               "times": [[1,0]]}},
                        "foo": {
                            "001.html": {
                        "foo": {
                            "001.html": {
-                               "results": "[1,\"F\"]",
-                               "times": "[1,0]"},
+                               "results": [[1,"F"]],
+                               "times": [[1,0]]},
                            "002.html": {
                            "002.html": {
-                               "results": "[1,\"I\"]",
-                               "times": "[1,0]"}}},
+                               "results": [[1,"I"]],
+                               "times": [[1,0]]}}},
              "version": 4},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"bar": {
                            "003.html": {
              "version": 4},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"bar": {
                            "003.html": {
-                               "results": "[1,\"N\"],[25,\"F\"]",
-                               "times": "[26,0]"}},
+                               "results": [[1,"N"],[25,"F"]],
+                               "times": [[26,0]]}},
                        "baz": {
                            "004.html": {
                        "baz": {
                            "004.html": {
-                               "results": "[1,\"I\"]",
-                               "times": "[1,0]"}},
+                               "results": [[1,"I"]],
+                               "times": [[1,0]]}},
                        "foo": {
                            "001.html": {
                        "foo": {
                            "001.html": {
-                               "results": "[51,\"F\"]",
-                               "times": "[51,0]"},
+                               "results": [[51,"F"]],
+                               "times": [[51,0]]},
                            "002.html": {
                            "002.html": {
-                               "results": "[101,\"I\"]",
-                               "times": "[101,0]"}}},
+                               "results": [[101,"I"]],
+                               "times": [[101,0]]}}},
              "version": 4})
 
     def test_merge_build_directory_hierarchy(self):
              "version": 4})
 
     def test_merge_build_directory_hierarchy(self):
@@ -603,47 +580,47 @@ class JsonResultsTest(unittest.TestCase):
             {"builds": ["2", "1"],
              "tests": {"bar": {"baz": {
                            "003.html": {
             {"builds": ["2", "1"],
              "tests": {"bar": {"baz": {
                            "003.html": {
-                                "results": "[25,\"F\"]",
-                                "times": "[25,0]"}}},
+                                "results": [[25,"F"]],
+                                "times": [[25,0]]}}},
                        "foo": {
                            "001.html": {
                        "foo": {
                            "001.html": {
-                                "results": "[50,\"F\"]",
-                                "times": "[50,0]"},
+                                "results": [[50,"F"]],
+                                "times": [[50,0]]},
                            "002.html": {
                            "002.html": {
-                                "results": "[100,\"I\"]",
-                                "times": "[100,0]"}}},
+                                "results": [[100,"I"]],
+                                "times": [[100,0]]}}},
               "version": 4},
             # Incremental results
             {"builds": ["3"],
              "tests": {"baz": {
                            "004.html": {
               "version": 4},
             # Incremental results
             {"builds": ["3"],
              "tests": {"baz": {
                            "004.html": {
-                               "results": "[1,\"I\"]",
-                               "times": "[1,0]"}},
+                               "results": [[1,"I"]],
+                               "times": [[1,0]]}},
                        "foo": {
                            "001.html": {
                        "foo": {
                            "001.html": {
-                               "results": "[1,\"F\"]",
-                               "times": "[1,0]"},
+                               "results": [[1,"F"]],
+                               "times": [[1,0]]},
                            "002.html": {
                            "002.html": {
-                               "results": "[1,\"I\"]",
-                               "times": "[1,0]"}}},
+                               "results": [[1,"I"]],
+                               "times": [[1,0]]}}},
              "version": 4},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"bar": {"baz": {
                            "003.html": {
              "version": 4},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"bar": {"baz": {
                            "003.html": {
-                               "results": "[1,\"N\"],[25,\"F\"]",
-                               "times": "[26,0]"}}},
+                               "results": [[1,"N"],[25,"F"]],
+                               "times": [[26,0]]}}},
                        "baz": {
                            "004.html": {
                        "baz": {
                            "004.html": {
-                               "results": "[1,\"I\"]",
-                               "times": "[1,0]"}},
+                               "results": [[1,"I"]],
+                               "times": [[1,0]]}},
                        "foo": {
                            "001.html": {
                        "foo": {
                            "001.html": {
-                               "results": "[51,\"F\"]",
-                               "times": "[51,0]"},
+                               "results": [[51,"F"]],
+                               "times": [[51,0]]},
                            "002.html": {
                            "002.html": {
-                               "results": "[101,\"I\"]",
-                               "times": "[101,0]"}}},
+                               "results": [[101,"I"]],
+                               "times": [[101,0]]}}},
              "version": 4})
 
     # FIXME(aboxhall): Add some tests for xhtml/svg test results.
              "version": 4})
 
     # FIXME(aboxhall): Add some tests for xhtml/svg test results.
@@ -651,68 +628,74 @@ class JsonResultsTest(unittest.TestCase):
     def test_get_test_name_list(self):
         # Get test name list only. Don't include non-test-list data and
         # of test result details.
     def test_get_test_name_list(self):
         # Get test name list only. Don't include non-test-list data and
         # of test result details.
+        # FIXME: This also tests a temporary bug in the data where directory-level
+        # results have a results and times values. Once that bug is fixed,
+        # remove this test-case and assert we don't ever hit it.
         self._test_get_test_list(
             # Input results
             {"builds": ["3", "2", "1"],
         self._test_get_test_list(
             # Input results
             {"builds": ["3", "2", "1"],
-             "tests": {"001.html": {
-                           "results": "[200,\"P\"]",
-                           "times": "[200,0]"},
+             "tests": {"foo": {
+                           "001.html": {
+                               "results": [[200,"P"]],
+                               "times": [[200,0]]},
+                           "results": [[1,"N"]],
+                           "times": [[1,0]]},
                        "002.html": {
                        "002.html": {
-                           "results": "[10,\"F\"]",
-                           "times": "[10,0]"}}},
+                           "results": [[10,"F"]],
+                           "times": [[10,0]]}}},
             # Expected results
             # Expected results
-            ["001.html", "002.html"])
+            {"foo": {"001.html":{}}, "002.html":{}})
 
     def test_remove_gtest_modifiers(self):
         self._test_merge(
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"foo.bar": {
 
     def test_remove_gtest_modifiers(self):
         self._test_merge(
             # Aggregated results
             {"builds": ["2", "1"],
              "tests": {"foo.bar": {
-                           "results": "[50,\"F\"]",
-                           "times": "[50,0]"},
+                           "results": [[50,"F"]],
+                           "times": [[50,0]]},
                        "foo.bar2": {
                        "foo.bar2": {
-                           "results": "[100,\"I\"]",
-                           "times": "[100,0]"},
+                           "results": [[100,"I"]],
+                           "times": [[100,0]]},
                        "foo.FAILS_bar3": {
                        "foo.FAILS_bar3": {
-                           "results": "[100,\"I\"]",
-                           "times": "[100,0]"},
+                           "results": [[100,"I"]],
+                           "times": [[100,0]]},
                        },
              "version": 3},
             # Incremental results
             {"builds": ["3"],
              "tests": {"foo.FLAKY_bar": {
                        },
              "version": 3},
             # Incremental results
             {"builds": ["3"],
              "tests": {"foo.FLAKY_bar": {
-                           "results": "[1,\"F\"]",
-                           "times": "[1,0]"},
+                           "results": [[1,"F"]],
+                           "times": [[1,0]]},
                        "foo.DISABLED_bar2": {
                        "foo.DISABLED_bar2": {
-                           "results": "[1,\"I\"]",
-                           "times": "[1,0]"},
+                           "results": [[1,"I"]],
+                           "times": [[1,0]]},
                        "foo.bar3": {
                        "foo.bar3": {
-                           "results": "[1,\"I\"]",
-                           "times": "[1,0]"},
+                           "results": [[1,"I"]],
+                           "times": [[1,0]]},
                        "foo.FAILS_bar3": {
                        "foo.FAILS_bar3": {
-                           "results": "[1,\"I\"]",
-                           "times": "[1,0]"},
+                           "results": [[1,"I"]],
+                           "times": [[1,0]]},
                        "foo.MAYBE_bar4": {
                        "foo.MAYBE_bar4": {
-                           "results": "[1,\"I\"]",
-                           "times": "[1,0]"}},
+                           "results": [[1,"I"]],
+                           "times": [[1,0]]}},
              "version": 4},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"foo.bar": {
              "version": 4},
             # Expected results
             {"builds": ["3", "2", "1"],
              "tests": {"foo.bar": {
-                           "results": "[51,\"F\"]",
-                           "times": "[51,0]"},
+                           "results": [[51,"F"]],
+                           "times": [[51,0]]},
                        "foo.bar2": {
                        "foo.bar2": {
-                           "results": "[101,\"I\"]",
-                           "times": "[101,0]"},
+                           "results": [[101,"I"]],
+                           "times": [[101,0]]},
                        "foo.bar3": {
                        "foo.bar3": {
-                           "results": "[1,\"I\"]",
-                           "times": "[1,0]"},
+                           "results": [[1,"I"]],
+                           "times": [[1,0]]},
                        "foo.FAILS_bar3": {
                        "foo.FAILS_bar3": {
-                              "results": "[1,\"N\"],[100,\"I\"]",
-                              "times": "[101,0]"},
+                              "results": [[1,"N"],[100,"I"]],
+                              "times": [[101,0]]},
                        "foo.bar4": {
                        "foo.bar4": {
-                           "results": "[1,\"I\"]",
-                           "times": "[1,0]"}},
+                           "results": [[1,"I"]],
+                           "times": [[1,0]]}},
              "version": 4})
 
 if __name__ == '__main__':
              "version": 4})
 
 if __name__ == '__main__':