2010-10-27 Ojan Vafai <ojan@chromium.org>

        Reviewed by Dimitri Glazkov.

        [chromium] Make the test results server store which master the bot is on
        https://bugs.webkit.org/show_bug.cgi?id=48478

        The chromium bots recently changed so that there are multiple slaves with
        the same name on different masters. Up till now, the test results server
        assumed slave names were unique. Adds a master field to the file in order
        to distinguish.

        Also, for files that currently lack a master or testtype, set them appropriately.

        * TestResultServer/handlers/testfilehandler.py:
        * TestResultServer/index.yaml:
        * TestResultServer/model/jsonresults.py:
        * TestResultServer/model/testfile.py:
        * TestResultServer/templates/showfilelist.html:
        * TestResultServer/templates/uploadform.html:

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@70724 268f45cc-cd09-0410-ab3c-d52691b4dbfc

diff --git a/WebKitTools/ChangeLog b/WebKitTools/ChangeLog
index 86f39dc1667b85f4a84fad60868b2b019fcfb3f7..431041cc33dba9aa71a085a7e977bf8d7d496c3b 100644 (file)
--- a/WebKitTools/ChangeLog
+++ b/WebKitTools/ChangeLog
@@ -1,3 +1,24 @@
+2010-10-27  Ojan Vafai  <ojan@chromium.org>
+
+        Reviewed by Dimitri Glazkov.
+
+        [chromium] Make the test results server store which master the bot is on
+        https://bugs.webkit.org/show_bug.cgi?id=48478
+
+        The chromium bots recently changed so that there are multiple slaves with
+        the same name on different masters. Up till now, the test results server
+        assumed slave names were unique. Adds a master field to the file in order
+        to distinguish.
+
+        Also, for files that currently lack a master or testtype, set them appropriately.
+
+        * TestResultServer/handlers/testfilehandler.py:
+        * TestResultServer/index.yaml:
+        * TestResultServer/model/jsonresults.py:
+        * TestResultServer/model/testfile.py:
+        * TestResultServer/templates/showfilelist.html:
+        * TestResultServer/templates/uploadform.html:
+
 2010-10-26  Darin Adler  <darin@apple.com>
 
         Reviewed by Sam Weinig.

diff --git a/WebKitTools/TestResultServer/handlers/testfilehandler.py b/WebKitTools/TestResultServer/handlers/testfilehandler.py
index 4d1320fe2a24bb753edbddc160337109d7faecc3..d81789017d3abb72ea608d347e9618df49548480 100644 (file)
--- a/WebKitTools/TestResultServer/handlers/testfilehandler.py
+++ b/WebKitTools/TestResultServer/handlers/testfilehandler.py
@@ -36,6 +36,7 @@ from google.appengine.ext.webapp import template
 from model.jsonresults import JsonResults
 from model.testfile import TestFile
 
+PARAM_MASTER = "master"
 PARAM_BUILDER = "builder"
 PARAM_DIR = "dir"
 PARAM_FILE = "file"
@@ -51,25 +52,26 @@ class DeleteFile(webapp.RequestHandler):
 
     def get(self):
         key = self.request.get(PARAM_KEY)
+        master = self.request.get(PARAM_MASTER)
         builder = self.request.get(PARAM_BUILDER)
         test_type = self.request.get(PARAM_TEST_TYPE)
         name = self.request.get(PARAM_NAME)
 
         logging.debug(
-            "Deleting File, builder: %s, test_type: %s, name: %s, key: %s.",
-            builder, test_type, name, key)
+            "Deleting File, master: %s, builder: %s, test_type: %s, name: %s, key: %s.",
+            master, builder, test_type, name, key)
 
-        TestFile.delete_file(key, builder, test_type, name, 100)
+        TestFile.delete_file(key, master, builder, test_type, name, 100)
 
         # Display file list after deleting the file.
-        self.redirect("/testfile?builder=%s&testtype=%s&name=%s"
-            % (builder, test_type, name))
+        self.redirect("/testfile?master=%s&builder=%s&testtype=%s&name=%s"
+            % (master, builder, test_type, name))
 
 
 class GetFile(webapp.RequestHandler):
     """Get file content or list of files for given builder and name."""
 
-    def _get_file_list(self, builder, test_type, name):
+    def _get_file_list(self, master, builder, test_type, name):
         """Get and display a list of files that matches builder and file name.
 
         Args:
@@ -79,15 +81,16 @@ class GetFile(webapp.RequestHandler):
         """
 
         files = TestFile.get_files(
-            builder, test_type, name, load_data=False, limit=100)
+            master, builder, test_type, name, load_data=False, limit=100)
         if not files:
-            logging.info("File not found, builder: %s, test_type: %s, name: %s.",
-                         builder, test_type, name)
+            logging.info("File not found, master: %s, builder: %s, test_type: %s, name: %s.",
+                         master, builder, test_type, name)
             self.response.out.write("File not found")
             return
 
         template_values = {
             "admin": users.is_current_user_admin(),
+            "master": master,
             "builder": builder,
             "test_type": test_type,
             "name": name,
@@ -96,7 +99,7 @@ class GetFile(webapp.RequestHandler):
         self.response.out.write(template.render("templates/showfilelist.html",
                                                 template_values))
 
-    def _get_file_content(self, builder, test_type, name):
+    def _get_file_content(self, master, builder, test_type, name):
         """Return content of the file that matches builder and file name.
 
         Args:
@@ -106,15 +109,15 @@ class GetFile(webapp.RequestHandler):
         """
 
         files = TestFile.get_files(
-            builder, test_type, name, load_data=True, limit=1)
+            master, builder, test_type, name, load_data=True, limit=1)
         if not files:
-            logging.info("File not found, builder: %s, test_type: %s, name: %s.",
-                         builder, test_type, name)
+            logging.info("File not found, master %s, builder: %s, test_type: %s, name: %s.",
+                         master, builder, test_type, name)
             return None
 
         return files[0].data
 
-    def _get_test_list_json(self, builder, test_type):
+    def _get_test_list_json(self, master, builder, test_type):
         """Return json file with test name list only, do not include test
            results and other non-test-data .
 
@@ -123,13 +126,14 @@ class GetFile(webapp.RequestHandler):
             test_type: type of test results.
         """
 
-        json = self._get_file_content(builder, test_type, "results.json")
+        json = self._get_file_content(master, builder, test_type, "results.json")
         if not json:
             return None
 
         return JsonResults.get_test_list(builder, json)
 
     def get(self):
+        master = self.request.get(PARAM_MASTER)
         builder = self.request.get(PARAM_BUILDER)
         test_type = self.request.get(PARAM_TEST_TYPE)
         name = self.request.get(PARAM_NAME)
@@ -137,19 +141,19 @@ class GetFile(webapp.RequestHandler):
         test_list_json = self.request.get(PARAM_TEST_LIST_JSON)
 
         logging.debug(
-            "Getting files, builder: %s, test_type: %s, name: %s.",
-            builder, test_type, name)
+            "Getting files, master %s, builder: %s, test_type: %s, name: %s.",
+            master, builder, test_type, name)
 
         # If parameter "dir" is specified or there is no builder or filename
         # specified in the request, return list of files, otherwise, return
         # file content.
         if dir or not builder or not name:
-            return self._get_file_list(builder, test_type, name)
+            return self._get_file_list(master, builder, test_type, name)
 
         if name == "results.json" and test_list_json:
-            json = self._get_test_list_json(builder, test_type)
+            json = self._get_test_list_json(master, builder, test_type)
         else:
-            json = self._get_file_content(builder, test_type, name)
+            json = self._get_file_content(master, builder, test_type, name)
 
         if json:
             self.response.headers["Content-Type"] = "text/plain; charset=utf-8"
@@ -170,12 +174,13 @@ class Upload(webapp.RequestHandler):
             self.response.out.write("FAIL: missing builder parameter.")
             return
 
+        master = self.request.get(PARAM_MASTER)
         test_type = self.request.get(PARAM_TEST_TYPE)
         incremental = self.request.get(PARAM_INCREMENTAL)
 
         logging.debug(
-            "Processing upload request, builder: %s, test_type: %s.",
-            builder, test_type)
+            "Processing upload request, master: %s, builder: %s, test_type: %s.",
+            master, builder, test_type)
 
         # There are two possible types of each file_params in the request:
         # one file item or a list of file items.
@@ -193,15 +198,15 @@ class Upload(webapp.RequestHandler):
             if ((incremental and filename == "results.json") or
                 (filename == "incremental_results.json")):
                 # Merge incremental json results.
-                saved_file = JsonResults.update(builder, test_type, file.value)
+                saved_file = JsonResults.update(master, builder, test_type, file.value)
             else:
                 saved_file = TestFile.update(
-                    builder, test_type, file.filename, file.value)
+                    master, builder, test_type, file.filename, file.value)
 
             if not saved_file:
                 errors.append(
-                    "Upload failed, builder: %s, test_type: %s, name: %s." %
-                    (builder, test_type, file.filename))
+                    "Upload failed, master: %s, builder: %s, test_type: %s, name: %s." %
+                    (master, builder, test_type, file.filename))
 
         if errors:
             messages = "FAIL: " + "; ".join(errors)

diff --git a/WebKitTools/TestResultServer/index.yaml b/WebKitTools/TestResultServer/index.yaml
index 50284dc82f50bef247cd57a54efc2d6b1ae28f6c..a7d3e48b090c9ed5a9d2227d1f2836dc3ad0c8ca 100644 (file)
--- a/WebKitTools/TestResultServer/index.yaml
+++ b/WebKitTools/TestResultServer/index.yaml
@@ -22,6 +22,15 @@ indexes:
   - name: date
     direction: desc
 
+- kind: TestFile
+  properties:
+  - name: builder
+  - name: master
+  - name: name
+  - name: test_type
+  - name: date
+    direction: desc
+
 - kind: TestFile
   properties:
   - name: builder
@@ -37,6 +46,12 @@ indexes:
   - name: date
     direction: desc
 
+- kind: TestFile
+  properties:
+  - name: master
+  - name: date
+    direction: desc
+
 - kind: TestFile
   properties:
   - name: name

diff --git a/WebKitTools/TestResultServer/model/jsonresults.py b/WebKitTools/TestResultServer/model/jsonresults.py
index 4520e967563e762c41b2c8aa4e59a2f22a554cec..879cf78b6a08cc5785da2e405db5ffe665bc9b5a 100755 (executable)
--- a/WebKitTools/TestResultServer/model/jsonresults.py
+++ b/WebKitTools/TestResultServer/model/jsonresults.py
@@ -391,7 +391,7 @@ class JsonResults(object):
         return cls._generate_file_data(aggregated_json, sort_keys)
 
     @classmethod
-    def update(cls, builder, test_type, incremental):
+    def update(cls, master, builder, test_type, incremental):
         """Update datastore json file data by merging it with incremental json
            file.
 
@@ -405,13 +405,23 @@ class JsonResults(object):
             None on failure.
         """
 
-        files = TestFile.get_files(builder, test_type, JSON_RESULTS_FILE)
+        files = TestFile.get_files(master, builder, test_type, JSON_RESULTS_FILE)
         if files:
             file = files[0]
+
+            # FIXME: This is here to fill in the missing master/test_type for the already uploaded
+            # results files, which all are layout_tests from the chromium master.
+            # Remove this once all the builders upload with the master/test_type field set.
+            if not file.master:
+                file.master = "chromium"
+            if not file.test_type:
+                file.test_type = "layout-tests"
+
             new_results = cls.merge(builder, file.data, incremental)
         else:
             # Use the incremental data if there is no aggregated file to merge.
             file = TestFile()
+            file.master = master
             file.builder = builder
             file.test_type = test_type
             file.name = JSON_RESULTS_FILE

diff --git a/WebKitTools/TestResultServer/model/testfile.py b/WebKitTools/TestResultServer/model/testfile.py
index ce92b65f95d2311ad606206615e873a2e21faf23..9d57023a92956e4f64e8843511f6ab70deb809a8 100644 (file)
--- a/WebKitTools/TestResultServer/model/testfile.py
+++ b/WebKitTools/TestResultServer/model/testfile.py
@@ -35,11 +35,12 @@ from model.datastorefile import DataStoreFile
 
 
 class TestFile(DataStoreFile):
+    master = db.StringProperty()
     builder = db.StringProperty()
     test_type = db.StringProperty()
 
     @classmethod
-    def delete_file(cls, key, builder, test_type, name, limit):
+    def delete_file(cls, key, master, builder, test_type, name, limit):
         if key:
             file = db.get(key)
             if not file:
@@ -48,10 +49,10 @@ class TestFile(DataStoreFile):
 
             file._delete_all()
         else:
-            files = cls.get_files(builder, test_type, name, limit)
+            files = cls.get_files(master, builder, test_type, name, limit)
             if not files:
                 logging.warning(
-                    "File not found, builder: %s, test_type:%s, name: %s.",
+                    "File not found, master: %s, builder: %s, test_type:%s, name: %s.",
                     builder, test_type, name)
                 return False
 
@@ -61,8 +62,10 @@ class TestFile(DataStoreFile):
         return True
 
     @classmethod
-    def get_files(cls, builder, test_type, name, load_data=True, limit=1):
+    def get_files(cls, master, builder, test_type, name, load_data=True, limit=1):
         query = TestFile.all()
+        if master:
+            query = query.filter("master =", master)
         if builder:
             query = query.filter("builder =", builder)
         if test_type:
@@ -78,8 +81,9 @@ class TestFile(DataStoreFile):
         return files
 
     @classmethod
-    def add_file(cls, builder, test_type, name, data):
+    def add_file(cls, master, builder, test_type, name, data):
         file = TestFile()
+        file.master = master
         file.builder = builder
         file.test_type = test_type
         file.name = name
@@ -88,24 +92,33 @@ class TestFile(DataStoreFile):
             return None
 
         logging.info(
-            "File saved, builder: %s, test_type: %s, name: %s, key: %s.",
-            builder, test_type, file.name, str(file.data_keys))
+            "File saved, master: %s, builder: %s, test_type: %s, name: %s, key: %s.",
+            master, builder, test_type, file.name, str(file.data_keys))
 
         return file
 
     @classmethod
-    def update(cls, builder, test_type, name, data):
-        files = cls.get_files(builder, test_type, name)
+    def update(cls, master, builder, test_type, name, data):
+        files = cls.get_files(master, builder, test_type, name)
         if not files:
-            return cls.add_file(builder, test_type, name, data)
+            return cls.add_file(master, builder, test_type, name, data)
 
         file = files[0]
+
+        # FIXME: This is here to fill in the missing master/test_type for the already uploaded
+        # results files, which all are layout_tests from the chromium master.
+        # Remove this once all the builders upload with the master/test_type field set.
+        if not file.master:
+            file.master = "chromium"
+        if not file.test_type:
+            file.test_type = "layout-tests"
+
         if not file.save(data):
             return None
 
         logging.info(
-            "File replaced, builder: %s, test_type: %s, name: %s, data key: %s.",
-            builder, test_type, file.name, str(file.data_keys))
+            "File replaced, master: %s, builder: %s, test_type: %s, name: %s, data key: %s.",
+            master, builder, test_type, file.name, str(file.data_keys))
 
         return file
 

diff --git a/WebKitTools/TestResultServer/templates/showfilelist.html b/WebKitTools/TestResultServer/templates/showfilelist.html
index fa72b7fa5660352282785815378ef4898efc7fe9..d292fe27e2068c020e3c8560f175c3be1ddf9a5e 100644 (file)
--- a/WebKitTools/TestResultServer/templates/showfilelist.html
+++ b/WebKitTools/TestResultServer/templates/showfilelist.html
@@ -13,6 +13,7 @@
 <div>
     <table>
         <tr>
+            <th>Master</th>
             <th>Builder</th>
             <th>Test Type</th>
             <th>File</th>
@@ -22,6 +23,10 @@
             {% endif %}
         {% for file in files %}
         <tr>{% if file.builder and file.name %}
+            <td><a href="/testfile?master={{ file.master }}" >
+                {{ file.master }}
+                </a>
+            </td>
             <td><a href="/testfile?builder={{ file.builder }}" >
                 {{ file.builder }}
                 </a>

diff --git a/WebKitTools/TestResultServer/templates/uploadform.html b/WebKitTools/TestResultServer/templates/uploadform.html
index 3506c9cb4d5d7c168773885d8196ed8b7173fe51..7427c8504d18b1b218e1a61c3445997404cebd30 100644 (file)
--- a/WebKitTools/TestResultServer/templates/uploadform.html
+++ b/WebKitTools/TestResultServer/templates/uploadform.html
@@ -9,13 +9,17 @@
 <form id="uploadForm" name="test_result_upload" accept="text/html" action="{{ upload_url }}" enctype="multipart/form-data" method="post">
     <br>
     <table>
+    <tr>
+        <td class=label><label>Master:</label></td>
+        <td><input class=inputtext type="text" name="master" placeholder="chromium"/></td>
+    </tr>
     <tr>
         <td class=label><label>Builder:</label></td>
-        <td><input class=inputtext type="text" name="builder" value="Webkit"/></td>
+        <td><input class=inputtext type="text" name="builder" placeholder="Webkit"/></td>
     </tr>
     <tr>
         <td class=label><label>Test Type:</label></td>
-        <td><input class=inputtext type="text" name="testtype" value=""/></td>
+        <td><input class=inputtext type="text" name="testtype" placeholder="layout-tests"/></td>
     </tr>
     </table>
     <br>