Add monitoring of patches and queues to the QueueStatusServer
authorcommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 23 Jan 2013 07:36:10 +0000 (07:36 +0000)
committercommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 23 Jan 2013 07:36:10 +0000 (07:36 +0000)
https://bugs.webkit.org/show_bug.cgi?id=107612

Patch by Alan Cutter <alancutter@chromium.org> on 2013-01-22
Reviewed by Adam Barth.

Created classes for recording events into the datastore and integrated them into the existing handlers.
Code for presenting the recorded data will come in a separate patch.

* QueueStatusServer/app.yaml:
* QueueStatusServer/config/__init__.py: Added.
* QueueStatusServer/config/logging.py: Copied from Tools/QueueStatusServer/model/queuestatus.py.
* QueueStatusServer/config/messages.py: Copied from Tools/QueueStatusServer/model/queuestatus.py.
* QueueStatusServer/handlers/nextpatch.py:
(NextPatch.get):
(NextPatch._assign_patch):
* QueueStatusServer/handlers/releasepatch.py:
(ReleasePatch.post):
* QueueStatusServer/handlers/updatestatus.py:
(UpdateStatus.post):
* QueueStatusServer/handlers/updateworkitems.py:
(UpdateWorkItems._parse_work_items_string):
(UpdateWorkItems):
(UpdateWorkItems._update_work_items_from_request):
(UpdateWorkItems._queue_from_request):
(UpdateWorkItems.post):
* QueueStatusServer/loggers/__init__.py: Added.
* QueueStatusServer/loggers/recordbotevent.py: Copied from Tools/QueueStatusServer/model/queuestatus.py.
(RecordBotEvent):
(RecordBotEvent.activity):
* QueueStatusServer/loggers/recordpatchevent.py: Added.
(RecordPatchEvent):
(RecordPatchEvent.added):
(RecordPatchEvent.retrying):
(RecordPatchEvent.started):
(RecordPatchEvent.stopped):
(RecordPatchEvent.updated):
(RecordPatchEvent._get_patches_waiting):
* QueueStatusServer/model/patchlog.py: Copied from Tools/QueueStatusServer/model/queuestatus.py.
(PatchLog):
(PatchLog.lookup):
(PatchLog.calculate_wait_duration):
(PatchLog.calculate_process_duration):
* QueueStatusServer/model/queuelog.py: Copied from Tools/QueueStatusServer/handlers/nextpatch.py.
(QueueLog):
(QueueLog.get_current):
(QueueLog.create_key):
* QueueStatusServer/model/queuestatus.py:
(QueueStatus.is_retry_request):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@140513 268f45cc-cd09-0410-ab3c-d52691b4dbfc

15 files changed:
Tools/ChangeLog
Tools/QueueStatusServer/app.yaml
Tools/QueueStatusServer/config/__init__.py [new file with mode: 0644]
Tools/QueueStatusServer/config/logging.py [new file with mode: 0644]
Tools/QueueStatusServer/config/messages.py [new file with mode: 0644]
Tools/QueueStatusServer/handlers/nextpatch.py
Tools/QueueStatusServer/handlers/releasepatch.py
Tools/QueueStatusServer/handlers/updatestatus.py
Tools/QueueStatusServer/handlers/updateworkitems.py
Tools/QueueStatusServer/loggers/__init__.py [new file with mode: 0644]
Tools/QueueStatusServer/loggers/recordbotevent.py [new file with mode: 0644]
Tools/QueueStatusServer/loggers/recordpatchevent.py [new file with mode: 0644]
Tools/QueueStatusServer/model/patchlog.py [new file with mode: 0644]
Tools/QueueStatusServer/model/queuelog.py [new file with mode: 0644]
Tools/QueueStatusServer/model/queuestatus.py

index 55ecbd2..f98c425 100644 (file)
@@ -1,3 +1,54 @@
+2013-01-22  Alan Cutter  <alancutter@chromium.org>
+
+        Add monitoring of patches and queues to the QueueStatusServer
+        https://bugs.webkit.org/show_bug.cgi?id=107612
+
+        Reviewed by Adam Barth.
+
+        Created classes for recording events into the datastore and integrated them into the existing handlers.
+        Code for presenting the recorded data will come in a separate patch.
+
+        * QueueStatusServer/app.yaml:
+        * QueueStatusServer/config/__init__.py: Added.
+        * QueueStatusServer/config/logging.py: Copied from Tools/QueueStatusServer/model/queuestatus.py.
+        * QueueStatusServer/config/messages.py: Copied from Tools/QueueStatusServer/model/queuestatus.py.
+        * QueueStatusServer/handlers/nextpatch.py:
+        (NextPatch.get):
+        (NextPatch._assign_patch):
+        * QueueStatusServer/handlers/releasepatch.py:
+        (ReleasePatch.post):
+        * QueueStatusServer/handlers/updatestatus.py:
+        (UpdateStatus.post):
+        * QueueStatusServer/handlers/updateworkitems.py:
+        (UpdateWorkItems._parse_work_items_string):
+        (UpdateWorkItems):
+        (UpdateWorkItems._update_work_items_from_request):
+        (UpdateWorkItems._queue_from_request):
+        (UpdateWorkItems.post):
+        * QueueStatusServer/loggers/__init__.py: Added.
+        * QueueStatusServer/loggers/recordbotevent.py: Copied from Tools/QueueStatusServer/model/queuestatus.py.
+        (RecordBotEvent):
+        (RecordBotEvent.activity):
+        * QueueStatusServer/loggers/recordpatchevent.py: Added.
+        (RecordPatchEvent):
+        (RecordPatchEvent.added):
+        (RecordPatchEvent.retrying):
+        (RecordPatchEvent.started):
+        (RecordPatchEvent.stopped):
+        (RecordPatchEvent.updated):
+        (RecordPatchEvent._get_patches_waiting):
+        * QueueStatusServer/model/patchlog.py: Copied from Tools/QueueStatusServer/model/queuestatus.py.
+        (PatchLog):
+        (PatchLog.lookup):
+        (PatchLog.calculate_wait_duration):
+        (PatchLog.calculate_process_duration):
+        * QueueStatusServer/model/queuelog.py: Copied from Tools/QueueStatusServer/handlers/nextpatch.py.
+        (QueueLog):
+        (QueueLog.get_current):
+        (QueueLog.create_key):
+        * QueueStatusServer/model/queuestatus.py:
+        (QueueStatus.is_retry_request):
+
 2013-01-22  Timothy Loh  <timloh@chromium.com>
 
         prepare-Changelog should support updating the list of changed files
index e320eb0..35b4ed5 100644 (file)
@@ -1,5 +1,5 @@
 application: webkit-commit-queue
-version: 1
+version: 107612 # Bugzilla bug ID of last major change
 runtime: python
 api_version: 1
 
diff --git a/Tools/QueueStatusServer/config/__init__.py b/Tools/QueueStatusServer/config/__init__.py
new file mode 100644 (file)
index 0000000..ef65bee
--- /dev/null
@@ -0,0 +1 @@
+# Required for Python to search this directory for module files
diff --git a/Tools/QueueStatusServer/config/logging.py b/Tools/QueueStatusServer/config/logging.py
new file mode 100644 (file)
index 0000000..b18b071
--- /dev/null
@@ -0,0 +1,30 @@
+# Copyright (C) 2013 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Specified in seconds
+queue_log_duration = 60 * 60
diff --git a/Tools/QueueStatusServer/config/messages.py b/Tools/QueueStatusServer/config/messages.py
new file mode 100644 (file)
index 0000000..96547ab
--- /dev/null
@@ -0,0 +1,33 @@
+# Copyright (C) 2013 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# These must be in sync with webkit-patch's AbstractQueue.
+pass_status = "Pass"
+fail_status = "Fail"
+retry_status = "Retry"
+error_status = "Error"
index 5f6d71d..e5ba655 100644 (file)
@@ -1,4 +1,4 @@
-# Copyright (C) 2010 Google Inc. All rights reserved.
+# Copyright (C) 2013 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
@@ -31,6 +31,7 @@ from datetime import datetime
 from google.appengine.ext import db
 from google.appengine.ext import webapp
 
+from loggers.recordpatchevent import RecordPatchEvent
 from model.queues import Queue
 
 
@@ -47,11 +48,12 @@ class NextPatch(webapp.RequestHandler):
         if not patch_id:
             self.error(404)
             return
+        RecordPatchEvent.started(patch_id, queue_name)
         self.response.out.write(patch_id)
 
     @staticmethod
     def _assign_patch(key, work_item_ids):
-        now = datetime.now()
+        now = datetime.utcnow()
         active_work_items = db.get(key)
         active_work_items.deactivate_expired(now)
         next_item = active_work_items.next_item(work_item_ids, now)
index 0e46e69..88f46b0 100644 (file)
@@ -1,4 +1,4 @@
-# Copyright (C) 2010 Google Inc. All rights reserved.
+# Copyright (C) 2013 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
@@ -30,6 +30,7 @@ from google.appengine.ext import webapp, db
 from google.appengine.ext.webapp import template
 
 from handlers.updatebase import UpdateBase
+from loggers.recordpatchevent import RecordPatchEvent
 from model.attachment import Attachment
 from model.queues import Queue
 
@@ -57,6 +58,9 @@ class ReleasePatch(UpdateBase):
         # Allow removing it from the queue even if there is no last_status for easier testing.
         if not last_status or not last_status.is_retry_request():
             queue.work_items().remove_work_item(attachment_id)
+            RecordPatchEvent.stopped(attachment_id, queue_name)
+        else:
+            RecordPatchEvent.retrying(attachment_id, queue_name)
 
         # Always release the lock on the item.
         queue.active_work_items().expire_item(attachment_id)
index 7301101..a0e7194 100644 (file)
@@ -1,4 +1,4 @@
-# Copyright (C) 2009 Google Inc. All rights reserved.
+# Copyright (C) 2013 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
@@ -31,6 +31,8 @@ from google.appengine.ext import webapp, db
 from google.appengine.ext.webapp import template
 
 from handlers.updatebase import UpdateBase
+from loggers.recordbotevent import RecordBotEvent
+from loggers.recordpatchevent import RecordPatchEvent
 from model.attachment import Attachment
 from model.queuestatus import QueueStatus
 
@@ -62,5 +64,8 @@ class UpdateStatus(UpdateBase):
     def post(self):
         queue_status = self._queue_status_from_request()
         queue_status.put()
+        RecordBotEvent.record_activity(queue_status.queue_name, queue_status.bot_id)
+        if queue_status.active_patch_id:
+            RecordPatchEvent.updated(queue_status.active_patch_id, queue_status.queue_name, queue_status.bot_id)
         Attachment.dirty(queue_status.active_patch_id)
         self.response.out.write(queue_status.key().id())
index 16a9d49..6b3dded 100644 (file)
@@ -1,4 +1,4 @@
-# Copyright (C) 2010 Google Inc. All rights reserved.
+# Copyright (C) 2013 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
@@ -30,6 +30,7 @@ from google.appengine.ext import webapp, db
 from google.appengine.ext.webapp import template
 
 from handlers.updatebase import UpdateBase
+from loggers.recordpatchevent import RecordPatchEvent
 from model.queues import Queue
 from model.workitems import WorkItems
 
@@ -41,26 +42,46 @@ class UpdateWorkItems(UpdateBase):
         self.response.out.write(template.render("templates/updateworkitems.html", None))
 
     def _parse_work_items_string(self, items_string):
-        # Our parsing could be much more robust.
-        item_strings = items_string.split(" ") if items_string else []
-        return map(int, item_strings)
+        try:
+            item_strings = items_string.split(" ") if items_string else []
+            return map(int, item_strings)
+        except ValueError:
+            return None
+
+    def _update_work_items_from_request(self, work_items):
+        items_string = self.request.get("work_items")
+        new_work_items = self._parse_work_items_string(items_string)
+        if new_work_items == None:
+            self.response.out.write("Failed to parse work items: %s" % items_string)
+            return False
+        work_items.item_ids = new_work_items
+        work_items.date = datetime.utcnow()
+        return True
 
-    def _work_items_from_request(self):
+    def _queue_from_request(self):
         queue_name = self.request.get("queue_name")
         queue = Queue.queue_with_name(queue_name)
         if not queue:
             self.response.out.write("\"%s\" is not in queues %s" % (queue_name, Queue.all()))
             return None
+        return queue
 
-        items_string = self.request.get("work_items")
+    def post(self):
+        queue = self._queue_from_request()
+        if not queue:
+            self.response.set_status(500)
+            return
         work_items = queue.work_items()
-        work_items.item_ids = self._parse_work_items_string(items_string)
-        work_items.date = datetime.now()
-        return work_items
+        old_items = set(work_items.item_ids)
 
-    def post(self):
-        work_items = self._work_items_from_request()
-        if not work_items:
+        success = self._update_work_items_from_request(work_items)
+        if not success:
             self.response.set_status(500)
             return
+        new_items = set(work_items.item_ids)
         work_items.put()
+
+        for work_item in new_items - old_items:
+            RecordPatchEvent.added(work_item, queue.name())
+        for work_item in old_items - new_items:
+            RecordPatchEvent.stopped(work_item, queue.name())
diff --git a/Tools/QueueStatusServer/loggers/__init__.py b/Tools/QueueStatusServer/loggers/__init__.py
new file mode 100644 (file)
index 0000000..ef65bee
--- /dev/null
@@ -0,0 +1 @@
+# Required for Python to search this directory for module files
diff --git a/Tools/QueueStatusServer/loggers/recordbotevent.py b/Tools/QueueStatusServer/loggers/recordbotevent.py
new file mode 100644 (file)
index 0000000..cb0e224
--- /dev/null
@@ -0,0 +1,39 @@
+# Copyright (C) 2013 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from config.logging import queue_log_duration
+from model.queuelog import QueueLog
+
+
+class RecordBotEvent(object):
+    @classmethod
+    def record_activity(cls, queue_name, bot_id):
+        queue_log = QueueLog.get_current(queue_name, queue_log_duration)
+        if queue_log and bot_id not in queue_log.bot_ids_seen:
+            queue_log.bot_ids_seen.append(bot_id)
+            queue_log.put()
diff --git a/Tools/QueueStatusServer/loggers/recordpatchevent.py b/Tools/QueueStatusServer/loggers/recordpatchevent.py
new file mode 100644 (file)
index 0000000..cb55925
--- /dev/null
@@ -0,0 +1,101 @@
+# Copyright (C) 2013 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from config.logging import queue_log_duration
+from model.patchlog import PatchLog
+from model.queuelog import QueueLog
+from model.workitems import WorkItems
+from model.activeworkitems import ActiveWorkItems
+
+
+class RecordPatchEvent(object):
+    @classmethod
+    def added(cls, attachment_id, queue_name):
+        PatchLog.lookup(attachment_id, queue_name)
+        queue_log = QueueLog.get_current(queue_name, queue_log_duration)
+        patches_waiting = cls._get_patches_waiting(queue_name)
+        if patches_waiting and queue_log.max_patches_waiting < patches_waiting:
+            queue_log.max_patches_waiting = patches_waiting
+            queue_log.put()
+
+    @classmethod
+    def retrying(cls, attachment_id, queue_name, bot_id=None):
+        patch_log = PatchLog.lookup(attachment_id, queue_name)
+        if bot_id:
+            patch_log.bot_id = bot_id
+        patch_log.retry_count += 1
+        patch_log.put()
+
+        queue_log = QueueLog.get_current(queue_name, queue_log_duration)
+        queue_log.patch_retry_count += 1
+        queue_log.put()
+
+    @classmethod
+    def started(cls, attachment_id, queue_name, bot_id=None):
+        patch_log = PatchLog.lookup(attachment_id, queue_name)
+        if bot_id:
+            patch_log.bot_id = bot_id
+        patch_log.calculate_wait_duration()
+        patch_log.put()
+
+        queue_log = QueueLog.get_current(queue_name, queue_log_duration)
+        queue_log.patch_wait_durations.append(patch_log.wait_duration)
+        queue_log.put()
+
+    @classmethod
+    def stopped(cls, attachment_id, queue_name, bot_id=None):
+        patch_log = PatchLog.lookup(attachment_id, queue_name)
+        if bot_id:
+            patch_log.bot_id = bot_id
+        patch_log.finished = True
+        patch_log.calculate_process_duration()
+        patch_log.put()
+
+        if patch_log.process_duration:
+            queue_log = QueueLog.get_current(queue_name, queue_log_duration)
+            queue_log.patch_process_durations.append(patch_log.process_duration)
+            queue_log.put()
+
+    @classmethod
+    def updated(cls, attachment_id, queue_name, bot_id=None):
+        patch_log = PatchLog.lookup(attachment_id, queue_name)
+        if bot_id:
+            patch_log.bot_id = bot_id
+        patch_log.status_update_count += 1
+        patch_log.put()
+
+        queue_log = QueueLog.get_current(queue_name, queue_log_duration)
+        queue_log.status_update_count += 1
+        queue_log.put()
+
+    @classmethod
+    def _get_patches_waiting(cls, queue_name):
+        work_items = WorkItems.lookup_by_queue(queue_name)
+        active_work_items = ActiveWorkItems.lookup_by_queue(queue_name)
+        if work_items and active_work_items:
+            return len(set(work_items.item_ids) - set(active_work_items.item_ids))
diff --git a/Tools/QueueStatusServer/model/patchlog.py b/Tools/QueueStatusServer/model/patchlog.py
new file mode 100644 (file)
index 0000000..fbb986e
--- /dev/null
@@ -0,0 +1,57 @@
+# Copyright (C) 2013 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from datetime import datetime
+
+from google.appengine.ext import db
+
+
+class PatchLog(db.Model):
+    attachment_id = db.IntegerProperty()
+    queue_name = db.StringProperty()
+    date = db.DateTimeProperty(auto_now_add=True)
+    bot_id = db.StringProperty()
+    retry_count = db.IntegerProperty(default=0)
+    status_update_count = db.IntegerProperty(default=0)
+    finished = db.BooleanProperty(default=False)
+    wait_duration = db.IntegerProperty()
+    process_duration = db.IntegerProperty()
+
+    @classmethod
+    def lookup(cls, attachment_id, queue_name):
+        key = "%s-%s" % (attachment_id, queue_name)
+        return cls.get_or_insert(key, attachment_id=attachment_id, queue_name=queue_name)
+
+    def calculate_wait_duration(self):
+        time_delta = datetime.utcnow() - self.date
+        self.wait_duration = int(time_delta.total_seconds())
+
+    def calculate_process_duration(self):
+        if self.wait_duration:
+            time_delta = datetime.utcnow() - self.date
+            self.process_duration = int(time_delta.total_seconds()) - self.wait_duration
diff --git a/Tools/QueueStatusServer/model/queuelog.py b/Tools/QueueStatusServer/model/queuelog.py
new file mode 100644 (file)
index 0000000..829b8a6
--- /dev/null
@@ -0,0 +1,57 @@
+# Copyright (C) 2013 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from time import time
+from datetime import datetime
+
+from google.appengine.ext import db
+
+
+class QueueLog(db.Model):
+    date = db.DateTimeProperty()
+    # duration specifies in seconds the time period these log values apply to.
+    duration = db.IntegerProperty()
+    queue_name = db.StringProperty()
+    bot_ids_seen = db.StringListProperty()
+    max_patches_waiting = db.IntegerProperty(default=0)
+    patch_wait_durations = db.ListProperty(int)
+    patch_process_durations = db.ListProperty(int)
+    patch_retry_count = db.IntegerProperty(default=0)
+    status_update_count = db.IntegerProperty(default=0)
+
+    @classmethod
+    def get_current(cls, queue_name, duration):
+        timestamp_now = time()
+        timestamp = int(timestamp_now / duration) * duration
+        date = datetime.utcfromtimestamp(timestamp)
+        key = cls.create_key(queue_name, duration, timestamp)
+        return cls.get_or_insert(key, date=date, duration=duration, queue_name=queue_name)
+
+    @staticmethod
+    def create_key(queue_name, duration, timestamp):
+        return "%s-%s-%s" % (queue_name, duration, timestamp)
index 8002f89..0f4d38b 100644 (file)
@@ -1,4 +1,4 @@
-# Copyright (C) 2009 Google Inc. All rights reserved.
+# Copyright (C) 2013 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
@@ -26,6 +26,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+from config import messages
 from google.appengine.ext import db
 from model.queuepropertymixin import QueuePropertyMixin
 
@@ -41,4 +42,4 @@ class QueueStatus(db.Model, QueuePropertyMixin):
     results_file = db.BlobProperty()
 
     def is_retry_request(self):
-        return self.message == "Retry"  # From AbstractQueue._retry_status
+        return self.message == messages.retry_status