Dashboard metrics page should have EWS statistics
authorap@apple.com <ap@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 10 Oct 2014 23:29:52 +0000 (23:29 +0000)
committerap@apple.com <ap@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 10 Oct 2014 23:29:52 +0000 (23:29 +0000)
https://bugs.webkit.org/show_bug.cgi?id=137626

Reviewed by Ryosuke Niwa.

* BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/BubbleQueue.js:
(BubbleQueue): Put shortName in the object, so that it's actually useful.

* BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/BubbleQueueServer.js:
(BubbleQueueServer): Re-ordered queues to match bubble order.
(BubbleQueueServer.prototype.jsonProcessingTimesURL): Added URL for the new handler.
(BubbleQueueServer.prototype.loadProcessingTimes): Load the data from webkit-queues.

* BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsAnalyzer.js:
Added analysis for bubble queues.

* BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsBubbleView.js: Added.

* BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsMain.js:
(buildBubbleQueuesTable): Build the UI.

* BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/metrics.html:
Added JS sources to load.

* QueueStatusServer/app.yaml: To update app version.

* QueueStatusServer/handlers/processingtimesjson.py: Added.

* QueueStatusServer/main.py: Added a handler for processing-times-json.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@174622 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/BubbleQueue.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/BubbleQueueServer.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsAnalyzer.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsBubbleView.js [new file with mode: 0644]
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsMain.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/metrics.html
Tools/ChangeLog
Tools/QueueStatusServer/app.yaml
Tools/QueueStatusServer/handlers/processingtimesjson.py [new file with mode: 0644]
Tools/QueueStatusServer/main.py

index 9c247dd..2c27497 100644 (file)
@@ -32,6 +32,7 @@ BubbleQueue = function(queueServer, id, info)
 
     this.queueServer = queueServer;
     this.id = id;
+    this.shortName = info.shortName || id;
     this.title = info.title || "\xa0";
 
     this.platform = info.platform ? info.platform.name : "unknown";
index b4c4db1..737e9f2 100644 (file)
@@ -28,10 +28,10 @@ BubbleQueueServer = function()
     const queueInfo = {
         "commit-queue": {platform: Dashboard.Platform.MacOSXMountainLion, shortName: "commit", title: "Commit Queue"},
         "style-queue": {shortName: "style", title: "Style Checker Queue"},
+        "gtk-wk2-ews": {platform: Dashboard.Platform.LinuxGTK, shortName: "gtk-wk2", title: "WebKit2\xa0Release\xa0Build\xa0EWS"},
         "mac-ews": {platform: Dashboard.Platform.MacOSXMountainLion, shortName: "mac", title: "WebKit1\xa0Release\xa0Tests\xa0EWS"},
         "mac-wk2-ews": {platform: Dashboard.Platform.MacOSXMountainLion, shortName: "mac-wk2", title: "WebKit2\xa0Release\xa0Tests\xa0EWS"},
         "win-ews": {platform: Dashboard.Platform.Windows7, shortName: "win", title: "WebKit1\xa0Release\xa0Build\xa0EWS"},
-        "gtk-wk2-ews": {platform: Dashboard.Platform.LinuxGTK, shortName: "gtk-wk2", title: "WebKit2\xa0Release\xa0Build\xa0EWS"},
         "efl-wk2-ews": {platform: Dashboard.Platform.LinuxEFL, shortName: "efl-wk2", title: "WebKit2\xa0Release\xa0Build\xa0EWS"}
     };
 
@@ -60,8 +60,39 @@ BubbleQueueServer.prototype = {
         return this.baseURL + "queue-status-json/" + encodeURIComponent(queueID);
     },
 
+    jsonProcessingTimesURL: function(fromTime, toTime)
+    {
+        return this.baseURL + "processing-times-json/" + [fromTime.getUTCFullYear(), fromTime.getUTCMonth() + 1, fromTime.getUTCDate(), fromTime.getUTCHours(), fromTime.getUTCMinutes(), fromTime.getUTCSeconds()].join("-")
+            + "-" + [toTime.getUTCFullYear(), toTime.getUTCMonth() + 1, toTime.getUTCDate(), toTime.getUTCHours(), toTime.getUTCMinutes(), toTime.getUTCSeconds()].join("-");
+    },
+
     queueStatusURL: function(queueID)
     {
         return this.baseURL + "queue-status/" + encodeURIComponent(queueID);
     },
+
+    // Retrieves information about all patches that were submitted in the time range:
+    // {
+    //     patch_id_1: {
+    //         queue_name_1: {
+    //             date: <date/time when the patch was submitted to the queue>,
+    //             retry_count: <number of times a bot had to bail out and drop the lock, for another bot to start from scratch>,
+    //             wait_duration: <how long it took before a bot first locked the patch for processing>,
+    //             process_duration: <how long it took from end of wait to finish, only valid for finished patches. Includes wait time between retries>
+    //             final_message: <(pass|fail|not processed|could not apply|internal error|in progress)>
+    //         },
+    //         ...
+    //     },
+    //     ...
+    // }
+    loadProcessingTimes: function(fromTime, toTime, callback)
+    {
+        JSON.load(this.jsonProcessingTimesURL(fromTime, toTime), function(data) {
+            for (patch in data) {
+                for (queue in patch)
+                    queue.date = new Date(queue.date);
+            }
+            callback(data, fromTime, toTime);
+        }.bind(this));
+    },
 };
index fa40ff6..f1c1c46 100644 (file)
@@ -78,6 +78,8 @@ Analyzer.prototype = {
         }, this);
 
         webkitTrac.load(this._rangeStartTime, this._rangeEndTime);
+
+        bubbleQueueServer.loadProcessingTimes(this._rangeStartTime, this._rangeEndTime, this._loadedBubblesTiming.bind(this));
     },
 
     _triggeringQueue: function(queue)
@@ -380,5 +382,117 @@ Analyzer.prototype = {
 
         if (this._hasTracData)
             this._analyze();
-    }
+    },
+
+    _analyzeBubblePerformance: function(queueID, patches)
+    {
+        var patchesThatCausedInternalError = [];
+        for (patchID in patches) {
+            if (patches[patchID].resolution === "internal error")
+                patchesThatCausedInternalError.push(patchID);
+        }
+
+        var waitTimes = [];
+        var totalTimes = [];
+        var totalTimesForPatchesThatWereNotRetried = [];
+        var totalTimesForPatchesThatSpinnedAndPassedOrFailed = [];
+        var patchesThatDidNotComplete = [];
+        var retryCounts = [];
+        var patchesThatSpinnedAndDidNotComplete = [];
+        var patchesThatSpinnedAndCeasedToApply = [];
+        var patchesThatSpinnedAndPassedOrFailed = [];
+        var patchesDidNotApply = [];
+        for (patchID in patches) {
+            var patch = patches[patchID];
+
+            // Wait time is equally interesting for all patches.
+            waitTimes.push(patch.wait_duration);
+
+            if (patch.resolution === "not processed")
+                patchesThatDidNotComplete.push(patchID);
+
+            if (patch.retry_count === 0)
+                totalTimesForPatchesThatWereNotRetried.push(patch.wait_duration + patch.process_duration);
+            else {
+                retryCounts.push(patch.retry_count);
+                if (patch.resolution === "not processed")
+                    patchesThatSpinnedAndDidNotComplete.push(patchID);
+                else if (patch.resolution === "could not apply")
+                    patchesThatSpinnedAndCeasedToApply.push(patchID);
+                else if (patch.resolution === "pass" || patch.resolution === "fail") {
+                    patchesThatSpinnedAndPassedOrFailed.push(patchID);
+                    totalTimesForPatchesThatSpinnedAndPassedOrFailed.push(patch.wait_duration + patch.process_duration);
+                }
+            }
+
+            // Analyze processing performance for patches that were definitely processed.
+            // We can't target improving performance of others (such as patches that were obsoleted while in the queue).
+            // Patches that don't apply to trunk have to be excluded, because otherwise we
+            // get times for patches that spinned until they ceased to apply.
+            if (patch.resolution === "pass" || patch.resolution === "fail")
+                totalTimes.push(patch.wait_duration + patch.process_duration);
+
+            if (patch.resolution === "could not apply")
+                patchesDidNotApply.push(patchID);
+        }
+
+        var result = {
+            queueID: queueID,
+            totalPatches: Object.keys(patches).length,
+            patchesThatDidNotCompleteCount: patchesThatDidNotComplete.length,
+            patchesWithRetriesCount: retryCounts.length,
+            patchesThatDidNotApplyCount: patchesDidNotApply.length,
+            patchesThatSpinnedAndDidNotCompleteCount: patchesThatSpinnedAndDidNotComplete.length,
+            patchesThatSpinnedAndCeasedToApplyCount: patchesThatSpinnedAndCeasedToApply.length,
+            patchesThatSpinnedAndPassedOrFailedCount: patchesThatSpinnedAndPassedOrFailed.length,
+            medianTotalTimeForPatchesThatSpinnedAndPassedOrFailedInSeconds: totalTimesForPatchesThatSpinnedAndPassedOrFailed.median(),
+            averageTotalTimeForPatchesThatSpinnedAndPassedOrFailedInSeconds: totalTimesForPatchesThatSpinnedAndPassedOrFailed.average(),
+            medianWaitTimeInSeconds: waitTimes.median(),
+            averageWaitTimeInSeconds: waitTimes.average(),
+            patchesThatCausedInternalError: patchesThatCausedInternalError,
+        };
+
+        if (totalTimes.length) {
+            result.medianTotalTimeInSeconds = totalTimes.median();
+            result.averageTotalTimeInSeconds = totalTimes.average();
+        }
+
+        if (totalTimesForPatchesThatWereNotRetried.length) {
+            result.medianTotalTimeForPatchesThatWereNotRetriedInSeconds = totalTimesForPatchesThatWereNotRetried.median();
+            result.averageTotalTimeForPatchesThatWereNotRetriedInSeconds = totalTimesForPatchesThatWereNotRetried.average();
+        }
+
+        this.dispatchEventToListeners(Analyzer.Event.QueueResults, result);
+    },
+
+    _analyzeAllBubblesPerformance: function(dataByPatch)
+    {
+        var data = {};
+        for (queueID in bubbleQueueServer.queues)
+            data[queueID] = {};
+
+        for (patchID in dataByPatch) {
+            for (queueID in dataByPatch[patchID]) {
+                if (!queueID in data)
+                    continue;
+                var patchData = dataByPatch[patchID][queueID];
+                if (patchData.date < this.fromDate || patchData.date > this.toDate)
+                    continue;
+                if (patchData.resolution === "in progress")
+                    continue;
+                data[queueID][patchID] = patchData;
+            };
+        };
+        for (queueID in data)
+            this._analyzeBubblePerformance(queueID, data[queueID]);
+    },
+
+    _loadedBubblesTiming: function(data, fromTime, toTime)
+    {
+        // Only analyze if the data covers the latest range requested by the user.
+        // It may be different from the loaded one if the user quickly requested multiple ranges.
+        if (fromTime > this._rangeStartTime || toTime < this._rangeEndTime)
+            return;
+        this._analyzeAllBubblesPerformance(data);
+    },
 };
diff --git a/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsBubbleView.js b/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsBubbleView.js
new file mode 100644 (file)
index 0000000..d467469
--- /dev/null
@@ -0,0 +1,181 @@
+/*
+ * Copyright (C) 2014 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+MetricsBubbleView = function(analyzer, queue)
+{
+    BaseObject.call(this);
+
+    this.element = document.createElement("div");
+    this.element.classList.add("queue-view");
+
+    this._results = null;
+
+    this._queue = queue;
+
+    analyzer.addEventListener(Analyzer.Event.Starting, this._clearResults, this);
+    analyzer.addEventListener(Analyzer.Event.QueueResults, this._queueResultsAdded, this);
+    
+    this._updateSoon();
+};
+
+MetricsBubbleView.UpdateSoonTimeout = 100; // 100 ms
+
+BaseObject.addConstructorFunctions(MetricsBubbleView);
+
+MetricsBubbleView.prototype = {
+    constructor: MetricsBubbleView,
+    __proto__: BaseObject.prototype,
+
+    _clearResults: function()
+    {
+        this._results = null;
+        this._updateSoon();
+    },
+
+    _queueResultsAdded: function(event)
+    {
+        if (this._queue.id !== event.data.queueID)
+            return;
+
+        this._results = event.data;
+        this._updateSoon();
+    },
+
+    _update: function()
+    {
+        this.element.removeChildren();
+
+        function addLine(element, text)
+        {
+            var line = document.createElement("div");
+            line.textContent = text;
+            line.classList.add("result-line");
+            element.appendChild(line);
+        }
+
+        function addError(element, text)
+        {
+            addLine(element, text);
+            element.lastChild.classList.add("error-line");
+        }
+
+        function addDivider(element)
+        {
+            var divider = document.createElement("div");
+            divider.classList.add("divider");
+            element.appendChild(divider);
+        }
+
+        function pluralizeMinutes(intervalInSeconds)
+        {
+            if (intervalInSeconds < 60)
+                return "less than a minute";
+            else if (intervalInSeconds < 120)
+                return "1\xa0minute";
+            else
+                return Math.round(intervalInSeconds / 60) + "\xa0minutes";
+        }
+
+        function formatPercentage(fraction)
+        {
+            if (fraction > 0 && fraction < 0.01)
+                return "< 1%";
+            else
+                return Math.round(fraction * 100) + "%";
+        }
+
+        if (!this._results)
+            return;
+
+        if (this._queue.id === "style-queue") {
+            addDivider(this.element);
+            addLine(this.element, "Time to result:");
+            addLine(this.element, "- median: " + pluralizeMinutes(this._results.medianTotalTimeInSeconds) + ";");
+            addLine(this.element, "- average: " + pluralizeMinutes(this._results.averageTotalTimeInSeconds) + ".");
+        } else {
+            if (this._results.totalPatches !== this._results.patchesWithRetriesCount) {
+                addDivider(this.element);
+                var numberOfPatchesThatHadFinalResultsAtFirstTry = this._results.totalPatches - this._results.patchesWithRetriesCount - this._results.patchesThatDidNotCompleteCount + this._results.patchesThatSpinnedAndDidNotCompleteCount;
+                var text = formatPercentage(numberOfPatchesThatHadFinalResultsAtFirstTry / this._results.totalPatches) + " of patches ";
+                text += (this._queue.id === "commit-queue") ? "were landed or rejected" : "had final results";
+                text += " at first try. Time to result:"
+                addLine(this.element, text);
+                addLine(this.element, "- median: " + pluralizeMinutes(this._results.medianTotalTimeForPatchesThatWereNotRetriedInSeconds) + ";");
+                addLine(this.element, "- average: " + pluralizeMinutes(this._results.averageTotalTimeForPatchesThatWereNotRetriedInSeconds) + ".");
+                if (this._results.patchesThatDidNotApplyCount !== this._results.patchesThatSpinnedAndCeasedToApplyCount) {
+                    if (this._results.patchesThatDidNotApplyCount - this._results.patchesThatSpinnedAndCeasedToApplyCount === numberOfPatchesThatHadFinalResultsAtFirstTry)
+                        addLine(this.element, "None of these applied to trunk.");
+                    else
+                        addLine(this.element, "This includes " + formatPercentage((this._results.patchesThatDidNotApplyCount - this._results.patchesThatSpinnedAndCeasedToApplyCount) / this._results.totalPatches) + " that did not apply to trunk.");
+                }
+            }
+
+            if (this._results.patchesThatDidNotCompleteCount !== this._results.patchesThatSpinnedAndDidNotCompleteCount) {
+                addDivider(this.element);
+                addLine(this.element, formatPercentage((this._results.patchesThatDidNotCompleteCount - this._results.patchesThatSpinnedAndDidNotCompleteCount) / this._results.totalPatches) + " of patches ceased to be eligible for processing before the first try finished.");
+            }
+
+            if (this._results.patchesWithRetriesCount) {
+                addDivider(this.element);
+                var text = formatPercentage(this._results.patchesWithRetriesCount / this._results.totalPatches) + " of patches had to be retried";
+                if (this._results.patchesThatSpinnedAndDidNotCompleteCount) {
+                    text += ", including " + formatPercentage(this._results.patchesThatSpinnedAndDidNotCompleteCount / this._results.totalPatches) + " that kept being retried until the patch became ineligible for processing";
+                    if (this._results.patchesThatSpinnedAndCeasedToApplyCount)
+                        text += ", and " + formatPercentage(this._results.patchesThatSpinnedAndCeasedToApplyCount / this._results.totalPatches) + " that kept being retried until the patch ceased to apply to trunk.";
+                    else
+                        text += ".";
+                } else if (this._results.patchesThatSpinnedAndCeasedToApplyCount)
+                    text += ", including " + formatPercentage(this._results.patchesThatSpinnedAndCeasedToApplyCount / this._results.totalPatches) + " that were spinning until the patch ceased to apply to trunk.";
+                else
+                    text += ".";
+                if (this._results.patchesThatSpinnedAndPassedOrFailedCount) {
+                    text += (this._results.patchesWithRetriesCount === this._results.patchesThatSpinnedAndPassedOrFailedCount) ? " All of them" : " " + formatPercentage(this._results.patchesThatSpinnedAndPassedOrFailedCount / this._results.totalPatches);
+                    text += " finally produced a result, which took:";
+                    addLine(this.element, text);
+                    addLine(this.element, "- median: " + pluralizeMinutes(this._results.medianTotalTimeForPatchesThatSpinnedAndPassedOrFailedInSeconds) + ";");
+                    addLine(this.element, "- average: " + pluralizeMinutes(this._results.averageTotalTimeForPatchesThatSpinnedAndPassedOrFailedInSeconds) + ".");
+                } else
+                    addLine(this.element, text);
+            }
+        }
+
+        addDivider(this.element);
+        addLine(this.element, "Median wait time before processing started: " + pluralizeMinutes(this._results.medianWaitTimeInSeconds) + ".");
+
+        if (this._results.patchesThatCausedInternalError.length) {
+            addDivider(this.element);
+            if (this._results.patchesThatCausedInternalError.length === 1)
+                addError(this.element, "One patch caused internal error");
+            else
+                addError(this.element, this._results.patchesThatCausedInternalError.length + "\xa0patches caused internal error, please see patch numbers in console log.");
+            console.log("Patches that caused internal error for " + this._results.queueID + ": " + this._results.patchesThatCausedInternalError);
+        }
+    },
+
+    _updateSoon: function()
+    {
+        setTimeout(this._update.bind(this), MetricsBubbleView.UpdateSoonTimeout);
+    },
+};
index 88ebf3c..6dbf38e 100644 (file)
@@ -23,6 +23,8 @@
  * THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+var hasBubbles = typeof bubbleQueueServer != "undefined";
+
 var analyzer = new Analyzer;
 
 var allBuilderResultsPseudoQueue = { id: "allBuilderResultsPseudoQueue" };
@@ -260,6 +262,39 @@ function buildQueuesTable()
     return table;
 }
 
+function buildBubbleQueuesTable()
+{
+    var table = document.createElement("table");
+    table.classList.add("queue-grid");
+
+    var row = document.createElement("tr");
+    row.classList.add("headers");
+
+    for (id in bubbleQueueServer.queues) {
+        var header = document.createElement("th");
+        header.textContent = bubbleQueueServer.queues[id].shortName;
+        row.appendChild(header);
+    }
+
+    table.appendChild(row);
+
+    row = document.createElement("tr");
+    row.classList.add("platform");
+
+    for (id in bubbleQueueServer.queues) {
+        var cell = document.createElement("td");
+        var view = new MetricsBubbleView(analyzer, bubbleQueueServer.queues[id]);
+        cell.appendChild(view.element);
+        row.appendChild(cell);
+    }
+
+    table.appendChild(row);
+
+    document.body.appendChild(table);
+
+    return table;
+}
+
 function documentReady()
 {
     var rangePicker = document.createElement("span");
@@ -302,6 +337,12 @@ function documentReady()
     }, this);
 
     buildAggregateTable();
+    if (hasBubbles) {
+        var tablesDivider = document.createElement("div");
+        tablesDivider.classList.add("tables-divider");
+        document.body.appendChild(tablesDivider);
+        buildBubbleQueuesTable();
+    }
 
     var tablesDivider = document.createElement("div");
     tablesDivider.classList.add("tables-divider");
index f701826..f89561a 100644 (file)
@@ -36,8 +36,10 @@ THE POSSIBILITY OF SUCH DAMAGE.
     <script src="Scripts/BaseObject.js"></script>
     <script src="Scripts/Dashboard.js"></script>
     <script src="Scripts/Buildbot.js"></script>
+    <script src="Scripts/BubbleQueueServer.js"></script>
     <script src="Scripts/WebKitBuildbot.js"></script>
     <script src="Scripts/BuildbotQueue.js"></script>
+    <script src="Scripts/BubbleQueue.js"></script>
     <script src="Scripts/BuildbotIteration.js"></script>
     <script src="Scripts/BuildbotTestResults.js"></script>
     <script src="Scripts/Settings.js"></script>
@@ -45,6 +47,7 @@ THE POSSIBILITY OF SUCH DAMAGE.
     <script src="Scripts/Initialization.js"></script>
     <script src="Scripts/MetricsAnalyzer.js"></script>
     <script src="Scripts/MetricsView.js"></script>
+    <script src="Scripts/MetricsBubbleView.js"></script>
     <script src="External/jquery-1.11.1.min.js"></script>
     <script src="External/moment.min.js"></script>
     <script src="External/jquery.daterangepicker.js"></script>
index 3174289..2010f52 100644 (file)
@@ -1,5 +1,37 @@
 2014-10-10  Alexey Proskuryakov  <ap@apple.com>
 
+        Dashboard metrics page should have EWS statistics
+        https://bugs.webkit.org/show_bug.cgi?id=137626
+
+        Reviewed by Ryosuke Niwa.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/BubbleQueue.js:
+        (BubbleQueue): Put shortName in the object, so that it's actually useful.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/BubbleQueueServer.js:
+        (BubbleQueueServer): Re-ordered queues to match bubble order.
+        (BubbleQueueServer.prototype.jsonProcessingTimesURL): Added URL for the new handler.
+        (BubbleQueueServer.prototype.loadProcessingTimes): Load the data from webkit-queues.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsAnalyzer.js:
+        Added analysis for bubble queues.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsBubbleView.js: Added.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsMain.js:
+        (buildBubbleQueuesTable): Build the UI.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/metrics.html:
+        Added JS sources to load.
+
+        * QueueStatusServer/app.yaml: To update app version.
+
+        * QueueStatusServer/handlers/processingtimesjson.py: Added.
+
+        * QueueStatusServer/main.py: Added a handler for processing-times-json.
+
+2014-10-10  Alexey Proskuryakov  <ap@apple.com>
+
         REGRESSION: Commit queue doesn't process rollouts
         https://bugs.webkit.org/show_bug.cgi?id=137623
 
index 13e9012..9150704 100644 (file)
@@ -1,5 +1,5 @@
 application: webkit-queues
-version: 174611 # SVN revision of last major change
+version: 174622 # SVN revision of last major change
 runtime: python
 api_version: 1
 
diff --git a/Tools/QueueStatusServer/handlers/processingtimesjson.py b/Tools/QueueStatusServer/handlers/processingtimesjson.py
new file mode 100644 (file)
index 0000000..eee99a7
--- /dev/null
@@ -0,0 +1,71 @@
+# Copyright (C) 2014 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1.  Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+# 2.  Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import datetime
+
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp import template
+
+from model.patchlog import PatchLog
+
+# Fall back to simplejson, because we are still on Python 2.5.
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+
+class ProcessingTimesJSON(webapp.RequestHandler):
+    def _resultFromFinalStatus(self, status_message, queue_name):
+        if status_message == "Pass":
+            return "pass"
+        elif status_message == "Fail":
+            return "fail"
+        elif status_message == "Error: " + queue_name + " did not process patch.":
+            return "not processed"
+        elif status_message == "Error: " + queue_name + " unable to apply patch.":
+            return "could not apply"
+        else:
+            return "internal error"
+
+    def _fetch_patch_log(self, start_date, end_date):
+        all_entries = PatchLog.all().filter('date >', start_date).filter('date <', end_date).fetch(limit=None)
+        result = {}
+        for entry in all_entries:
+            result.setdefault(entry.attachment_id, {})
+            result[entry.attachment_id][entry.queue_name] = {
+                "date": entry.date,
+                "wait_duration": entry.wait_duration,
+                "process_duration": entry.process_duration,
+                "retry_count": entry.retry_count,
+                "resolution": self._resultFromFinalStatus(entry.latest_message, entry.queue_name) if entry.finished else "in progress"
+            }
+        return result
+
+    def get(self, start_year, start_month, start_day, start_hour, start_minute, start_second, end_year, end_month, end_day, end_hour, end_minute, end_second):
+        self.response.headers["Access-Control-Allow-Origin"] = "*"
+        self.response.headers['Content-Type'] = 'application/json'
+
+        patch_log = self._fetch_patch_log(datetime.datetime(int(start_year), int(start_month), int(start_day), int(start_hour), int(start_minute), int(start_second)),
+            datetime.datetime(int(end_year), int(end_month), int(end_day), int(end_hour), int(end_minute), int(end_second)))
+        dthandler = lambda obj: (obj.isoformat() + "Z") if isinstance(obj, datetime.datetime) or isinstance(obj, datetime.date) else None
+        self.response.out.write(json.dumps(patch_log, default=dthandler))
index 2892819..a16261e 100644 (file)
@@ -1,4 +1,5 @@
 # Copyright (C) 2009 Google Inc. All rights reserved.
+# Copyright (C) 2014 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
@@ -38,6 +39,7 @@ from handlers.gc import GC
 from handlers.nextpatch import NextPatch
 from handlers.patch import Patch
 from handlers.patchstatus import PatchStatus
+from handlers.processingtimesjson import ProcessingTimesJSON
 from handlers.queuecharts import QueueCharts
 from handlers.queuelengthjson import QueueLengthJSON
 from handlers.queuestatus import QueueStatus
@@ -79,6 +81,7 @@ routes = [
     ('/update-work-items', UpdateWorkItems),
     ('/update-svn-revision', UpdateSVNRevision),
     ('/active-bots', ActiveBots),
+    (r'/processing-times-json/(\d+)\-(\d+)\-(\d+)\-(\d+)\-(\d+)\-(\d+)\-(\d+)\-(\d+)\-(\d+)\-(\d+)\-(\d+)\-(\d+)', ProcessingTimesJSON),
 ]
 
 application = webapp.WSGIApplication(routes, debug=True)