Resource Load Statistics: Shortcut classification for redirect to prevalent resource
authorwilander@apple.com <wilander@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 14 Jun 2018 20:06:15 +0000 (20:06 +0000)
committerwilander@apple.com <wilander@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 14 Jun 2018 20:06:15 +0000 (20:06 +0000)
https://bugs.webkit.org/show_bug.cgi?id=186627
<rdar://problem/41132308>

Reviewed by Brent Fulgham.

Source/WebKit:

This patch shortcuts classification of redirect collusion so that we more seldom
have to rely on the recursive backtrace of the redirect graph. The initial
implementation of Resource Load Statistics actually had this classification method.

* UIProcess/WebResourceLoadStatisticsStore.cpp:
(WebKit::WebResourceLoadStatisticsStore::markAsPrevalentIfHasRedirectedToPrevalent):
    Iterates through a non-classified resource's data for where it has redirected
    and classifies it as prevalent if has redirected to
(WebKit::WebResourceLoadStatisticsStore::processStatisticsAndDataRecords):
    Now calls WebResourceLoadStatisticsStore::markAsPrevalentIfHasRedirectedToPrevalent()
    before regular classification steps.
* UIProcess/WebResourceLoadStatisticsStore.h:

LayoutTests:

* http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-subresource-redirect-to-prevalent-expected.txt: Added.
* http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-subresource-redirect-to-prevalent.html: Added.
* http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-top-frame-redirect-to-prevalent-expected.txt: Added.
* http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-top-frame-redirect-to-prevalent.html: Added.
* platform/wk2/TestExpectations:
    New tests marked as [ Pass ].

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@232850 268f45cc-cd09-0410-ab3c-d52691b4dbfc

LayoutTests/ChangeLog
LayoutTests/http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-subresource-redirect-to-prevalent-expected.txt [new file with mode: 0644]
LayoutTests/http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-subresource-redirect-to-prevalent.html [new file with mode: 0644]
LayoutTests/http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-top-frame-redirect-to-prevalent-expected.txt [new file with mode: 0644]
LayoutTests/http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-top-frame-redirect-to-prevalent.html [new file with mode: 0644]
LayoutTests/platform/wk2/TestExpectations
Source/WebKit/ChangeLog
Source/WebKit/UIProcess/WebResourceLoadStatisticsStore.cpp
Source/WebKit/UIProcess/WebResourceLoadStatisticsStore.h

index cfab066..ca92a88 100644 (file)
@@ -1,3 +1,18 @@
+2018-06-14  John Wilander  <wilander@apple.com>
+
+        Resource Load Statistics: Shortcut classification for redirect to prevalent resource
+        https://bugs.webkit.org/show_bug.cgi?id=186627
+        <rdar://problem/41132308>
+
+        Reviewed by Brent Fulgham.
+
+        * http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-subresource-redirect-to-prevalent-expected.txt: Added.
+        * http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-subresource-redirect-to-prevalent.html: Added.
+        * http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-top-frame-redirect-to-prevalent-expected.txt: Added.
+        * http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-top-frame-redirect-to-prevalent.html: Added.
+        * platform/wk2/TestExpectations:
+            New tests marked as [ Pass ].
+
 2018-06-14  Youenn Fablet  <youenn@apple.com>
 
         Apply CSP checks before Content blocker checks in NetworkLoadChecker as done by CachedResourceLoader
diff --git a/LayoutTests/http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-subresource-redirect-to-prevalent-expected.txt b/LayoutTests/http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-subresource-redirect-to-prevalent-expected.txt
new file mode 100644 (file)
index 0000000..b8fcbd8
--- /dev/null
@@ -0,0 +1,10 @@
+Tests for classification based on subresource redirect to other prevalent.
+
+On success, you will see a series of "PASS" messages, followed by "TEST COMPLETE".
+
+
+PASS Host classified as prevalent resource.
+PASS successfullyParsed is true
+
+TEST COMPLETE
+
diff --git a/LayoutTests/http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-subresource-redirect-to-prevalent.html b/LayoutTests/http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-subresource-redirect-to-prevalent.html
new file mode 100644 (file)
index 0000000..bdd4731
--- /dev/null
@@ -0,0 +1,54 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <script src="/js-test-resources/js-test.js"></script>
+</head>
+<body>
+<script>
+    description("Tests for classification based on subresource redirect to other prevalent.");
+    jsTestIsAsync = true;
+
+    const hostUnderTest = "127.0.0.1:8000";
+    const statisticsUrl = "http://" + hostUnderTest + "/temp";
+    const topFrameOrigin1 = "http://127.0.0.2:8000/temp";
+
+    function setEnableFeature(enable) {
+        if (!enable)
+            testRunner.statisticsResetToConsistentState();
+        internals.setResourceLoadStatisticsEnabled(enable);
+        testRunner.setCookieStoragePartitioningEnabled(enable);
+    }
+
+    function completeTest() {
+        if (!testRunner.isStatisticsPrevalentResource(statisticsUrl))
+            testFailed("Host did not get classified as prevalent resource.");
+        else if (testRunner.isStatisticsVeryPrevalentResource(statisticsUrl))
+            testFailed("Host got classified as very prevalent resource.");
+        else
+            testPassed("Host classified as prevalent resource.");
+
+        setEnableFeature(false);
+        finishJSTest();
+    }
+
+    function runTest() {
+        testRunner.setStatisticsPrevalentResource(topFrameOrigin1, true);
+        testRunner.setStatisticsSubresourceUniqueRedirectTo(statisticsUrl, topFrameOrigin1);
+
+        testRunner.installStatisticsDidScanDataRecordsCallback(completeTest);
+        testRunner.statisticsProcessStatisticsAndDataRecords();
+    }
+
+    if (document.location.host === hostUnderTest && window.testRunner && window.internals) {
+        setEnableFeature(true);
+        testRunner.setStatisticsNotifyPagesWhenDataRecordsWereScanned(true);
+
+        testRunner.setStatisticsPrevalentResource(statisticsUrl, false);
+        if (testRunner.isStatisticsPrevalentResource(statisticsUrl))
+            testFailed("Host did not get set as non-prevalent resource.");
+        runTest();
+    }
+</script>
+</body>
+</html>
diff --git a/LayoutTests/http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-top-frame-redirect-to-prevalent-expected.txt b/LayoutTests/http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-top-frame-redirect-to-prevalent-expected.txt
new file mode 100644 (file)
index 0000000..59f9953
--- /dev/null
@@ -0,0 +1,10 @@
+Tests for classification based on top frame redirect to other prevalent.
+
+On success, you will see a series of "PASS" messages, followed by "TEST COMPLETE".
+
+
+PASS Host classified as prevalent resource.
+PASS successfullyParsed is true
+
+TEST COMPLETE
+
diff --git a/LayoutTests/http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-top-frame-redirect-to-prevalent.html b/LayoutTests/http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-top-frame-redirect-to-prevalent.html
new file mode 100644 (file)
index 0000000..9866087
--- /dev/null
@@ -0,0 +1,54 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <script src="/js-test-resources/js-test.js"></script>
+</head>
+<body>
+<script>
+    description("Tests for classification based on top frame redirect to other prevalent.");
+    jsTestIsAsync = true;
+
+    const hostUnderTest = "127.0.0.1:8000";
+    const statisticsUrl = "http://" + hostUnderTest + "/temp";
+    const topFrameOrigin1 = "http://127.0.0.2:8000/temp";
+
+    function setEnableFeature(enable) {
+        if (!enable)
+            testRunner.statisticsResetToConsistentState();
+        internals.setResourceLoadStatisticsEnabled(enable);
+        testRunner.setCookieStoragePartitioningEnabled(enable);
+    }
+
+    function completeTest() {
+        if (!testRunner.isStatisticsPrevalentResource(statisticsUrl))
+            testFailed("Host did not get classified as prevalent resource.");
+        else if (testRunner.isStatisticsVeryPrevalentResource(statisticsUrl))
+            testFailed("Host got classified as very prevalent resource.");
+        else
+            testPassed("Host classified as prevalent resource.");
+
+        setEnableFeature(false);
+        finishJSTest();
+    }
+
+    function runTest() {
+        testRunner.setStatisticsPrevalentResource(topFrameOrigin1, true);
+        testRunner.setStatisticsTopFrameUniqueRedirectTo(statisticsUrl, topFrameOrigin1);
+
+        testRunner.installStatisticsDidScanDataRecordsCallback(completeTest);
+        testRunner.statisticsProcessStatisticsAndDataRecords();
+    }
+
+    if (document.location.host === hostUnderTest && window.testRunner && window.internals) {
+        setEnableFeature(true);
+        testRunner.setStatisticsNotifyPagesWhenDataRecordsWereScanned(true);
+
+        testRunner.setStatisticsPrevalentResource(statisticsUrl, false);
+        if (testRunner.isStatisticsPrevalentResource(statisticsUrl))
+            testFailed("Host did not get set as non-prevalent resource.");
+        runTest();
+    }
+</script>
+</body>
+</html>
index 48a7876..b87ddb0 100644 (file)
@@ -705,7 +705,9 @@ http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-sub-frame-under
 http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-subresource-under-top-frame-origins.html [ Pass ]
 http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-subresource-unique-redirects-to.html [ Pass ]
 http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-subresource-redirect-collusion.html [ Pass ]
+http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-subresource-redirect-to-prevalent.html [ Pass ]
 http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-top-frame-redirect-collusion.html [ Pass ]
+http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-top-frame-redirect-to-prevalent.html [ Pass ]
 http/tests/resourceLoadStatistics/classify-as-prevalent-based-on-top-frame-unique-redirects-to.html [ Pass ]
 http/tests/resourceLoadStatistics/clear-in-memory-and-persistent-store.html [ Pass ]
 http/tests/resourceLoadStatistics/grandfathering.html [ Pass ]
index 8e35d57..4df9019 100644 (file)
@@ -1,3 +1,24 @@
+2018-06-14  John Wilander  <wilander@apple.com>
+
+        Resource Load Statistics: Shortcut classification for redirect to prevalent resource
+        https://bugs.webkit.org/show_bug.cgi?id=186627
+        <rdar://problem/41132308>
+
+        Reviewed by Brent Fulgham.
+
+        This patch shortcuts classification of redirect collusion so that we more seldom
+        have to rely on the recursive backtrace of the redirect graph. The initial
+        implementation of Resource Load Statistics actually had this classification method.
+
+        * UIProcess/WebResourceLoadStatisticsStore.cpp:
+        (WebKit::WebResourceLoadStatisticsStore::markAsPrevalentIfHasRedirectedToPrevalent):
+            Iterates through a non-classified resource's data for where it has redirected
+            and classifies it as prevalent if has redirected to 
+        (WebKit::WebResourceLoadStatisticsStore::processStatisticsAndDataRecords):
+            Now calls WebResourceLoadStatisticsStore::markAsPrevalentIfHasRedirectedToPrevalent()
+            before regular classification steps. 
+        * UIProcess/WebResourceLoadStatisticsStore.h:
+
 2018-06-14  Youenn Fablet  <youenn@apple.com>
 
         Apply CSP checks before Content blocker checks in NetworkLoadChecker as done by CachedResourceLoader
index 06ef9ff..6dab263 100644 (file)
@@ -296,6 +296,30 @@ unsigned WebResourceLoadStatisticsStore::recursivelyGetAllDomainsThatHaveRedirec
     return numberOfRecursiveCalls;
 }
 
+void WebResourceLoadStatisticsStore::markAsPrevalentIfHasRedirectedToPrevalent(WebCore::ResourceLoadStatistics& resourceStatistic)
+{
+    ASSERT(!RunLoop::isMain());
+
+    if (resourceStatistic.isPrevalentResource)
+        return;
+    
+    for (auto& subresourceDomainRedirectedTo : resourceStatistic.subresourceUniqueRedirectsTo.values()) {
+        auto mapEntry = m_resourceStatisticsMap.find(subresourceDomainRedirectedTo);
+        if (mapEntry != m_resourceStatisticsMap.end() && mapEntry->value.isPrevalentResource) {
+            setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
+            return;
+        }
+    }
+
+    for (auto& topFrameDomainRedirectedTo : resourceStatistic.topFrameUniqueRedirectsTo.values()) {
+        auto mapEntry = m_resourceStatisticsMap.find(topFrameDomainRedirectedTo);
+        if (mapEntry != m_resourceStatisticsMap.end() && mapEntry->value.isPrevalentResource) {
+            setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
+            return;
+        }
+    }
+}
+
 void WebResourceLoadStatisticsStore::processStatisticsAndDataRecords()
 {
     ASSERT(!RunLoop::isMain());
@@ -303,6 +327,7 @@ void WebResourceLoadStatisticsStore::processStatisticsAndDataRecords()
     if (m_parameters.shouldClassifyResourcesBeforeDataRecordsRemoval) {
         for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
             if (!resourceStatistic.isVeryPrevalentResource) {
+                markAsPrevalentIfHasRedirectedToPrevalent(resourceStatistic);
                 auto currentPrevalence = resourceStatistic.isPrevalentResource ? ResourceLoadPrevalence::High : ResourceLoadPrevalence::Low;
                 auto newPrevalence = m_resourceLoadStatisticsClassifier.calculateResourcePrevalence(resourceStatistic, currentPrevalence);
                 if (newPrevalence != currentPrevalence)
index 1fa8646..4f11624 100644 (file)
@@ -185,6 +185,7 @@ private:
     void mergeStatistics(Vector<WebCore::ResourceLoadStatistics>&&);
     WebCore::ResourceLoadStatistics& ensureResourceStatisticsForPrimaryDomain(const String&);
     unsigned recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(const WebCore::ResourceLoadStatistics&, HashSet<String>& domainsThatHaveRedirectedTo, unsigned numberOfRecursiveCalls);
+    void markAsPrevalentIfHasRedirectedToPrevalent(WebCore::ResourceLoadStatistics&);
     void setPrevalentResource(WebCore::ResourceLoadStatistics&, ResourceLoadPrevalence);
     void processStatisticsAndDataRecords();