Add way to dump cache meta data to file
[WebKit-https.git] / Source / WebKit2 / NetworkProcess / cache / NetworkCache.cpp
1 /*
2  * Copyright (C) 2014 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "NetworkCache.h"
28
29 #if ENABLE(NETWORK_CACHE)
30
31 #include "Logging.h"
32 #include "NetworkCacheCoders.h"
33 #include "NetworkCacheStatistics.h"
34 #include "NetworkCacheStorage.h"
35 #include "NetworkResourceLoader.h"
36 #include "WebCoreArgumentCoders.h"
37 #include <JavaScriptCore/JSONObject.h>
38 #include <WebCore/CacheValidation.h>
39 #include <WebCore/FileSystem.h>
40 #include <WebCore/HTTPHeaderNames.h>
41 #include <WebCore/ResourceResponse.h>
42 #include <WebCore/SharedBuffer.h>
43 #include <wtf/NeverDestroyed.h>
44 #include <wtf/StringHasher.h>
45 #include <wtf/text/StringBuilder.h>
46
47 #if PLATFORM(COCOA)
48 #include <notify.h>
49 #endif
50
51 namespace WebKit {
52
53 NetworkCache& NetworkCache::singleton()
54 {
55     static NeverDestroyed<NetworkCache> instance;
56     return instance;
57 }
58
59 bool NetworkCache::initialize(const String& cachePath, bool enableEfficacyLogging)
60 {
61     m_storage = NetworkCacheStorage::open(cachePath);
62
63     if (enableEfficacyLogging)
64         m_statistics = NetworkCacheStatistics::open(cachePath);
65
66 #if PLATFORM(COCOA)
67     // Triggers with "notifyutil -p com.apple.WebKit.Cache.dump".
68     if (m_storage) {
69         int token;
70         notify_register_dispatch("com.apple.WebKit.Cache.dump", &token, dispatch_get_main_queue(), ^(int) {
71             dumpContentsToFile();
72         });
73     }
74 #endif
75
76     LOG(NetworkCache, "(NetworkProcess) opened cache storage, success %d", !!m_storage);
77     return !!m_storage;
78 }
79
80 void NetworkCache::setMaximumSize(size_t maximumSize)
81 {
82     if (!m_storage)
83         return;
84     m_storage->setMaximumSize(maximumSize);
85 }
86
87 static NetworkCacheStorage::Entry encodeStorageEntry(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, PassRefPtr<WebCore::SharedBuffer> responseData)
88 {
89     NetworkCacheEncoder encoder;
90     encoder << response;
91
92     String varyValue = response.httpHeaderField(WebCore::HTTPHeaderName::Vary);
93     bool hasVaryingRequestHeaders = !varyValue.isEmpty();
94
95     encoder << hasVaryingRequestHeaders;
96
97     if (hasVaryingRequestHeaders) {
98         Vector<String> varyingHeaderNames;
99         varyValue.split(',', false, varyingHeaderNames);
100
101         Vector<std::pair<String, String>> varyingRequestHeaders;
102         for (auto& varyHeaderName : varyingHeaderNames) {
103             String headerName = varyHeaderName.stripWhiteSpace();
104             varyingRequestHeaders.append(std::make_pair(headerName, request.httpHeaderField(headerName)));
105         }
106         encoder << varyingRequestHeaders;
107     }
108     encoder.encodeChecksum();
109
110     auto timeStamp = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch());
111     NetworkCacheStorage::Data header(encoder.buffer(), encoder.bufferSize());
112     NetworkCacheStorage::Data body;
113     if (responseData)
114         body = NetworkCacheStorage::Data(reinterpret_cast<const uint8_t*>(responseData->data()), responseData->size());
115
116     return NetworkCacheStorage::Entry { timeStamp, header, body };
117 }
118
119 static bool verifyVaryingRequestHeaders(const Vector<std::pair<String, String>>& varyingRequestHeaders, const WebCore::ResourceRequest& request)
120 {
121     for (auto& varyingRequestHeader : varyingRequestHeaders) {
122         // FIXME: Vary: * in response would ideally trigger a cache delete instead of a store.
123         if (varyingRequestHeader.first == "*")
124             return false;
125         String requestValue = request.httpHeaderField(varyingRequestHeader.first);
126         if (requestValue != varyingRequestHeader.second)
127             return false;
128     }
129     return true;
130 }
131
132 static bool cachePolicyAllowsExpired(WebCore::ResourceRequestCachePolicy policy)
133 {
134     switch (policy) {
135     case WebCore::ReturnCacheDataElseLoad:
136     case WebCore::ReturnCacheDataDontLoad:
137         return true;
138     case WebCore::UseProtocolCachePolicy:
139     case WebCore::ReloadIgnoringCacheData:
140         return false;
141     }
142     ASSERT_NOT_REACHED();
143     return false;
144 }
145
146 static std::unique_ptr<NetworkCache::Entry> decodeStorageEntry(const NetworkCacheStorage::Entry& storageEntry, const WebCore::ResourceRequest& request, NetworkCache::CachedEntryReuseFailure& failure)
147 {
148     NetworkCacheDecoder decoder(storageEntry.header.data(), storageEntry.header.size());
149
150     WebCore::ResourceResponse cachedResponse;
151     if (!decoder.decode(cachedResponse)) {
152         LOG(NetworkCache, "(NetworkProcess) response decoding failed\n");
153         failure = NetworkCache::CachedEntryReuseFailure::Other;
154         return nullptr;
155     }
156
157     bool hasVaryingRequestHeaders;
158     if (!decoder.decode(hasVaryingRequestHeaders)) {
159         failure = NetworkCache::CachedEntryReuseFailure::Other;
160         return nullptr;
161     }
162
163     if (hasVaryingRequestHeaders) {
164         Vector<std::pair<String, String>> varyingRequestHeaders;
165         if (!decoder.decode(varyingRequestHeaders)) {
166             failure = NetworkCache::CachedEntryReuseFailure::Other;
167             return nullptr;
168         }
169
170         if (!verifyVaryingRequestHeaders(varyingRequestHeaders, request)) {
171             LOG(NetworkCache, "(NetworkProcess) varying header mismatch\n");
172             failure = NetworkCache::CachedEntryReuseFailure::VaryingHeaderMismatch;
173             return nullptr;
174         }
175     }
176     if (!decoder.verifyChecksum()) {
177         LOG(NetworkCache, "(NetworkProcess) checksum verification failure\n");
178         failure = NetworkCache::CachedEntryReuseFailure::Other;
179         return nullptr;
180     }
181
182     bool allowExpired = cachePolicyAllowsExpired(request.cachePolicy()) && !cachedResponse.cacheControlContainsMustRevalidate();
183     auto timeStamp = std::chrono::duration_cast<std::chrono::duration<double>>(storageEntry.timeStamp);
184     double age = WebCore::computeCurrentAge(cachedResponse, timeStamp.count());
185     double lifetime = WebCore::computeFreshnessLifetimeForHTTPFamily(cachedResponse, timeStamp.count());
186     bool isExpired = age > lifetime;
187     bool needsRevalidation = (isExpired && !allowExpired) || cachedResponse.cacheControlContainsNoCache();
188
189     if (needsRevalidation) {
190         bool hasValidatorFields = cachedResponse.hasCacheValidatorFields();
191         LOG(NetworkCache, "(NetworkProcess) needsRevalidation hasValidatorFields=%d isExpired=%d age=%f lifetime=%f", isExpired, hasValidatorFields, age, lifetime);
192         if (!hasValidatorFields) {
193             failure = NetworkCache::CachedEntryReuseFailure::MissingValidatorFields;
194             return nullptr;
195         }
196     }
197
198     auto entry = std::make_unique<NetworkCache::Entry>();
199     entry->storageEntry = storageEntry;
200     entry->needsRevalidation = needsRevalidation;
201
202     cachedResponse.setSource(needsRevalidation ? WebCore::ResourceResponse::Source::DiskCacheAfterValidation : WebCore::ResourceResponse::Source::DiskCache);
203     entry->response = cachedResponse;
204
205 #if ENABLE(SHAREABLE_RESOURCE)
206     RefPtr<SharedMemory> sharedMemory = storageEntry.body.isMap() ? SharedMemory::createFromVMBuffer(const_cast<uint8_t*>(storageEntry.body.data()), storageEntry.body.size()) : nullptr;
207     RefPtr<ShareableResource> shareableResource = sharedMemory ? ShareableResource::create(sharedMemory.release(), 0, storageEntry.body.size()) : nullptr;
208
209     if (shareableResource && shareableResource->createHandle(entry->shareableResourceHandle))
210         entry->buffer = entry->shareableResourceHandle.tryWrapInSharedBuffer();
211     else
212 #endif
213         entry->buffer = WebCore::SharedBuffer::create(storageEntry.body.data(), storageEntry.body.size());
214
215     return entry;
216 }
217
218 static NetworkCache::RetrieveDecision canRetrieve(const WebCore::ResourceRequest& request)
219 {
220     if (!request.url().protocolIsInHTTPFamily())
221         return NetworkCache::RetrieveDecision::NoDueToProtocol;
222     // FIXME: Support HEAD and OPTIONS requests.
223     if (request.httpMethod() != "GET")
224         return NetworkCache::RetrieveDecision::NoDueToHTTPMethod;
225     // FIXME: We should be able to validate conditional requests using cache.
226     if (request.isConditional())
227         return NetworkCache::RetrieveDecision::NoDueToConditionalRequest;
228     if (request.cachePolicy() == WebCore::ReloadIgnoringCacheData)
229         return NetworkCache::RetrieveDecision::NoDueToReloadIgnoringCache;
230
231     return NetworkCache::RetrieveDecision::Yes;
232 }
233
234 static NetworkCacheKey makeCacheKey(const WebCore::ResourceRequest& request)
235 {
236 #if ENABLE(CACHE_PARTITIONING)
237     String partition = request.cachePartition();
238 #else
239     String partition;
240 #endif
241     if (partition.isEmpty())
242         partition = ASCIILiteral("No partition");
243     return NetworkCacheKey(request.httpMethod(), partition, request.url().string());
244 }
245
246 void NetworkCache::retrieve(const WebCore::ResourceRequest& originalRequest, uint64_t webPageID, std::function<void (std::unique_ptr<Entry>)> completionHandler)
247 {
248     ASSERT(isEnabled());
249
250     LOG(NetworkCache, "(NetworkProcess) retrieving %s priority %u", originalRequest.url().string().ascii().data(), originalRequest.priority());
251
252     NetworkCacheKey storageKey = makeCacheKey(originalRequest);
253     RetrieveDecision retrieveDecision = canRetrieve(originalRequest);
254     if (retrieveDecision != RetrieveDecision::Yes) {
255         if (m_statistics)
256             m_statistics->recordNotUsingCacheForRequest(webPageID, storageKey, originalRequest, retrieveDecision);
257
258         completionHandler(nullptr);
259         return;
260     }
261
262     auto startTime = std::chrono::system_clock::now();
263     unsigned priority = originalRequest.priority();
264
265     m_storage->retrieve(storageKey, priority, [this, originalRequest, completionHandler, startTime, storageKey, webPageID](std::unique_ptr<NetworkCacheStorage::Entry> entry) {
266         if (!entry) {
267             LOG(NetworkCache, "(NetworkProcess) not found in storage");
268
269             if (m_statistics)
270                 m_statistics->recordRetrievalFailure(webPageID, storageKey, originalRequest);
271
272             completionHandler(nullptr);
273             return false;
274         }
275         CachedEntryReuseFailure failure = CachedEntryReuseFailure::None;
276         auto decodedEntry = decodeStorageEntry(*entry, originalRequest, failure);
277         bool success = !!decodedEntry;
278         if (m_statistics)
279             m_statistics->recordRetrievedCachedEntry(webPageID, storageKey, originalRequest, failure);
280
281 #if !LOG_DISABLED
282         auto elapsedMS = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count();
283 #endif
284         LOG(NetworkCache, "(NetworkProcess) retrieve complete success=%d priority=%u time=%lldms", success, originalRequest.priority(), elapsedMS);
285         completionHandler(WTF::move(decodedEntry));
286         return success;
287     });
288 }
289
290 static NetworkCache::StoreDecision canStore(const WebCore::ResourceRequest& originalRequest, const WebCore::ResourceResponse& response)
291 {
292     if (!originalRequest.url().protocolIsInHTTPFamily() || !response.isHTTP()) {
293         LOG(NetworkCache, "(NetworkProcess) not HTTP");
294         return NetworkCache::StoreDecision::NoDueToProtocol;
295     }
296     if (originalRequest.httpMethod() != "GET") {
297         LOG(NetworkCache, "(NetworkProcess) method %s", originalRequest.httpMethod().utf8().data());
298         return NetworkCache::StoreDecision::NoDueToHTTPMethod;
299     }
300     if (response.isAttachment()) {
301         LOG(NetworkCache, "(NetworkProcess) attachment");
302         return NetworkCache::StoreDecision::NoDueToAttachmentResponse;
303     }
304
305     switch (response.httpStatusCode()) {
306     case 200: // OK
307     case 203: // Non-Authoritative Information
308     case 300: // Multiple Choices
309     case 301: // Moved Permanently
310     case 302: // Found
311     case 307: // Temporary Redirect
312     case 410: // Gone
313         if (response.cacheControlContainsNoStore()) {
314             LOG(NetworkCache, "(NetworkProcess) Cache-control:no-store");
315             return NetworkCache::StoreDecision::NoDueToNoStoreResponse;
316         }
317         return NetworkCache::StoreDecision::Yes;
318     default:
319         LOG(NetworkCache, "(NetworkProcess) status code %d", response.httpStatusCode());
320     }
321
322     return NetworkCache::StoreDecision::NoDueToHTTPStatusCode;
323 }
324
325 void NetworkCache::store(const WebCore::ResourceRequest& originalRequest, const WebCore::ResourceResponse& response, RefPtr<WebCore::SharedBuffer>&& responseData, std::function<void (MappedBody&)> completionHandler)
326 {
327     ASSERT(isEnabled());
328     ASSERT(responseData);
329
330     LOG(NetworkCache, "(NetworkProcess) storing %s, partition %s", originalRequest.url().string().latin1().data(), originalRequest.cachePartition().latin1().data());
331
332     auto key = makeCacheKey(originalRequest);
333     StoreDecision storeDecision = canStore(originalRequest, response);
334     if (storeDecision != StoreDecision::Yes) {
335         LOG(NetworkCache, "(NetworkProcess) didn't store");
336         if (m_statistics)
337             m_statistics->recordNotCachingResponse(key, storeDecision);
338         return;
339     }
340
341     auto storageEntry = encodeStorageEntry(originalRequest, response, WTF::move(responseData));
342
343     m_storage->store(key, storageEntry, [completionHandler](bool success, const NetworkCacheStorage::Data& bodyData) {
344         MappedBody mappedBody;
345 #if ENABLE(SHAREABLE_RESOURCE)
346         if (bodyData.isMap()) {
347             RefPtr<SharedMemory> sharedMemory = SharedMemory::createFromVMBuffer(const_cast<uint8_t*>(bodyData.data()), bodyData.size());
348             mappedBody.shareableResource = sharedMemory ? ShareableResource::create(WTF::move(sharedMemory), 0, bodyData.size()) : nullptr;
349             if (mappedBody.shareableResource)
350                 mappedBody.shareableResource->createHandle(mappedBody.shareableResourceHandle);
351         }
352 #endif
353         completionHandler(mappedBody);
354         LOG(NetworkCache, "(NetworkProcess) store success=%d", success);
355     });
356 }
357
358 void NetworkCache::update(const WebCore::ResourceRequest& originalRequest, const Entry& entry, const WebCore::ResourceResponse& validatingResponse)
359 {
360     LOG(NetworkCache, "(NetworkProcess) updating %s", originalRequest.url().string().latin1().data());
361
362     WebCore::ResourceResponse response = entry.response;
363     WebCore::updateResponseHeadersAfterRevalidation(response, validatingResponse);
364
365     auto key = makeCacheKey(originalRequest);
366     auto updateEntry = encodeStorageEntry(originalRequest, response, entry.buffer);
367
368     m_storage->update(key, updateEntry, entry.storageEntry, [](bool success, const NetworkCacheStorage::Data&) {
369         LOG(NetworkCache, "(NetworkProcess) updated, success=%d", success);
370     });
371 }
372
373 void NetworkCache::traverse(std::function<void (const Entry*)>&& traverseHandler)
374 {
375     ASSERT(isEnabled());
376
377     m_storage->traverse([traverseHandler](const NetworkCacheKey& key, const NetworkCacheStorage::Entry* entry) {
378         if (!entry) {
379             traverseHandler(nullptr);
380             return;
381         }
382
383         NetworkCache::Entry cacheEntry;
384         cacheEntry.storageEntry = *entry;
385
386         NetworkCacheDecoder decoder(cacheEntry.storageEntry.header.data(), cacheEntry.storageEntry.header.size());
387         if (!decoder.decode(cacheEntry.response))
388             return;
389
390         traverseHandler(&cacheEntry);
391     });
392 }
393
394 String NetworkCache::dumpFilePath() const
395 {
396     return WebCore::pathByAppendingComponent(m_storage->baseDirectoryPath(), "dump.json");
397 }
398
399 static bool entryAsJSON(StringBuilder& json, const NetworkCacheKey& key, const NetworkCacheStorage::Entry& entry)
400 {
401     NetworkCacheDecoder decoder(entry.header.data(), entry.header.size());
402     WebCore::ResourceResponse cachedResponse;
403     if (!decoder.decode(cachedResponse))
404         return false;
405     json.append("{\n");
406     json.append("\"hash\": ");
407     JSC::appendQuotedJSONStringToBuilder(json, key.hashAsString());
408     json.append(",\n");
409     json.append("\"partition\": ");
410     JSC::appendQuotedJSONStringToBuilder(json, key.partition());
411     json.append(",\n");
412     json.append("\"timestamp\": ");
413     json.appendNumber(entry.timeStamp.count());
414     json.append(",\n");
415     json.append("\"URL\": ");
416     JSC::appendQuotedJSONStringToBuilder(json, cachedResponse.url().string());
417     json.append(",\n");
418     json.append("\"headers\": {\n");
419     bool firstHeader = true;
420     for (auto& header : cachedResponse.httpHeaderFields()) {
421         if (!firstHeader)
422             json.append(",\n");
423         firstHeader = false;
424         json.append("    ");
425         JSC::appendQuotedJSONStringToBuilder(json, header.key);
426         json.append(": ");
427         JSC::appendQuotedJSONStringToBuilder(json, header.value);
428     }
429     json.append("\n}\n");
430     json.append("}");
431     return true;
432 }
433
434 void NetworkCache::dumpContentsToFile()
435 {
436     if (!m_storage)
437         return;
438     auto dumpFileHandle = WebCore::openFile(dumpFilePath(), WebCore::OpenForWrite);
439     if (!dumpFileHandle)
440         return;
441     WebCore::writeToFile(dumpFileHandle, "[\n", 2);
442     m_storage->traverse([dumpFileHandle](const NetworkCacheKey& key, const NetworkCacheStorage::Entry* entry) {
443         if (!entry) {
444             WebCore::writeToFile(dumpFileHandle, "{}\n]\n", 5);
445             auto handle = dumpFileHandle;
446             WebCore::closeFile(handle);
447             return;
448         }
449         StringBuilder json;
450         if (!entryAsJSON(json, key, *entry))
451             return;
452         json.append(",\n");
453         auto writeData = json.toString().utf8();
454         WebCore::writeToFile(dumpFileHandle, writeData.data(), writeData.length());
455     });
456 }
457
458 void NetworkCache::clear()
459 {
460     LOG(NetworkCache, "(NetworkProcess) clearing cache");
461     if (m_storage) {
462         m_storage->clear();
463
464         auto queue = WorkQueue::create("com.apple.WebKit.Cache.delete");
465         StringCapture dumpFilePathCapture(dumpFilePath());
466         queue->dispatch([dumpFilePathCapture] {
467             WebCore::deleteFile(dumpFilePathCapture.string());
468         });
469     }
470     if (m_statistics)
471         m_statistics->clear();
472 }
473
474 String NetworkCache::storagePath() const
475 {
476     return m_storage ? m_storage->directoryPath() : String();
477 }
478
479 }
480
481 #endif