Add interface to delete disk cache entries
[WebKit-https.git] / Source / WebKit2 / NetworkProcess / cache / NetworkCache.cpp
1 /*
2  * Copyright (C) 2014-2015 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "NetworkCache.h"
28
29 #if ENABLE(NETWORK_CACHE)
30
31 #include "Logging.h"
32 #include "NetworkCacheCoders.h"
33 #include "NetworkCacheStatistics.h"
34 #include "NetworkCacheStorage.h"
35 #include "NetworkResourceLoader.h"
36 #include "WebCoreArgumentCoders.h"
37 #include <JavaScriptCore/JSONObject.h>
38 #include <WebCore/CacheValidation.h>
39 #include <WebCore/FileSystem.h>
40 #include <WebCore/HTTPHeaderNames.h>
41 #include <WebCore/ResourceResponse.h>
42 #include <WebCore/SharedBuffer.h>
43 #include <wtf/NeverDestroyed.h>
44 #include <wtf/StringHasher.h>
45 #include <wtf/text/StringBuilder.h>
46
47 #if PLATFORM(COCOA)
48 #include <notify.h>
49 #endif
50
51 namespace WebKit {
52 namespace NetworkCache {
53
54 Cache& singleton()
55 {
56     static NeverDestroyed<Cache> instance;
57     return instance;
58 }
59
60 bool Cache::initialize(const String& cachePath, bool enableEfficacyLogging)
61 {
62     m_storage = Storage::open(cachePath);
63
64     if (enableEfficacyLogging)
65         m_statistics = Statistics::open(cachePath);
66
67 #if PLATFORM(COCOA)
68     // Triggers with "notifyutil -p com.apple.WebKit.Cache.dump".
69     if (m_storage) {
70         int token;
71         notify_register_dispatch("com.apple.WebKit.Cache.dump", &token, dispatch_get_main_queue(), ^(int) {
72             dumpContentsToFile();
73         });
74     }
75 #endif
76
77     LOG(NetworkCache, "(NetworkProcess) opened cache storage, success %d", !!m_storage);
78     return !!m_storage;
79 }
80
81 void Cache::setMaximumSize(size_t maximumSize)
82 {
83     if (!m_storage)
84         return;
85     m_storage->setMaximumSize(maximumSize);
86 }
87
88 static Key makeCacheKey(const WebCore::ResourceRequest& request)
89 {
90 #if ENABLE(CACHE_PARTITIONING)
91     String partition = request.cachePartition();
92 #else
93     String partition;
94 #endif
95     if (partition.isEmpty())
96         partition = ASCIILiteral("No partition");
97     return { request.httpMethod(), partition, request.url().string()  };
98 }
99
100 static Storage::Entry encodeStorageEntry(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, PassRefPtr<WebCore::SharedBuffer> responseData)
101 {
102     Encoder encoder;
103     encoder << response;
104
105     String varyValue = response.httpHeaderField(WebCore::HTTPHeaderName::Vary);
106     bool hasVaryingRequestHeaders = !varyValue.isEmpty();
107
108     encoder << hasVaryingRequestHeaders;
109
110     if (hasVaryingRequestHeaders) {
111         Vector<String> varyingHeaderNames;
112         varyValue.split(',', false, varyingHeaderNames);
113
114         Vector<std::pair<String, String>> varyingRequestHeaders;
115         for (auto& varyHeaderName : varyingHeaderNames) {
116             String headerName = varyHeaderName.stripWhiteSpace();
117             varyingRequestHeaders.append(std::make_pair(headerName, request.httpHeaderField(headerName)));
118         }
119         encoder << varyingRequestHeaders;
120     }
121     encoder.encodeChecksum();
122
123     auto timeStamp = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch());
124     Data header(encoder.buffer(), encoder.bufferSize());
125     Data body;
126     if (responseData)
127         body = { reinterpret_cast<const uint8_t*>(responseData->data()), responseData->size() };
128
129     return { makeCacheKey(request), timeStamp, header, body };
130 }
131
132 static bool verifyVaryingRequestHeaders(const Vector<std::pair<String, String>>& varyingRequestHeaders, const WebCore::ResourceRequest& request)
133 {
134     for (auto& varyingRequestHeader : varyingRequestHeaders) {
135         // FIXME: Vary: * in response would ideally trigger a cache delete instead of a store.
136         if (varyingRequestHeader.first == "*")
137             return false;
138         String requestValue = request.httpHeaderField(varyingRequestHeader.first);
139         if (requestValue != varyingRequestHeader.second)
140             return false;
141     }
142     return true;
143 }
144
145 static bool cachePolicyAllowsExpired(WebCore::ResourceRequestCachePolicy policy)
146 {
147     switch (policy) {
148     case WebCore::ReturnCacheDataElseLoad:
149     case WebCore::ReturnCacheDataDontLoad:
150         return true;
151     case WebCore::UseProtocolCachePolicy:
152     case WebCore::ReloadIgnoringCacheData:
153         return false;
154     }
155     ASSERT_NOT_REACHED();
156     return false;
157 }
158
159 static std::unique_ptr<Entry> decodeStorageEntry(const Storage::Entry& storageEntry, const WebCore::ResourceRequest& request, CachedEntryReuseFailure& failure)
160 {
161     Decoder decoder(storageEntry.header.data(), storageEntry.header.size());
162
163     WebCore::ResourceResponse cachedResponse;
164     if (!decoder.decode(cachedResponse)) {
165         LOG(NetworkCache, "(NetworkProcess) response decoding failed\n");
166         failure = CachedEntryReuseFailure::Other;
167         return nullptr;
168     }
169
170     bool hasVaryingRequestHeaders;
171     if (!decoder.decode(hasVaryingRequestHeaders)) {
172         failure = CachedEntryReuseFailure::Other;
173         return nullptr;
174     }
175
176     if (hasVaryingRequestHeaders) {
177         Vector<std::pair<String, String>> varyingRequestHeaders;
178         if (!decoder.decode(varyingRequestHeaders)) {
179             failure = CachedEntryReuseFailure::Other;
180             return nullptr;
181         }
182
183         if (!verifyVaryingRequestHeaders(varyingRequestHeaders, request)) {
184             LOG(NetworkCache, "(NetworkProcess) varying header mismatch\n");
185             failure = CachedEntryReuseFailure::VaryingHeaderMismatch;
186             return nullptr;
187         }
188     }
189     if (!decoder.verifyChecksum()) {
190         LOG(NetworkCache, "(NetworkProcess) checksum verification failure\n");
191         failure = CachedEntryReuseFailure::Other;
192         return nullptr;
193     }
194
195     bool allowExpired = cachePolicyAllowsExpired(request.cachePolicy()) && !cachedResponse.cacheControlContainsMustRevalidate();
196     auto timeStamp = std::chrono::duration_cast<std::chrono::duration<double>>(storageEntry.timeStamp);
197     double age = WebCore::computeCurrentAge(cachedResponse, timeStamp.count());
198     double lifetime = WebCore::computeFreshnessLifetimeForHTTPFamily(cachedResponse, timeStamp.count());
199     bool isExpired = age > lifetime;
200     bool needsRevalidation = (isExpired && !allowExpired) || cachedResponse.cacheControlContainsNoCache();
201
202     if (needsRevalidation) {
203         bool hasValidatorFields = cachedResponse.hasCacheValidatorFields();
204         LOG(NetworkCache, "(NetworkProcess) needsRevalidation hasValidatorFields=%d isExpired=%d age=%f lifetime=%f", isExpired, hasValidatorFields, age, lifetime);
205         if (!hasValidatorFields) {
206             failure = CachedEntryReuseFailure::MissingValidatorFields;
207             return nullptr;
208         }
209     }
210
211     auto entry = std::make_unique<Entry>();
212     entry->storageEntry = storageEntry;
213     entry->needsRevalidation = needsRevalidation;
214
215     cachedResponse.setSource(needsRevalidation ? WebCore::ResourceResponse::Source::DiskCacheAfterValidation : WebCore::ResourceResponse::Source::DiskCache);
216     entry->response = cachedResponse;
217
218 #if ENABLE(SHAREABLE_RESOURCE)
219     RefPtr<SharedMemory> sharedMemory = storageEntry.body.isMap() ? SharedMemory::createFromVMBuffer(const_cast<uint8_t*>(storageEntry.body.data()), storageEntry.body.size()) : nullptr;
220     RefPtr<ShareableResource> shareableResource = sharedMemory ? ShareableResource::create(sharedMemory.release(), 0, storageEntry.body.size()) : nullptr;
221
222     if (shareableResource && shareableResource->createHandle(entry->shareableResourceHandle))
223         entry->buffer = entry->shareableResourceHandle.tryWrapInSharedBuffer();
224     else
225 #endif
226         entry->buffer = WebCore::SharedBuffer::create(storageEntry.body.data(), storageEntry.body.size());
227
228     return entry;
229 }
230
231 static RetrieveDecision canRetrieve(const WebCore::ResourceRequest& request)
232 {
233     if (!request.url().protocolIsInHTTPFamily())
234         return RetrieveDecision::NoDueToProtocol;
235     // FIXME: Support HEAD and OPTIONS requests.
236     if (request.httpMethod() != "GET")
237         return RetrieveDecision::NoDueToHTTPMethod;
238     // FIXME: We should be able to validate conditional requests using cache.
239     if (request.isConditional())
240         return RetrieveDecision::NoDueToConditionalRequest;
241     if (request.cachePolicy() == WebCore::ReloadIgnoringCacheData)
242         return RetrieveDecision::NoDueToReloadIgnoringCache;
243
244     return RetrieveDecision::Yes;
245 }
246
247 void Cache::retrieve(const WebCore::ResourceRequest& originalRequest, uint64_t webPageID, std::function<void (std::unique_ptr<Entry>)> completionHandler)
248 {
249     ASSERT(isEnabled());
250
251     LOG(NetworkCache, "(NetworkProcess) retrieving %s priority %u", originalRequest.url().string().ascii().data(), originalRequest.priority());
252
253     Key storageKey = makeCacheKey(originalRequest);
254     RetrieveDecision retrieveDecision = canRetrieve(originalRequest);
255     if (retrieveDecision != RetrieveDecision::Yes) {
256         if (m_statistics)
257             m_statistics->recordNotUsingCacheForRequest(webPageID, storageKey, originalRequest, retrieveDecision);
258
259         completionHandler(nullptr);
260         return;
261     }
262
263     auto startTime = std::chrono::system_clock::now();
264     unsigned priority = originalRequest.priority();
265
266     m_storage->retrieve(storageKey, priority, [this, originalRequest, completionHandler, startTime, storageKey, webPageID](std::unique_ptr<Storage::Entry> entry) {
267         if (!entry) {
268             LOG(NetworkCache, "(NetworkProcess) not found in storage");
269
270             if (m_statistics)
271                 m_statistics->recordRetrievalFailure(webPageID, storageKey, originalRequest);
272
273             completionHandler(nullptr);
274             return false;
275         }
276         ASSERT(entry->key == storageKey);
277
278         CachedEntryReuseFailure failure = CachedEntryReuseFailure::None;
279         auto decodedEntry = decodeStorageEntry(*entry, originalRequest, failure);
280         bool success = !!decodedEntry;
281         if (m_statistics)
282             m_statistics->recordRetrievedCachedEntry(webPageID, storageKey, originalRequest, failure);
283
284 #if !LOG_DISABLED
285         auto elapsedMS = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count();
286 #endif
287         LOG(NetworkCache, "(NetworkProcess) retrieve complete success=%d priority=%u time=%lldms", success, originalRequest.priority(), elapsedMS);
288         completionHandler(WTF::move(decodedEntry));
289         return success;
290     });
291 }
292
293 static StoreDecision canStore(const WebCore::ResourceRequest& originalRequest, const WebCore::ResourceResponse& response)
294 {
295     if (!originalRequest.url().protocolIsInHTTPFamily() || !response.isHTTP()) {
296         LOG(NetworkCache, "(NetworkProcess) not HTTP");
297         return StoreDecision::NoDueToProtocol;
298     }
299     if (originalRequest.httpMethod() != "GET") {
300         LOG(NetworkCache, "(NetworkProcess) method %s", originalRequest.httpMethod().utf8().data());
301         return StoreDecision::NoDueToHTTPMethod;
302     }
303     if (response.isAttachment()) {
304         LOG(NetworkCache, "(NetworkProcess) attachment");
305         return StoreDecision::NoDueToAttachmentResponse;
306     }
307
308     switch (response.httpStatusCode()) {
309     case 200: // OK
310     case 203: // Non-Authoritative Information
311     case 300: // Multiple Choices
312     case 301: // Moved Permanently
313     case 302: // Found
314     case 307: // Temporary Redirect
315     case 410: // Gone
316         if (response.cacheControlContainsNoStore()) {
317             LOG(NetworkCache, "(NetworkProcess) Cache-control:no-store");
318             return StoreDecision::NoDueToNoStoreResponse;
319         }
320         return StoreDecision::Yes;
321     default:
322         LOG(NetworkCache, "(NetworkProcess) status code %d", response.httpStatusCode());
323     }
324
325     return StoreDecision::NoDueToHTTPStatusCode;
326 }
327
328 void Cache::store(const WebCore::ResourceRequest& originalRequest, const WebCore::ResourceResponse& response, RefPtr<WebCore::SharedBuffer>&& responseData, std::function<void (MappedBody&)> completionHandler)
329 {
330     ASSERT(isEnabled());
331     ASSERT(responseData);
332
333     LOG(NetworkCache, "(NetworkProcess) storing %s, partition %s", originalRequest.url().string().latin1().data(), originalRequest.cachePartition().latin1().data());
334
335     StoreDecision storeDecision = canStore(originalRequest, response);
336     if (storeDecision != StoreDecision::Yes) {
337         LOG(NetworkCache, "(NetworkProcess) didn't store");
338         if (m_statistics) {
339             auto key = makeCacheKey(originalRequest);
340             m_statistics->recordNotCachingResponse(key, storeDecision);
341         }
342         return;
343     }
344
345     auto storageEntry = encodeStorageEntry(originalRequest, response, WTF::move(responseData));
346
347     m_storage->store(storageEntry, [completionHandler](bool success, const Data& bodyData) {
348         MappedBody mappedBody;
349 #if ENABLE(SHAREABLE_RESOURCE)
350         if (bodyData.isMap()) {
351             RefPtr<SharedMemory> sharedMemory = SharedMemory::createFromVMBuffer(const_cast<uint8_t*>(bodyData.data()), bodyData.size());
352             mappedBody.shareableResource = sharedMemory ? ShareableResource::create(WTF::move(sharedMemory), 0, bodyData.size()) : nullptr;
353             if (mappedBody.shareableResource)
354                 mappedBody.shareableResource->createHandle(mappedBody.shareableResourceHandle);
355         }
356 #endif
357         completionHandler(mappedBody);
358         LOG(NetworkCache, "(NetworkProcess) store success=%d", success);
359     });
360 }
361
362 void Cache::update(const WebCore::ResourceRequest& originalRequest, const Entry& entry, const WebCore::ResourceResponse& validatingResponse)
363 {
364     LOG(NetworkCache, "(NetworkProcess) updating %s", originalRequest.url().string().latin1().data());
365
366     WebCore::ResourceResponse response = entry.response;
367     WebCore::updateResponseHeadersAfterRevalidation(response, validatingResponse);
368
369     auto updateEntry = encodeStorageEntry(originalRequest, response, entry.buffer);
370
371     m_storage->update(updateEntry, entry.storageEntry, [](bool success, const Data&) {
372         LOG(NetworkCache, "(NetworkProcess) updated, success=%d", success);
373     });
374 }
375
376 void Cache::remove(const Entry& entry)
377 {
378     ASSERT(isEnabled());
379
380     m_storage->remove(entry.storageEntry.key);
381 }
382
383 void Cache::traverse(std::function<void (const Entry*)>&& traverseHandler)
384 {
385     ASSERT(isEnabled());
386
387     m_storage->traverse([traverseHandler](const Storage::Entry* entry) {
388         if (!entry) {
389             traverseHandler(nullptr);
390             return;
391         }
392
393         Entry cacheEntry;
394         cacheEntry.storageEntry = *entry;
395
396         Decoder decoder(cacheEntry.storageEntry.header.data(), cacheEntry.storageEntry.header.size());
397         if (!decoder.decode(cacheEntry.response))
398             return;
399
400         traverseHandler(&cacheEntry);
401     });
402 }
403
404 String Cache::dumpFilePath() const
405 {
406     return WebCore::pathByAppendingComponent(m_storage->baseDirectoryPath(), "dump.json");
407 }
408
409 static bool entryAsJSON(StringBuilder& json, const Storage::Entry& entry)
410 {
411     Decoder decoder(entry.header.data(), entry.header.size());
412     WebCore::ResourceResponse cachedResponse;
413     if (!decoder.decode(cachedResponse))
414         return false;
415     json.append("{\n");
416     json.append("\"hash\": ");
417     JSC::appendQuotedJSONStringToBuilder(json, entry.key.hashAsString());
418     json.append(",\n");
419     json.append("\"partition\": ");
420     JSC::appendQuotedJSONStringToBuilder(json, entry.key.partition());
421     json.append(",\n");
422     json.append("\"timestamp\": ");
423     json.appendNumber(entry.timeStamp.count());
424     json.append(",\n");
425     json.append("\"URL\": ");
426     JSC::appendQuotedJSONStringToBuilder(json, cachedResponse.url().string());
427     json.append(",\n");
428     json.append("\"headers\": {\n");
429     bool firstHeader = true;
430     for (auto& header : cachedResponse.httpHeaderFields()) {
431         if (!firstHeader)
432             json.append(",\n");
433         firstHeader = false;
434         json.append("    ");
435         JSC::appendQuotedJSONStringToBuilder(json, header.key);
436         json.append(": ");
437         JSC::appendQuotedJSONStringToBuilder(json, header.value);
438     }
439     json.append("\n}\n");
440     json.append("}");
441     return true;
442 }
443
444 void Cache::dumpContentsToFile()
445 {
446     if (!m_storage)
447         return;
448     auto dumpFileHandle = WebCore::openFile(dumpFilePath(), WebCore::OpenForWrite);
449     if (!dumpFileHandle)
450         return;
451     WebCore::writeToFile(dumpFileHandle, "[\n", 2);
452     m_storage->traverse([dumpFileHandle](const Storage::Entry* entry) {
453         if (!entry) {
454             WebCore::writeToFile(dumpFileHandle, "{}\n]\n", 5);
455             auto handle = dumpFileHandle;
456             WebCore::closeFile(handle);
457             return;
458         }
459         StringBuilder json;
460         if (!entryAsJSON(json, *entry))
461             return;
462         json.append(",\n");
463         auto writeData = json.toString().utf8();
464         WebCore::writeToFile(dumpFileHandle, writeData.data(), writeData.length());
465     });
466 }
467
468 void Cache::clear()
469 {
470     LOG(NetworkCache, "(NetworkProcess) clearing cache");
471     if (m_storage) {
472         m_storage->clear();
473
474         auto queue = WorkQueue::create("com.apple.WebKit.Cache.delete");
475         StringCapture dumpFilePathCapture(dumpFilePath());
476         queue->dispatch([dumpFilePathCapture] {
477             WebCore::deleteFile(dumpFilePathCapture.string());
478         });
479     }
480     if (m_statistics)
481         m_statistics->clear();
482 }
483
484 String Cache::storagePath() const
485 {
486     return m_storage ? m_storage->directoryPath() : String();
487 }
488
489 }
490 }
491
492 #endif