[WK2] Remove NetworkCache FIXME comment about caching responses to OPTIONS requests
[WebKit-https.git] / Source / WebKit2 / NetworkProcess / cache / NetworkCache.cpp
1 /*
2  * Copyright (C) 2014-2015 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "NetworkCache.h"
28
29 #if ENABLE(NETWORK_CACHE)
30
31 #include "Logging.h"
32 #include "NetworkCacheCoders.h"
33 #include "NetworkCacheStatistics.h"
34 #include "NetworkCacheStorage.h"
35 #include "NetworkResourceLoader.h"
36 #include "WebCoreArgumentCoders.h"
37 #include <JavaScriptCore/JSONObject.h>
38 #include <WebCore/CacheValidation.h>
39 #include <WebCore/FileSystem.h>
40 #include <WebCore/HTTPHeaderNames.h>
41 #include <WebCore/NetworkStorageSession.h>
42 #include <WebCore/PlatformCookieJar.h>
43 #include <WebCore/ResourceResponse.h>
44 #include <WebCore/SharedBuffer.h>
45 #include <wtf/NeverDestroyed.h>
46 #include <wtf/StringHasher.h>
47 #include <wtf/text/StringBuilder.h>
48
49 #if PLATFORM(COCOA)
50 #include <notify.h>
51 #endif
52
53 namespace WebKit {
54 namespace NetworkCache {
55
56 Cache& singleton()
57 {
58     static NeverDestroyed<Cache> instance;
59     return instance;
60 }
61
62 bool Cache::initialize(const String& cachePath, bool enableEfficacyLogging)
63 {
64     m_storage = Storage::open(cachePath);
65
66     if (enableEfficacyLogging)
67         m_statistics = Statistics::open(cachePath);
68
69 #if PLATFORM(COCOA)
70     // Triggers with "notifyutil -p com.apple.WebKit.Cache.dump".
71     if (m_storage) {
72         int token;
73         notify_register_dispatch("com.apple.WebKit.Cache.dump", &token, dispatch_get_main_queue(), ^(int) {
74             dumpContentsToFile();
75         });
76     }
77 #endif
78
79     LOG(NetworkCache, "(NetworkProcess) opened cache storage, success %d", !!m_storage);
80     return !!m_storage;
81 }
82
83 void Cache::setMaximumSize(size_t maximumSize)
84 {
85     if (!m_storage)
86         return;
87     m_storage->setMaximumSize(maximumSize);
88 }
89
90 static Key makeCacheKey(const WebCore::ResourceRequest& request)
91 {
92 #if ENABLE(CACHE_PARTITIONING)
93     String partition = request.cachePartition();
94 #else
95     String partition;
96 #endif
97     if (partition.isEmpty())
98         partition = ASCIILiteral("No partition");
99     return { request.httpMethod(), partition, request.url().string()  };
100 }
101
102 static String headerValueForVary(const WebCore::ResourceRequest& request, const String& headerName)
103 {
104     // Explicit handling for cookies is needed because they are added magically by the networking layer.
105     // FIXME: The value might have changed between making the request and retrieving the cookie here.
106     // We could fetch the cookie when making the request but that seems overkill as the case is very rare and it
107     // is a blocking operation. This should be sufficient to cover reasonable cases.
108     if (headerName == httpHeaderNameString(WebCore::HTTPHeaderName::Cookie))
109         return WebCore::cookieRequestHeaderFieldValue(WebCore::NetworkStorageSession::defaultStorageSession(), request.firstPartyForCookies(), request.url());
110     return request.httpHeaderField(headerName);
111 }
112
113 static Storage::Entry encodeStorageEntry(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, PassRefPtr<WebCore::SharedBuffer> responseData)
114 {
115     Encoder encoder;
116     encoder << response;
117
118     String varyValue = response.httpHeaderField(WebCore::HTTPHeaderName::Vary);
119     bool hasVaryingRequestHeaders = !varyValue.isEmpty();
120
121     encoder << hasVaryingRequestHeaders;
122
123     if (hasVaryingRequestHeaders) {
124         Vector<String> varyingHeaderNames;
125         varyValue.split(',', false, varyingHeaderNames);
126
127         Vector<std::pair<String, String>> varyingRequestHeaders;
128         for (auto& varyHeaderName : varyingHeaderNames) {
129             String headerName = varyHeaderName.stripWhiteSpace();
130             String headerValue = headerValueForVary(request, headerName);
131             varyingRequestHeaders.append(std::make_pair(headerName, headerValue));
132         }
133         encoder << varyingRequestHeaders;
134     }
135     encoder.encodeChecksum();
136
137     auto timeStamp = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch());
138     Data header(encoder.buffer(), encoder.bufferSize());
139     Data body;
140     if (responseData)
141         body = { reinterpret_cast<const uint8_t*>(responseData->data()), responseData->size() };
142
143     return { makeCacheKey(request), timeStamp, header, body };
144 }
145
146 static bool verifyVaryingRequestHeaders(const Vector<std::pair<String, String>>& varyingRequestHeaders, const WebCore::ResourceRequest& request)
147 {
148     for (auto& varyingRequestHeader : varyingRequestHeaders) {
149         // FIXME: Vary: * in response would ideally trigger a cache delete instead of a store.
150         if (varyingRequestHeader.first == "*")
151             return false;
152         String headerValue = headerValueForVary(request, varyingRequestHeader.first);
153         if (headerValue != varyingRequestHeader.second)
154             return false;
155     }
156     return true;
157 }
158
159 static bool cachePolicyAllowsExpired(WebCore::ResourceRequestCachePolicy policy)
160 {
161     switch (policy) {
162     case WebCore::ReturnCacheDataElseLoad:
163     case WebCore::ReturnCacheDataDontLoad:
164         return true;
165     case WebCore::UseProtocolCachePolicy:
166     case WebCore::ReloadIgnoringCacheData:
167         return false;
168     }
169     ASSERT_NOT_REACHED();
170     return false;
171 }
172
173 static std::unique_ptr<Entry> decodeStorageEntry(const Storage::Entry& storageEntry, const WebCore::ResourceRequest& request, CachedEntryReuseFailure& failure)
174 {
175     Decoder decoder(storageEntry.header.data(), storageEntry.header.size());
176
177     WebCore::ResourceResponse cachedResponse;
178     if (!decoder.decode(cachedResponse)) {
179         LOG(NetworkCache, "(NetworkProcess) response decoding failed\n");
180         failure = CachedEntryReuseFailure::Other;
181         return nullptr;
182     }
183
184     bool hasVaryingRequestHeaders;
185     if (!decoder.decode(hasVaryingRequestHeaders)) {
186         failure = CachedEntryReuseFailure::Other;
187         return nullptr;
188     }
189
190     if (hasVaryingRequestHeaders) {
191         Vector<std::pair<String, String>> varyingRequestHeaders;
192         if (!decoder.decode(varyingRequestHeaders)) {
193             failure = CachedEntryReuseFailure::Other;
194             return nullptr;
195         }
196
197         if (!verifyVaryingRequestHeaders(varyingRequestHeaders, request)) {
198             LOG(NetworkCache, "(NetworkProcess) varying header mismatch\n");
199             failure = CachedEntryReuseFailure::VaryingHeaderMismatch;
200             return nullptr;
201         }
202     }
203     if (!decoder.verifyChecksum()) {
204         LOG(NetworkCache, "(NetworkProcess) checksum verification failure\n");
205         failure = CachedEntryReuseFailure::Other;
206         return nullptr;
207     }
208
209     bool allowExpired = cachePolicyAllowsExpired(request.cachePolicy());
210     auto timeStamp = std::chrono::duration_cast<std::chrono::duration<double>>(storageEntry.timeStamp);
211     double age = WebCore::computeCurrentAge(cachedResponse, timeStamp.count());
212     double lifetime = WebCore::computeFreshnessLifetimeForHTTPFamily(cachedResponse, timeStamp.count());
213     bool isExpired = age > lifetime;
214     // We never revalidate in the case of a history navigation (i.e. allowExpired is true).
215     bool needsRevalidation = !allowExpired && (cachedResponse.cacheControlContainsNoCache() || isExpired);
216
217     if (needsRevalidation) {
218         bool hasValidatorFields = cachedResponse.hasCacheValidatorFields();
219         LOG(NetworkCache, "(NetworkProcess) needsRevalidation hasValidatorFields=%d isExpired=%d age=%f lifetime=%f", isExpired, hasValidatorFields, age, lifetime);
220         if (!hasValidatorFields) {
221             failure = CachedEntryReuseFailure::MissingValidatorFields;
222             return nullptr;
223         }
224     }
225
226     auto entry = std::make_unique<Entry>();
227     entry->storageEntry = storageEntry;
228     entry->needsRevalidation = needsRevalidation;
229
230     cachedResponse.setSource(needsRevalidation ? WebCore::ResourceResponse::Source::DiskCacheAfterValidation : WebCore::ResourceResponse::Source::DiskCache);
231     entry->response = cachedResponse;
232
233 #if ENABLE(SHAREABLE_RESOURCE)
234     RefPtr<SharedMemory> sharedMemory = storageEntry.body.isMap() ? SharedMemory::createFromVMBuffer(const_cast<uint8_t*>(storageEntry.body.data()), storageEntry.body.size()) : nullptr;
235     RefPtr<ShareableResource> shareableResource = sharedMemory ? ShareableResource::create(sharedMemory.release(), 0, storageEntry.body.size()) : nullptr;
236
237     if (shareableResource && shareableResource->createHandle(entry->shareableResourceHandle))
238         entry->buffer = entry->shareableResourceHandle.tryWrapInSharedBuffer();
239     else
240 #endif
241         entry->buffer = WebCore::SharedBuffer::create(storageEntry.body.data(), storageEntry.body.size());
242
243     return entry;
244 }
245
246 static RetrieveDecision canRetrieve(const WebCore::ResourceRequest& request)
247 {
248     // FIXME: Support HEAD requests.
249     if (request.httpMethod() != "GET")
250         return RetrieveDecision::NoDueToHTTPMethod;
251     // FIXME: We should be able to validate conditional requests using cache.
252     if (request.isConditional())
253         return RetrieveDecision::NoDueToConditionalRequest;
254     if (request.cachePolicy() == WebCore::ReloadIgnoringCacheData)
255         return RetrieveDecision::NoDueToReloadIgnoringCache;
256
257     return RetrieveDecision::Yes;
258 }
259
260 void Cache::retrieve(const WebCore::ResourceRequest& originalRequest, uint64_t webPageID, std::function<void (std::unique_ptr<Entry>)> completionHandler)
261 {
262     ASSERT(isEnabled());
263     ASSERT(originalRequest.url().protocolIsInHTTPFamily());
264
265     LOG(NetworkCache, "(NetworkProcess) retrieving %s priority %u", originalRequest.url().string().ascii().data(), originalRequest.priority());
266
267     if (m_statistics)
268         m_statistics->recordRetrievalRequest(webPageID);
269
270     Key storageKey = makeCacheKey(originalRequest);
271     RetrieveDecision retrieveDecision = canRetrieve(originalRequest);
272     if (retrieveDecision != RetrieveDecision::Yes) {
273         if (m_statistics)
274             m_statistics->recordNotUsingCacheForRequest(webPageID, storageKey, originalRequest, retrieveDecision);
275
276         completionHandler(nullptr);
277         return;
278     }
279
280     auto startTime = std::chrono::system_clock::now();
281     unsigned priority = originalRequest.priority();
282
283     m_storage->retrieve(storageKey, priority, [this, originalRequest, completionHandler, startTime, storageKey, webPageID](std::unique_ptr<Storage::Entry> entry) {
284         if (!entry) {
285             LOG(NetworkCache, "(NetworkProcess) not found in storage");
286
287             if (m_statistics)
288                 m_statistics->recordRetrievalFailure(webPageID, storageKey, originalRequest);
289
290             completionHandler(nullptr);
291             return false;
292         }
293         ASSERT(entry->key == storageKey);
294
295         CachedEntryReuseFailure failure = CachedEntryReuseFailure::None;
296         auto decodedEntry = decodeStorageEntry(*entry, originalRequest, failure);
297         bool success = !!decodedEntry;
298         if (m_statistics)
299             m_statistics->recordRetrievedCachedEntry(webPageID, storageKey, originalRequest, failure);
300
301 #if !LOG_DISABLED
302         auto elapsedMS = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count();
303 #endif
304         LOG(NetworkCache, "(NetworkProcess) retrieve complete success=%d priority=%u time=%lldms", success, originalRequest.priority(), elapsedMS);
305         completionHandler(WTF::move(decodedEntry));
306         return success;
307     });
308 }
309
310 static StoreDecision canStore(const WebCore::ResourceRequest& originalRequest, const WebCore::ResourceResponse& response)
311 {
312     if (!originalRequest.url().protocolIsInHTTPFamily() || !response.isHTTP()) {
313         LOG(NetworkCache, "(NetworkProcess) not HTTP");
314         return StoreDecision::NoDueToProtocol;
315     }
316     if (originalRequest.httpMethod() != "GET") {
317         LOG(NetworkCache, "(NetworkProcess) method %s", originalRequest.httpMethod().utf8().data());
318         return StoreDecision::NoDueToHTTPMethod;
319     }
320
321     switch (response.httpStatusCode()) {
322     case 200: // OK
323     case 203: // Non-Authoritative Information
324     case 300: // Multiple Choices
325     case 301: // Moved Permanently
326     case 302: // Found
327     case 307: // Temporary Redirect
328     case 410: // Gone
329         if (response.cacheControlContainsNoStore()) {
330             LOG(NetworkCache, "(NetworkProcess) Cache-control:no-store");
331             return StoreDecision::NoDueToNoStoreResponse;
332         }
333         return StoreDecision::Yes;
334     default:
335         LOG(NetworkCache, "(NetworkProcess) status code %d", response.httpStatusCode());
336     }
337
338     return StoreDecision::NoDueToHTTPStatusCode;
339 }
340
341 void Cache::store(const WebCore::ResourceRequest& originalRequest, const WebCore::ResourceResponse& response, RefPtr<WebCore::SharedBuffer>&& responseData, std::function<void (MappedBody&)> completionHandler)
342 {
343     ASSERT(isEnabled());
344     ASSERT(responseData);
345
346     LOG(NetworkCache, "(NetworkProcess) storing %s, partition %s", originalRequest.url().string().latin1().data(), originalRequest.cachePartition().latin1().data());
347
348     StoreDecision storeDecision = canStore(originalRequest, response);
349     if (storeDecision != StoreDecision::Yes) {
350         LOG(NetworkCache, "(NetworkProcess) didn't store");
351         if (m_statistics) {
352             auto key = makeCacheKey(originalRequest);
353             m_statistics->recordNotCachingResponse(key, storeDecision);
354         }
355         return;
356     }
357
358     auto storageEntry = encodeStorageEntry(originalRequest, response, WTF::move(responseData));
359
360     m_storage->store(storageEntry, [completionHandler](bool success, const Data& bodyData) {
361         MappedBody mappedBody;
362 #if ENABLE(SHAREABLE_RESOURCE)
363         if (bodyData.isMap()) {
364             RefPtr<SharedMemory> sharedMemory = SharedMemory::createFromVMBuffer(const_cast<uint8_t*>(bodyData.data()), bodyData.size());
365             mappedBody.shareableResource = sharedMemory ? ShareableResource::create(WTF::move(sharedMemory), 0, bodyData.size()) : nullptr;
366             if (mappedBody.shareableResource)
367                 mappedBody.shareableResource->createHandle(mappedBody.shareableResourceHandle);
368         }
369 #endif
370         completionHandler(mappedBody);
371         LOG(NetworkCache, "(NetworkProcess) store success=%d", success);
372     });
373 }
374
375 void Cache::update(const WebCore::ResourceRequest& originalRequest, const Entry& entry, const WebCore::ResourceResponse& validatingResponse)
376 {
377     LOG(NetworkCache, "(NetworkProcess) updating %s", originalRequest.url().string().latin1().data());
378
379     WebCore::ResourceResponse response = entry.response;
380     WebCore::updateResponseHeadersAfterRevalidation(response, validatingResponse);
381
382     auto updateEntry = encodeStorageEntry(originalRequest, response, entry.buffer);
383
384     m_storage->update(updateEntry, entry.storageEntry, [](bool success, const Data&) {
385         LOG(NetworkCache, "(NetworkProcess) updated, success=%d", success);
386     });
387 }
388
389 void Cache::remove(const Key& key)
390 {
391     ASSERT(isEnabled());
392
393     m_storage->remove(key);
394 }
395
396 void Cache::traverse(std::function<void (const Entry*)>&& traverseHandler)
397 {
398     ASSERT(isEnabled());
399
400     m_storage->traverse([traverseHandler](const Storage::Entry* entry) {
401         if (!entry) {
402             traverseHandler(nullptr);
403             return;
404         }
405
406         Entry cacheEntry;
407         cacheEntry.storageEntry = *entry;
408
409         Decoder decoder(cacheEntry.storageEntry.header.data(), cacheEntry.storageEntry.header.size());
410         if (!decoder.decode(cacheEntry.response))
411             return;
412
413         traverseHandler(&cacheEntry);
414     });
415 }
416
417 String Cache::dumpFilePath() const
418 {
419     return WebCore::pathByAppendingComponent(m_storage->baseDirectoryPath(), "dump.json");
420 }
421
422 static bool entryAsJSON(StringBuilder& json, const Storage::Entry& entry)
423 {
424     Decoder decoder(entry.header.data(), entry.header.size());
425     WebCore::ResourceResponse cachedResponse;
426     if (!decoder.decode(cachedResponse))
427         return false;
428     json.append("{\n");
429     json.append("\"hash\": ");
430     JSC::appendQuotedJSONStringToBuilder(json, entry.key.hashAsString());
431     json.append(",\n");
432     json.append("\"partition\": ");
433     JSC::appendQuotedJSONStringToBuilder(json, entry.key.partition());
434     json.append(",\n");
435     json.append("\"timestamp\": ");
436     json.appendNumber(entry.timeStamp.count());
437     json.append(",\n");
438     json.append("\"URL\": ");
439     JSC::appendQuotedJSONStringToBuilder(json, cachedResponse.url().string());
440     json.append(",\n");
441     json.append("\"headers\": {\n");
442     bool firstHeader = true;
443     for (auto& header : cachedResponse.httpHeaderFields()) {
444         if (!firstHeader)
445             json.append(",\n");
446         firstHeader = false;
447         json.append("    ");
448         JSC::appendQuotedJSONStringToBuilder(json, header.key);
449         json.append(": ");
450         JSC::appendQuotedJSONStringToBuilder(json, header.value);
451     }
452     json.append("\n}\n");
453     json.append("}");
454     return true;
455 }
456
457 void Cache::dumpContentsToFile()
458 {
459     if (!m_storage)
460         return;
461     auto dumpFileHandle = WebCore::openFile(dumpFilePath(), WebCore::OpenForWrite);
462     if (!dumpFileHandle)
463         return;
464     WebCore::writeToFile(dumpFileHandle, "[\n", 2);
465     m_storage->traverse([dumpFileHandle](const Storage::Entry* entry) {
466         if (!entry) {
467             WebCore::writeToFile(dumpFileHandle, "{}\n]\n", 5);
468             auto handle = dumpFileHandle;
469             WebCore::closeFile(handle);
470             return;
471         }
472         StringBuilder json;
473         if (!entryAsJSON(json, *entry))
474             return;
475         json.append(",\n");
476         auto writeData = json.toString().utf8();
477         WebCore::writeToFile(dumpFileHandle, writeData.data(), writeData.length());
478     });
479 }
480
481 void Cache::clear()
482 {
483     LOG(NetworkCache, "(NetworkProcess) clearing cache");
484     if (m_storage) {
485         m_storage->clear();
486
487         auto queue = WorkQueue::create("com.apple.WebKit.Cache.delete");
488         StringCapture dumpFilePathCapture(dumpFilePath());
489         queue->dispatch([dumpFilePathCapture] {
490             WebCore::deleteFile(dumpFilePathCapture.string());
491         });
492     }
493     if (m_statistics)
494         m_statistics->clear();
495 }
496
497 String Cache::storagePath() const
498 {
499     return m_storage ? m_storage->directoryPath() : String();
500 }
501
502 }
503 }
504
505 #endif