Use std::chrono types to represent time in response and cache classes
[WebKit-https.git] / Source / WebKit2 / NetworkProcess / cache / NetworkCache.cpp
1 /*
2  * Copyright (C) 2014-2015 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "NetworkCache.h"
28
29 #if ENABLE(NETWORK_CACHE)
30
31 #include "Logging.h"
32 #include "NetworkCacheStatistics.h"
33 #include "NetworkCacheStorage.h"
34 #include <WebCore/CacheValidation.h>
35 #include <WebCore/FileSystem.h>
36 #include <WebCore/HTTPHeaderNames.h>
37 #include <WebCore/NetworkStorageSession.h>
38 #include <WebCore/PlatformCookieJar.h>
39 #include <WebCore/ResourceRequest.h>
40 #include <WebCore/ResourceResponse.h>
41 #include <WebCore/SharedBuffer.h>
42 #include <wtf/NeverDestroyed.h>
43 #include <wtf/text/StringBuilder.h>
44
45 #if PLATFORM(COCOA)
46 #include <notify.h>
47 #endif
48
49 namespace WebKit {
50 namespace NetworkCache {
51
52 Cache& singleton()
53 {
54     static NeverDestroyed<Cache> instance;
55     return instance;
56 }
57
58 bool Cache::initialize(const String& cachePath, bool enableEfficacyLogging)
59 {
60     m_storage = Storage::open(cachePath);
61
62     if (enableEfficacyLogging)
63         m_statistics = Statistics::open(cachePath);
64
65 #if PLATFORM(COCOA)
66     // Triggers with "notifyutil -p com.apple.WebKit.Cache.dump".
67     if (m_storage) {
68         int token;
69         notify_register_dispatch("com.apple.WebKit.Cache.dump", &token, dispatch_get_main_queue(), ^(int) {
70             dumpContentsToFile();
71         });
72     }
73 #endif
74
75     LOG(NetworkCache, "(NetworkProcess) opened cache storage, success %d", !!m_storage);
76     return !!m_storage;
77 }
78
79 void Cache::setMaximumSize(size_t maximumSize)
80 {
81     if (!m_storage)
82         return;
83     m_storage->setMaximumSize(maximumSize);
84 }
85
86 static Key makeCacheKey(const WebCore::ResourceRequest& request)
87 {
88 #if ENABLE(CACHE_PARTITIONING)
89     String partition = request.cachePartition();
90 #else
91     String partition;
92 #endif
93     if (partition.isEmpty())
94         partition = ASCIILiteral("No partition");
95     return { request.httpMethod(), partition, request.url().string()  };
96 }
97
98 static String headerValueForVary(const WebCore::ResourceRequest& request, const String& headerName)
99 {
100     // Explicit handling for cookies is needed because they are added magically by the networking layer.
101     // FIXME: The value might have changed between making the request and retrieving the cookie here.
102     // We could fetch the cookie when making the request but that seems overkill as the case is very rare and it
103     // is a blocking operation. This should be sufficient to cover reasonable cases.
104     if (headerName == httpHeaderNameString(WebCore::HTTPHeaderName::Cookie))
105         return WebCore::cookieRequestHeaderFieldValue(WebCore::NetworkStorageSession::defaultStorageSession(), request.firstPartyForCookies(), request.url());
106     return request.httpHeaderField(headerName);
107 }
108
109 static Vector<std::pair<String, String>> collectVaryingRequestHeaders(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response)
110 {
111     String varyValue = response.httpHeaderField(WebCore::HTTPHeaderName::Vary);
112     if (varyValue.isEmpty())
113         return { };
114     Vector<String> varyingHeaderNames;
115     varyValue.split(',', /*allowEmptyEntries*/ false, varyingHeaderNames);
116     Vector<std::pair<String, String>> varyingRequestHeaders;
117     varyingRequestHeaders.reserveCapacity(varyingHeaderNames.size());
118     for (auto& varyHeaderName : varyingHeaderNames) {
119         String headerName = varyHeaderName.stripWhiteSpace();
120         String headerValue = headerValueForVary(request, headerName);
121         varyingRequestHeaders.append(std::make_pair(headerName, headerValue));
122     }
123     return varyingRequestHeaders;
124 }
125
126 static bool verifyVaryingRequestHeaders(const Vector<std::pair<String, String>>& varyingRequestHeaders, const WebCore::ResourceRequest& request)
127 {
128     for (auto& varyingRequestHeader : varyingRequestHeaders) {
129         // FIXME: Vary: * in response would ideally trigger a cache delete instead of a store.
130         if (varyingRequestHeader.first == "*")
131             return false;
132         String headerValue = headerValueForVary(request, varyingRequestHeader.first);
133         if (headerValue != varyingRequestHeader.second)
134             return false;
135     }
136     return true;
137 }
138
139 static bool cachePolicyAllowsExpired(WebCore::ResourceRequestCachePolicy policy)
140 {
141     switch (policy) {
142     case WebCore::ReturnCacheDataElseLoad:
143     case WebCore::ReturnCacheDataDontLoad:
144         return true;
145     case WebCore::UseProtocolCachePolicy:
146     case WebCore::ReloadIgnoringCacheData:
147         return false;
148     }
149     ASSERT_NOT_REACHED();
150     return false;
151 }
152
153 static bool responseHasExpired(const WebCore::ResourceResponse& response, std::chrono::system_clock::time_point timestamp, Optional<std::chrono::microseconds> maxStale)
154 {
155     if (response.cacheControlContainsNoCache())
156         return true;
157
158     auto age = WebCore::computeCurrentAge(response, timestamp);
159     auto lifetime = WebCore::computeFreshnessLifetimeForHTTPFamily(response, timestamp);
160
161     auto maximumStaleness = maxStale ? maxStale.value() : 0_ms;
162     bool hasExpired = age - lifetime > maximumStaleness;
163
164 #ifndef LOG_DISABLED
165     if (hasExpired)
166         LOG(NetworkCache, "(NetworkProcess) needsRevalidation hasExpired age=%f lifetime=%f max-stale=%g", age, lifetime, maxStale);
167 #endif
168
169     return hasExpired;
170 }
171
172 static bool responseNeedsRevalidation(const WebCore::ResourceResponse& response, const WebCore::ResourceRequest& request, std::chrono::system_clock::time_point timestamp)
173 {
174     auto requestDirectives = WebCore::parseCacheControlDirectives(request.httpHeaderFields());
175     if (requestDirectives.noCache)
176         return true;
177     // For requests we ignore max-age values other than zero.
178     if (requestDirectives.maxAge && requestDirectives.maxAge.value() == 0_ms)
179         return true;
180
181     return responseHasExpired(response, timestamp, requestDirectives.maxStale);
182 }
183
184 static UseDecision makeUseDecision(const Entry& entry, const WebCore::ResourceRequest& request)
185 {
186     if (!verifyVaryingRequestHeaders(entry.varyingRequestHeaders(), request))
187         return UseDecision::NoDueToVaryingHeaderMismatch;
188
189     // We never revalidate in the case of a history navigation.
190     if (cachePolicyAllowsExpired(request.cachePolicy()))
191         return UseDecision::Use;
192
193     if (!responseNeedsRevalidation(entry.response(), request, entry.timeStamp()))
194         return UseDecision::Use;
195
196     if (!entry.response().hasCacheValidatorFields())
197         return UseDecision::NoDueToMissingValidatorFields;
198
199     return UseDecision::Validate;
200 }
201
202 static RetrieveDecision makeRetrieveDecision(const WebCore::ResourceRequest& request)
203 {
204     // FIXME: Support HEAD requests.
205     if (request.httpMethod() != "GET")
206         return RetrieveDecision::NoDueToHTTPMethod;
207     // FIXME: We should be able to validate conditional requests using cache.
208     if (request.isConditional())
209         return RetrieveDecision::NoDueToConditionalRequest;
210     if (request.cachePolicy() == WebCore::ReloadIgnoringCacheData)
211         return RetrieveDecision::NoDueToReloadIgnoringCache;
212
213     return RetrieveDecision::Yes;
214 }
215
216 // http://tools.ietf.org/html/rfc7231#page-48
217 static bool isStatusCodeCacheableByDefault(int statusCode)
218 {
219     switch (statusCode) {
220     case 200: // OK
221     case 203: // Non-Authoritative Information
222     case 204: // No Content
223     case 300: // Multiple Choices
224     case 301: // Moved Permanently
225     case 404: // Not Found
226     case 405: // Method Not Allowed
227     case 410: // Gone
228     case 414: // URI Too Long
229     case 501: // Not Implemented
230         return true;
231     default:
232         return false;
233     }
234 }
235
236 static bool isStatusCodePotentiallyCacheable(int statusCode)
237 {
238     switch (statusCode) {
239     case 201: // Created
240     case 202: // Accepted
241     case 205: // Reset Content
242     case 302: // Found
243     case 303: // See Other
244     case 307: // Temporary redirect
245     case 403: // Forbidden
246     case 406: // Not Acceptable
247     case 415: // Unsupported Media Type
248         return true;
249     default:
250         return false;
251     }
252 }
253
254 static StoreDecision makeStoreDecision(const WebCore::ResourceRequest& originalRequest, const WebCore::ResourceResponse& response)
255 {
256     if (!originalRequest.url().protocolIsInHTTPFamily() || !response.isHTTP())
257         return StoreDecision::NoDueToProtocol;
258
259     if (originalRequest.httpMethod() != "GET")
260         return StoreDecision::NoDueToHTTPMethod;
261
262     auto requestDirectives = WebCore::parseCacheControlDirectives(originalRequest.httpHeaderFields());
263     if (requestDirectives.noStore)
264         return StoreDecision::NoDueToNoStoreRequest;
265
266     if (response.cacheControlContainsNoStore())
267         return StoreDecision::NoDueToNoStoreResponse;
268
269     if (!isStatusCodeCacheableByDefault(response.httpStatusCode())) {
270         // http://tools.ietf.org/html/rfc7234#section-4.3.2
271         bool hasExpirationHeaders = response.expires() || response.cacheControlMaxAge();
272         bool expirationHeadersAllowCaching = isStatusCodePotentiallyCacheable(response.httpStatusCode()) && hasExpirationHeaders;
273         if (!expirationHeadersAllowCaching)
274             return StoreDecision::NoDueToHTTPStatusCode;
275     }
276
277     // Main resource has ResourceLoadPriorityVeryHigh.
278     bool storeUnconditionallyForHistoryNavigation = originalRequest.priority() == WebCore::ResourceLoadPriorityVeryHigh;
279     if (!storeUnconditionallyForHistoryNavigation) {
280         auto now = std::chrono::system_clock::now();
281         bool hasNonZeroLifetime = !response.cacheControlContainsNoCache() && WebCore::computeFreshnessLifetimeForHTTPFamily(response, now) > 0_ms;
282
283         bool possiblyReusable = response.hasCacheValidatorFields() || hasNonZeroLifetime;
284         if (!possiblyReusable)
285             return StoreDecision::NoDueToUnlikelyToReuse;
286     }
287
288     return StoreDecision::Yes;
289 }
290
291 void Cache::retrieve(const WebCore::ResourceRequest& originalRequest, uint64_t webPageID, std::function<void (std::unique_ptr<Entry>)> completionHandler)
292 {
293     ASSERT(isEnabled());
294     ASSERT(originalRequest.url().protocolIsInHTTPFamily());
295
296     LOG(NetworkCache, "(NetworkProcess) retrieving %s priority %u", originalRequest.url().string().ascii().data(), originalRequest.priority());
297
298     if (m_statistics)
299         m_statistics->recordRetrievalRequest(webPageID);
300
301     Key storageKey = makeCacheKey(originalRequest);
302     auto retrieveDecision = makeRetrieveDecision(originalRequest);
303     if (retrieveDecision != RetrieveDecision::Yes) {
304         if (m_statistics)
305             m_statistics->recordNotUsingCacheForRequest(webPageID, storageKey, originalRequest, retrieveDecision);
306
307         completionHandler(nullptr);
308         return;
309     }
310
311     auto startTime = std::chrono::system_clock::now();
312     unsigned priority = originalRequest.priority();
313
314     m_storage->retrieve(storageKey, priority, [this, originalRequest, completionHandler, startTime, storageKey, webPageID](std::unique_ptr<Storage::Record> record) {
315         if (!record) {
316             LOG(NetworkCache, "(NetworkProcess) not found in storage");
317
318             if (m_statistics)
319                 m_statistics->recordRetrievalFailure(webPageID, storageKey, originalRequest);
320
321             completionHandler(nullptr);
322             return false;
323         }
324
325         ASSERT(record->key == storageKey);
326
327         auto entry = Entry::decodeStorageRecord(*record);
328
329         auto useDecision = entry ? makeUseDecision(*entry, originalRequest) : UseDecision::NoDueToDecodeFailure;
330         switch (useDecision) {
331         case UseDecision::Use:
332             break;
333         case UseDecision::Validate:
334             entry->setNeedsValidation();
335             break;
336         default:
337             entry = nullptr;
338         };
339
340 #if !LOG_DISABLED
341         auto elapsedMS = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count();
342         LOG(NetworkCache, "(NetworkProcess) retrieve complete useDecision=%d priority=%u time=%lldms", useDecision, originalRequest.priority(), elapsedMS);
343 #endif
344         completionHandler(WTF::move(entry));
345
346         if (m_statistics)
347             m_statistics->recordRetrievedCachedEntry(webPageID, storageKey, originalRequest, useDecision);
348         return useDecision != UseDecision::NoDueToDecodeFailure;
349     });
350 }
351
352 void Cache::store(const WebCore::ResourceRequest& originalRequest, const WebCore::ResourceResponse& response, RefPtr<WebCore::SharedBuffer>&& responseData, std::function<void (MappedBody&)> completionHandler)
353 {
354     ASSERT(isEnabled());
355     ASSERT(responseData);
356
357     LOG(NetworkCache, "(NetworkProcess) storing %s, partition %s", originalRequest.url().string().latin1().data(), originalRequest.cachePartition().latin1().data());
358
359     StoreDecision storeDecision = makeStoreDecision(originalRequest, response);
360     if (storeDecision != StoreDecision::Yes) {
361         LOG(NetworkCache, "(NetworkProcess) didn't store, storeDecision=%d", storeDecision);
362         if (m_statistics) {
363             auto key = makeCacheKey(originalRequest);
364             m_statistics->recordNotCachingResponse(key, storeDecision);
365         }
366         return;
367     }
368
369     Entry cacheEntry(makeCacheKey(originalRequest), response, WTF::move(responseData), collectVaryingRequestHeaders(originalRequest, response));
370
371     auto record = cacheEntry.encodeAsStorageRecord();
372
373     m_storage->store(record, [completionHandler](bool success, const Data& bodyData) {
374         MappedBody mappedBody;
375 #if ENABLE(SHAREABLE_RESOURCE)
376         if (bodyData.isMap()) {
377             RefPtr<SharedMemory> sharedMemory = SharedMemory::createFromVMBuffer(const_cast<uint8_t*>(bodyData.data()), bodyData.size());
378             mappedBody.shareableResource = sharedMemory ? ShareableResource::create(WTF::move(sharedMemory), 0, bodyData.size()) : nullptr;
379             if (mappedBody.shareableResource)
380                 mappedBody.shareableResource->createHandle(mappedBody.shareableResourceHandle);
381         }
382 #endif
383         completionHandler(mappedBody);
384         LOG(NetworkCache, "(NetworkProcess) store success=%d", success);
385     });
386 }
387
388 void Cache::update(const WebCore::ResourceRequest& originalRequest, const Entry& existingEntry, const WebCore::ResourceResponse& validatingResponse)
389 {
390     LOG(NetworkCache, "(NetworkProcess) updating %s", originalRequest.url().string().latin1().data());
391
392     WebCore::ResourceResponse response = existingEntry.response();
393     WebCore::updateResponseHeadersAfterRevalidation(response, validatingResponse);
394
395     Entry updateEntry(existingEntry.key(), response, existingEntry.buffer(), collectVaryingRequestHeaders(originalRequest, response));
396
397     auto updateRecord = updateEntry.encodeAsStorageRecord();
398
399     m_storage->update(updateRecord, existingEntry.sourceStorageRecord(), [](bool success, const Data&) {
400         LOG(NetworkCache, "(NetworkProcess) updated, success=%d", success);
401     });
402 }
403
404 void Cache::remove(const Key& key)
405 {
406     ASSERT(isEnabled());
407
408     m_storage->remove(key);
409 }
410
411 void Cache::traverse(std::function<void (const Entry*)>&& traverseHandler)
412 {
413     ASSERT(isEnabled());
414
415     m_storage->traverse(0, [traverseHandler](const Storage::Record* record, const Storage::RecordInfo&) {
416         if (!record) {
417             traverseHandler(nullptr);
418             return;
419         }
420
421         auto entry = Entry::decodeStorageRecord(*record);
422         if (!entry)
423             return;
424
425         traverseHandler(entry.get());
426     });
427 }
428
429 String Cache::dumpFilePath() const
430 {
431     return WebCore::pathByAppendingComponent(m_storage->baseDirectoryPath(), "dump.json");
432 }
433
434 void Cache::dumpContentsToFile()
435 {
436     if (!m_storage)
437         return;
438     auto fd = WebCore::openFile(dumpFilePath(), WebCore::OpenForWrite);
439     if (!fd)
440         return;
441     auto prologue = String("{\n\"entries\": [\n").utf8();
442     WebCore::writeToFile(fd, prologue.data(), prologue.length());
443
444     struct Totals {
445         unsigned count { 0 };
446         double worth { 0 };
447         size_t bodySize { 0 };
448     };
449     Totals totals;
450     m_storage->traverse(Storage::TraverseFlag::ComputeWorth, [fd, totals](const Storage::Record* record, const Storage::RecordInfo& info) mutable {
451         if (!record) {
452             StringBuilder epilogue;
453             epilogue.appendLiteral("{}\n],\n");
454             epilogue.appendLiteral("\"totals\": {\n");
455             epilogue.appendLiteral("\"count\": ");
456             epilogue.appendNumber(totals.count);
457             epilogue.appendLiteral(",\n");
458             epilogue.appendLiteral("\"bodySize\": ");
459             epilogue.appendNumber(totals.bodySize);
460             epilogue.appendLiteral(",\n");
461             epilogue.appendLiteral("\"averageWorth\": ");
462             epilogue.appendNumber(totals.count ? totals.worth / totals.count : 0);
463             epilogue.appendLiteral("\n");
464             epilogue.appendLiteral("}\n}\n");
465             auto writeData = epilogue.toString().utf8();
466             WebCore::writeToFile(fd, writeData.data(), writeData.length());
467             WebCore::closeFile(fd);
468             return;
469         }
470         auto entry = Entry::decodeStorageRecord(*record);
471         if (!entry)
472             return;
473         ++totals.count;
474         totals.worth += info.worth;
475         totals.bodySize += info.bodySize;
476
477         StringBuilder json;
478         entry->asJSON(json, info);
479         json.appendLiteral(",\n");
480         auto writeData = json.toString().utf8();
481         WebCore::writeToFile(fd, writeData.data(), writeData.length());
482     });
483 }
484
485 void Cache::clear()
486 {
487     LOG(NetworkCache, "(NetworkProcess) clearing cache");
488     if (m_storage) {
489         m_storage->clear();
490
491         auto queue = WorkQueue::create("com.apple.WebKit.Cache.delete");
492         StringCapture dumpFilePathCapture(dumpFilePath());
493         queue->dispatch([dumpFilePathCapture] {
494             WebCore::deleteFile(dumpFilePathCapture.string());
495         });
496     }
497     if (m_statistics)
498         m_statistics->clear();
499 }
500
501 String Cache::storagePath() const
502 {
503     return m_storage ? m_storage->directoryPath() : String();
504 }
505
506 }
507 }
508
509 #endif