[WTF] Move currentCPUTime and sleep(Seconds) to CPUTime.h and Seconds.h respectively
[WebKit-https.git] / Source / WebKit / NetworkProcess / cache / CacheStorageEngineCache.cpp
1 /*
2  * Copyright (C) 2017 Apple Inc.  All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "CacheStorageEngine.h"
28
29 #include "CacheStorageEngineCaches.h"
30 #include "NetworkCacheCoders.h"
31 #include "NetworkCacheIOChannel.h"
32 #include "NetworkCacheKey.h"
33 #include "NetworkProcess.h"
34 #include <WebCore/CacheQueryOptions.h>
35 #include <WebCore/HTTPParsers.h>
36 #include <pal/SessionID.h>
37 #include <wtf/MainThread.h>
38 #include <wtf/NeverDestroyed.h>
39 #include <wtf/UUID.h>
40 #include <wtf/persistence/PersistentCoders.h>
41 #include <wtf/persistence/PersistentDecoder.h>
42 #include <wtf/persistence/PersistentEncoder.h>
43 #include <wtf/text/StringBuilder.h>
44 #include <wtf/text/StringHash.h>
45
46
47 using namespace WebCore;
48 using namespace WebCore::DOMCacheEngine;
49 using namespace WebKit::NetworkCache;
50
51 namespace WebKit {
52
53 namespace CacheStorage {
54
55 static inline String computeKeyURL(const URL& url)
56 {
57     URL keyURL { url };
58     keyURL.removeQueryAndFragmentIdentifier();
59     return keyURL.string();
60 }
61
62 static inline Vector<uint64_t> queryCache(const Vector<RecordInformation>* records, const ResourceRequest& request, const CacheQueryOptions& options)
63 {
64     if (!records)
65         return { };
66
67     if (!options.ignoreMethod && request.httpMethod() != "GET")
68         return { };
69
70     Vector<uint64_t> results;
71     for (const auto& record : *records) {
72         if (WebCore::DOMCacheEngine::queryCacheMatch(request, record.url, record.hasVaryStar, record.varyHeaders, options))
73             results.append(record.identifier);
74     }
75     return results;
76 }
77
78 static inline void updateVaryInformation(RecordInformation& recordInformation, const ResourceRequest& request, const ResourceResponse& response)
79 {
80     auto varyValue = response.httpHeaderField(WebCore::HTTPHeaderName::Vary);
81     if (varyValue.isNull()) {
82         recordInformation.hasVaryStar = false;
83         recordInformation.varyHeaders = { };
84         return;
85     }
86
87     varyValue.split(',', false, [&](StringView view) {
88         if (!recordInformation.hasVaryStar && stripLeadingAndTrailingHTTPSpaces(view) == "*")
89             recordInformation.hasVaryStar = true;
90         String headerName = view.toString();
91         recordInformation.varyHeaders.add(headerName, request.httpHeaderField(headerName));
92     });
93
94     if (recordInformation.hasVaryStar)
95         recordInformation.varyHeaders = { };
96 }
97
98 RecordInformation Cache::toRecordInformation(const Record& record)
99 {
100     Key key { ASCIILiteral("record"), m_uniqueName, { }, createCanonicalUUIDString(), m_caches.salt() };
101     RecordInformation recordInformation { WTFMove(key), MonotonicTime::now().secondsSinceEpoch().milliseconds(), record.identifier, 0 , record.responseBodySize, record.request.url(), false, { } };
102
103     updateVaryInformation(recordInformation, record.request, record.response);
104
105     return recordInformation;
106 }
107
108 Cache::Cache(Caches& caches, uint64_t identifier, State state, String&& name, String&& uniqueName)
109     : m_caches(caches)
110     , m_state(state)
111     , m_identifier(identifier)
112     , m_name(WTFMove(name))
113     , m_uniqueName(WTFMove(uniqueName))
114 {
115 }
116
117 void Cache::dispose()
118 {
119     m_caches.dispose(*this);
120 }
121
122 void Cache::clearMemoryRepresentation()
123 {
124     m_records = { };
125     m_nextRecordIdentifier = 0;
126     m_state = State::Uninitialized;
127 }
128
129 static RecordInformation isolatedCopy(const RecordInformation& information)
130 {
131     auto result = RecordInformation { information.key, information.insertionTime, information.identifier, information.updateResponseCounter, information.size, information.url.isolatedCopy(), information.hasVaryStar, { } };
132     HashMap<String, String> varyHeaders;
133     for (const auto& keyValue : information.varyHeaders)
134         varyHeaders.set(keyValue.key.isolatedCopy(), keyValue.value.isolatedCopy());
135     result.varyHeaders = WTFMove(varyHeaders);
136     return result;
137 }
138
139 struct TraversalResult {
140     uint64_t cacheIdentifier;
141     HashMap<String, Vector<RecordInformation>> records;
142     Vector<Key> failedRecords;
143 };
144
145 static TraversalResult isolatedCopy(TraversalResult&& result)
146 {
147     HashMap<String, Vector<RecordInformation>> isolatedRecords;
148     for (auto& keyValue : result.records) {
149         auto& recordVector = keyValue.value;
150         for (size_t cptr = 0; cptr < recordVector.size(); cptr++)
151             recordVector[cptr] = isolatedCopy(recordVector[cptr]);
152
153         isolatedRecords.set(keyValue.key.isolatedCopy(), WTFMove(recordVector));
154     }
155
156     // No need to isolate keys since they are isolated through the copy constructor
157     return TraversalResult { result.cacheIdentifier, WTFMove(isolatedRecords), WTFMove(result.failedRecords) };
158 }
159
160 void Cache::open(CompletionCallback&& callback)
161 {
162     if (m_state == State::Open) {
163         callback(std::nullopt);
164         return;
165     }
166     if (m_state == State::Opening) {
167         m_pendingOpeningCallbacks.append(WTFMove(callback));
168         return;
169     }
170     m_state = State::Opening;
171     TraversalResult traversalResult { m_identifier, { }, { } };
172     m_caches.readRecordsList(*this, [caches = makeRef(m_caches), callback = WTFMove(callback), traversalResult = WTFMove(traversalResult)](const auto* storageRecord, const auto&) mutable {
173         if (!storageRecord) {
174             RunLoop::main().dispatch([caches = WTFMove(caches), callback = WTFMove(callback), traversalResult = isolatedCopy(WTFMove(traversalResult)) ]() mutable {
175                 for (auto& key : traversalResult.failedRecords)
176                     caches->removeCacheEntry(key);
177
178                 auto* cache = caches->find(traversalResult.cacheIdentifier);
179                 if (!cache) {
180                     callback(Error::Internal);
181                     return;
182                 }
183                 cache->m_records = WTFMove(traversalResult.records);
184                 cache->finishOpening(WTFMove(callback), std::nullopt);
185             });
186             return;
187         }
188
189         auto decoded = decodeRecordHeader(*storageRecord);
190         if (!decoded) {
191             traversalResult.failedRecords.append(storageRecord->key);
192             return;
193         }
194
195         auto& record = decoded->record;
196         auto insertionTime = decoded->insertionTime;
197
198         RecordInformation recordInformation { storageRecord->key, insertionTime, 0, 0, record.responseBodySize, record.request.url(), false, { } };
199         updateVaryInformation(recordInformation, record.request, record.response);
200
201         auto& sameURLRecords = traversalResult.records.ensure(computeKeyURL(recordInformation.url), [] { return Vector<RecordInformation> { }; }).iterator->value;
202         sameURLRecords.append(WTFMove(recordInformation));
203     });
204 }
205
206 void Cache::finishOpening(CompletionCallback&& callback, std::optional<Error>&& error)
207 {
208     Vector<std::reference_wrapper<RecordInformation>> records;
209     for (auto& value : m_records.values()) {
210         for (auto& record : value)
211             records.append(record);
212     }
213     std::sort(records.begin(), records.end(), [&](const auto& a, const auto& b) {
214         return a.get().insertionTime < b.get().insertionTime;
215     });
216     for (auto& record : records)
217         record.get().identifier = ++m_nextRecordIdentifier;
218
219     if (error) {
220         m_state = State::Uninitialized;
221         callback(error.value());
222         auto callbacks = WTFMove(m_pendingOpeningCallbacks);
223         for (auto& callback : callbacks)
224             callback(error.value());
225         return;
226     }
227     m_state = State::Open;
228
229     callback(std::nullopt);
230     auto callbacks = WTFMove(m_pendingOpeningCallbacks);
231     for (auto& callback : callbacks)
232         callback(std::nullopt);
233 }
234
235 class ReadRecordTaskCounter : public RefCounted<ReadRecordTaskCounter> {
236 public:
237     using ReadRecordsCallback = WTF::Function<void(Vector<Record>&&, Vector<uint64_t>&&)>;
238     static Ref<ReadRecordTaskCounter> create(ReadRecordsCallback&& callback) { return adoptRef(*new ReadRecordTaskCounter(WTFMove(callback))); }
239
240     ~ReadRecordTaskCounter()
241     {
242         ASSERT(RunLoop::isMain());
243         if (!m_callback)
244             return;
245         std::sort(m_records.begin(), m_records.end(), [&] (const auto& a, const auto& b) {
246             return a.identifier < b.identifier;
247         });
248         m_callback(WTFMove(m_records), WTFMove(m_failedRecords));
249     }
250
251     void appendRecord(Expected<Record, Error>&& result, uint64_t recordIdentifier, uint64_t updateCounter)
252     {
253         ASSERT(RunLoop::isMain());
254         if (!result.has_value()) {
255             m_failedRecords.append(recordIdentifier);
256             return;
257         }
258         result.value().identifier = recordIdentifier;
259         result.value().updateResponseCounter = updateCounter;
260         m_records.append(WTFMove(result.value()));
261     }
262
263 private:
264     explicit ReadRecordTaskCounter(ReadRecordsCallback&& callback)
265         : m_callback(WTFMove(callback))
266     {
267     }
268
269     ReadRecordsCallback m_callback;
270     Vector<Record> m_records;
271     Vector<uint64_t> m_failedRecords;
272 };
273
274 void Cache::retrieveRecord(const RecordInformation& record, Ref<ReadRecordTaskCounter>&& taskCounter)
275 {
276     readRecordFromDisk(record, [caches = makeRef(m_caches), identifier = m_identifier, recordIdentifier = record.identifier, updateCounter = record.updateResponseCounter, taskCounter = WTFMove(taskCounter)](Expected<Record, Error>&& result) mutable {
277         auto* cache = caches->find(identifier);
278         if (!cache)
279             return;
280         taskCounter->appendRecord(WTFMove(result), recordIdentifier, updateCounter);
281     });
282 }
283
284 void Cache::retrieveRecords(const URL& url, RecordsCallback&& callback)
285 {
286     ASSERT(m_state == State::Open);
287
288     auto taskCounter = ReadRecordTaskCounter::create([caches = makeRef(m_caches), identifier = m_identifier, callback = WTFMove(callback)](Vector<Record>&& records, Vector<uint64_t>&& failedRecordIdentifiers) mutable {
289         auto* cache = caches->find(identifier);
290         if (cache)
291             cache->removeFromRecordList(failedRecordIdentifiers);
292         callback(WTFMove(records));
293     });
294
295     if (url.isNull()) {
296         for (auto& records : m_records.values()) {
297             for (auto& record : records)
298                 retrieveRecord(record, taskCounter.copyRef());
299         }
300         return;
301     }
302
303     auto* records = recordsFromURL(url);
304     if (!records)
305         return;
306
307     for (auto& record : *records)
308         retrieveRecord(record, taskCounter.copyRef());
309 }
310
311 RecordInformation& Cache::addRecord(Vector<RecordInformation>* records, const Record& record)
312 {
313     if (!records) {
314         auto key = computeKeyURL(record.request.url());
315         ASSERT(!m_records.contains(key));
316         records = &m_records.set(key, Vector<RecordInformation> { }).iterator->value;
317     }
318     records->append(toRecordInformation(record));
319     return records->last();
320 }
321
322 Vector<RecordInformation>* Cache::recordsFromURL(const URL& url)
323 {
324     auto iterator = m_records.find(computeKeyURL(url));
325     if (iterator == m_records.end())
326         return nullptr;
327     return &iterator->value;
328 }
329
330 const Vector<RecordInformation>* Cache::recordsFromURL(const URL& url) const
331 {
332     auto iterator = m_records.find(computeKeyURL(url));
333     if (iterator == m_records.end())
334         return nullptr;
335     return &iterator->value;
336 }
337
338 class AsynchronousPutTaskCounter : public RefCounted<AsynchronousPutTaskCounter> {
339 public:
340     static Ref<AsynchronousPutTaskCounter> create(RecordIdentifiersCallback&& callback) { return adoptRef(*new AsynchronousPutTaskCounter(WTFMove(callback))); }
341     ~AsynchronousPutTaskCounter()
342     {
343         ASSERT(RunLoop::isMain());
344         if (!m_callback)
345             return;
346         if (m_error) {
347             m_callback(makeUnexpected(m_error.value()));
348             return;
349         }
350         m_callback(WTFMove(m_recordIdentifiers));
351     }
352
353     void setError(Error error)
354     {
355         ASSERT(RunLoop::isMain());
356         if (m_error)
357             return;
358
359         m_error = error;
360     }
361
362     void addRecordIdentifier(uint64_t identifier)
363     {
364         m_recordIdentifiers.append(identifier);
365     }
366
367 private:
368     explicit AsynchronousPutTaskCounter(RecordIdentifiersCallback&& callback)
369         : m_callback(WTFMove(callback))
370     {
371     }
372
373     std::optional<Error> m_error;
374     RecordIdentifiersCallback m_callback;
375     Vector<uint64_t> m_recordIdentifiers;
376 };
377
378 void Cache::storeRecords(Vector<Record>&& records, RecordIdentifiersCallback&& callback)
379 {
380     auto taskCounter = AsynchronousPutTaskCounter::create(WTFMove(callback));
381
382     WebCore::CacheQueryOptions options;
383     for (auto& record : records) {
384         auto* sameURLRecords = recordsFromURL(record.request.url());
385         auto matchingRecords = queryCache(sameURLRecords, record.request, options);
386
387         auto position = !matchingRecords.isEmpty() ? sameURLRecords->findMatching([&](const auto& item) { return item.identifier == matchingRecords[0]; }) : notFound;
388
389         if (position == notFound) {
390             record.identifier = ++m_nextRecordIdentifier;
391             taskCounter->addRecordIdentifier(record.identifier);
392
393             auto& recordToWrite = addRecord(sameURLRecords, record);
394             writeRecordToDisk(recordToWrite, WTFMove(record), taskCounter.copyRef(), 0);
395         } else {
396             auto& existingRecord = sameURLRecords->at(position);
397             taskCounter->addRecordIdentifier(existingRecord.identifier);
398             updateRecordToDisk(existingRecord, WTFMove(record), taskCounter.copyRef());
399         }
400     }
401 }
402
403 void Cache::put(Vector<Record>&& records, RecordIdentifiersCallback&& callback)
404 {
405     ASSERT(m_state == State::Open);
406
407     WebCore::CacheQueryOptions options;
408     uint64_t spaceRequired = 0;
409
410     for (auto& record : records) {
411         auto* sameURLRecords = recordsFromURL(record.request.url());
412         auto matchingRecords = queryCache(sameURLRecords, record.request, options);
413
414         auto position = (sameURLRecords && !matchingRecords.isEmpty()) ? sameURLRecords->findMatching([&](const auto& item) { return item.identifier == matchingRecords[0]; }) : notFound;
415
416         spaceRequired += record.responseBodySize;
417         if (position != notFound)
418             spaceRequired -= sameURLRecords->at(position).size;
419     }
420
421     if (m_caches.hasEnoughSpace(spaceRequired)) {
422         storeRecords(WTFMove(records), WTFMove(callback));
423         return;
424     }
425
426     m_caches.requestSpace(spaceRequired, [caches = makeRef(m_caches), identifier = m_identifier, records = WTFMove(records), callback = WTFMove(callback)](std::optional<DOMCacheEngine::Error>&& error) mutable {
427         if (error) {
428             callback(makeUnexpected(error.value()));
429             return;
430         }
431         auto* cache = caches->find(identifier);
432         if (!cache)
433             return;
434
435         cache->storeRecords(WTFMove(records), WTFMove(callback));
436     });
437 }
438
439 void Cache::remove(WebCore::ResourceRequest&& request, WebCore::CacheQueryOptions&& options, RecordIdentifiersCallback&& callback)
440 {
441     ASSERT(m_state == State::Open);
442
443     auto* records = recordsFromURL(request.url());
444     auto recordIdentifiers = queryCache(records, request, options);
445     if (recordIdentifiers.isEmpty()) {
446         callback({ });
447         return;
448     }
449
450     records->removeAllMatching([this, &recordIdentifiers](auto& item) {
451         bool shouldRemove = recordIdentifiers.findMatching([&item](auto identifier) { return identifier == item.identifier; }) != notFound;
452         if (shouldRemove)
453             this->removeRecordFromDisk(item);
454         return shouldRemove;
455     });
456
457     callback(WTFMove(recordIdentifiers));
458 }
459
460 void Cache::removeFromRecordList(const Vector<uint64_t>& recordIdentifiers)
461 {
462     if (recordIdentifiers.isEmpty())
463         return;
464
465     for (auto& records : m_records.values()) {
466         auto* cache = this;
467         records.removeAllMatching([cache, &recordIdentifiers](const auto& item) {
468             return notFound != recordIdentifiers.findMatching([cache, &item](const auto& identifier) {
469                 if (item.identifier != identifier)
470                     return false;
471                 cache->removeRecordFromDisk(item);
472                 return true;
473             });
474         });
475     }
476 }
477
478 void Cache::writeRecordToDisk(const RecordInformation& recordInformation, Record&& record, Ref<AsynchronousPutTaskCounter>&& taskCounter, uint64_t previousRecordSize)
479 {
480     m_caches.writeRecord(*this, recordInformation, WTFMove(record), previousRecordSize, [taskCounter = WTFMove(taskCounter)](std::optional<Error>&& error) {
481         if (error)
482             taskCounter->setError(error.value());
483     });
484 }
485
486 void Cache::updateRecordToDisk(RecordInformation& existingRecord, Record&& record, Ref<AsynchronousPutTaskCounter>&& taskCounter)
487 {
488     ++existingRecord.updateResponseCounter;
489     readRecordFromDisk(existingRecord, [caches = makeRef(m_caches), identifier = m_identifier, recordIdentifier = existingRecord.identifier, record = WTFMove(record), taskCounter = WTFMove(taskCounter)](Expected<Record, Error>&& result) mutable {
490         if (!result.has_value())
491             return;
492
493         auto* cache = caches->find(identifier);
494         if (!cache)
495             return;
496
497         auto* sameURLRecords = cache->recordsFromURL(result.value().request.url());
498         if (!sameURLRecords)
499             return;
500
501         auto position = sameURLRecords->findMatching([&] (const auto& item) { return item.identifier == recordIdentifier; });
502         if (position == notFound)
503             return;
504         auto& recordInfo = sameURLRecords->at(position);
505         auto previousSize = recordInfo.size;
506         recordInfo.size = record.responseBodySize;
507
508         auto& recordFromDisk = result.value();
509         record.requestHeadersGuard = recordFromDisk.requestHeadersGuard;
510         record.request = WTFMove(recordFromDisk.request);
511         record.options = WTFMove(recordFromDisk.options);
512         record.referrer = WTFMove(recordFromDisk.referrer);
513
514         updateVaryInformation(recordInfo, record.request, record.response);
515
516         cache->writeRecordToDisk(recordInfo, WTFMove(record), WTFMove(taskCounter), previousSize);
517     });
518 }
519
520 void Cache::readRecordFromDisk(const RecordInformation& record, WTF::Function<void(Expected<Record, Error>&&)>&& callback)
521 {
522     m_caches.readRecord(record.key, WTFMove(callback));
523 }
524
525 void Cache::removeRecordFromDisk(const RecordInformation& record)
526 {
527     m_caches.removeRecord(record);
528 }
529
530 Storage::Record Cache::encode(const RecordInformation& recordInformation, const Record& record)
531 {
532     WTF::Persistence::Encoder encoder;
533     encoder << recordInformation.insertionTime;
534     encoder << recordInformation.size;
535     encoder << record.requestHeadersGuard;
536     record.request.encodeWithoutPlatformData(encoder);
537     record.options.encodePersistent(encoder);
538     encoder << record.referrer;
539
540     encoder << record.responseHeadersGuard;
541     encoder << record.response;
542     encoder << record.responseBodySize;
543
544     encoder.encodeChecksum();
545
546     Data header(encoder.buffer(), encoder.bufferSize());
547     Data body;
548     WTF::switchOn(record.responseBody, [](const Ref<WebCore::FormData>& formData) {
549         // FIXME: Store form data body.
550     }, [&](const Ref<WebCore::SharedBuffer>& buffer) {
551         body = { reinterpret_cast<const uint8_t*>(buffer->data()), buffer->size() };
552     }, [](const std::nullptr_t&) {
553     });
554
555     return { recordInformation.key, { }, header, body, { } };
556 }
557
558 std::optional<Cache::DecodedRecord> Cache::decodeRecordHeader(const Storage::Record& storage)
559 {
560     WTF::Persistence::Decoder decoder(storage.header.data(), storage.header.size());
561
562     Record record;
563
564     double insertionTime;
565     if (!decoder.decode(insertionTime))
566         return std::nullopt;
567
568     uint64_t size;
569     if (!decoder.decode(size))
570         return std::nullopt;
571
572     if (!decoder.decode(record.requestHeadersGuard))
573         return std::nullopt;
574
575     if (!record.request.decodeWithoutPlatformData(decoder))
576         return std::nullopt;
577
578     if (!FetchOptions::decodePersistent(decoder, record.options))
579         return std::nullopt;
580
581     if (!decoder.decode(record.referrer))
582         return std::nullopt;
583
584     if (!decoder.decode(record.responseHeadersGuard))
585         return std::nullopt;
586
587     if (!decoder.decode(record.response))
588         return std::nullopt;
589
590     if (!decoder.decode(record.responseBodySize))
591         return std::nullopt;
592
593     if (!decoder.verifyChecksum())
594         return std::nullopt;
595
596     return DecodedRecord { insertionTime, size, WTFMove(record) };
597 }
598
599 std::optional<Record> Cache::decode(const Storage::Record& storage)
600 {
601     auto result = decodeRecordHeader(storage);
602
603     if (!result)
604         return std::nullopt;
605
606     auto record = WTFMove(result->record);
607     record.responseBody = WebCore::SharedBuffer::create(storage.body.data(), storage.body.size());
608
609     return WTFMove(record);
610 }
611
612 Vector<Key> Cache::keys() const
613 {
614     Vector<Key> keys;
615     for (auto& records : m_records.values()) {
616         for (auto& record : records)
617             keys.append(record.key);
618     }
619     return keys;
620 }
621
622 } // namespace CacheStorage
623
624 } // namespace WebKit