7b3aaa2b297cc9c001c2708c2e41aa56251e89f2
[WebKit-https.git] / Source / WebKit / NetworkProcess / cache / CacheStorageEngineCache.cpp
1 /*
2  * Copyright (C) 2017 Apple Inc.  All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "CacheStorageEngine.h"
28
29 #include "CacheStorageEngineCaches.h"
30 #include "NetworkCacheCoders.h"
31 #include "NetworkCacheIOChannel.h"
32 #include "NetworkCacheKey.h"
33 #include "NetworkProcess.h"
34 #include <WebCore/CacheQueryOptions.h>
35 #include <WebCore/HTTPParsers.h>
36 #include <pal/SessionID.h>
37 #include <wtf/MainThread.h>
38 #include <wtf/NeverDestroyed.h>
39 #include <wtf/UUID.h>
40 #include <wtf/persistence/PersistentCoders.h>
41 #include <wtf/persistence/PersistentDecoder.h>
42 #include <wtf/persistence/PersistentEncoder.h>
43 #include <wtf/text/StringBuilder.h>
44 #include <wtf/text/StringHash.h>
45
46
47 using namespace WebCore;
48 using namespace WebCore::DOMCacheEngine;
49 using namespace WebKit::NetworkCache;
50
51 namespace WebKit {
52
53 namespace CacheStorage {
54
55 static inline String computeKeyURL(const URL& url)
56 {
57     URL keyURL { url };
58     keyURL.removeQueryAndFragmentIdentifier();
59     return keyURL.string();
60 }
61
62 static inline Vector<uint64_t> queryCache(const Vector<RecordInformation>* records, const ResourceRequest& request, const CacheQueryOptions& options)
63 {
64     if (!records)
65         return { };
66
67     if (!options.ignoreMethod && request.httpMethod() != "GET")
68         return { };
69
70     Vector<uint64_t> results;
71     for (const auto& record : *records) {
72         if (WebCore::DOMCacheEngine::queryCacheMatch(request, record.url, record.hasVaryStar, record.varyHeaders, options))
73             results.append(record.identifier);
74     }
75     return results;
76 }
77
78 static inline void updateVaryInformation(RecordInformation& recordInformation, const ResourceRequest& request, const ResourceResponse& response)
79 {
80     auto varyValue = response.httpHeaderField(WebCore::HTTPHeaderName::Vary);
81     if (varyValue.isNull()) {
82         recordInformation.hasVaryStar = false;
83         recordInformation.varyHeaders = { };
84         return;
85     }
86
87     varyValue.split(',', false, [&](StringView view) {
88         if (!recordInformation.hasVaryStar && stripLeadingAndTrailingHTTPSpaces(view) == "*")
89             recordInformation.hasVaryStar = true;
90         String headerName = view.toString();
91         recordInformation.varyHeaders.add(headerName, request.httpHeaderField(headerName));
92     });
93
94     if (recordInformation.hasVaryStar)
95         recordInformation.varyHeaders = { };
96 }
97
98 RecordInformation Cache::toRecordInformation(const Record& record)
99 {
100     Key key { ASCIILiteral("record"), m_uniqueName, { }, createCanonicalUUIDString(), m_caches.salt() };
101     RecordInformation recordInformation { WTFMove(key), monotonicallyIncreasingTimeMS(), record.identifier, 0 , record.responseBodySize, record.request.url(), false, { } };
102
103     updateVaryInformation(recordInformation, record.request, record.response);
104
105     return recordInformation;
106 }
107
108 Cache::Cache(Caches& caches, uint64_t identifier, State state, String&& name, String&& uniqueName)
109     : m_caches(caches)
110     , m_state(state)
111     , m_identifier(identifier)
112     , m_name(WTFMove(name))
113     , m_uniqueName(WTFMove(uniqueName))
114 {
115 }
116
117 void Cache::dispose()
118 {
119     m_caches.dispose(*this);
120 }
121
122 void Cache::clearMemoryRepresentation()
123 {
124     for (auto& records : m_records.values()) {
125         for (auto& record : records)
126             removeRecordFromDisk(record);
127     }
128
129     m_records = { };
130     m_nextRecordIdentifier = 0;
131     m_state = State::Uninitialized;
132 }
133
134 static RecordInformation isolatedCopy(const RecordInformation& information)
135 {
136     auto result = RecordInformation { information.key, information.insertionTime, information.identifier, information.updateResponseCounter, information.size, information.url.isolatedCopy(), information.hasVaryStar, { } };
137     HashMap<String, String> varyHeaders;
138     for (const auto& keyValue : information.varyHeaders)
139         varyHeaders.set(keyValue.key.isolatedCopy(), keyValue.value.isolatedCopy());
140     result.varyHeaders = WTFMove(varyHeaders);
141     return result;
142 }
143
144 struct TraversalResult {
145     uint64_t cacheIdentifier;
146     HashMap<String, Vector<RecordInformation>> records;
147     Vector<Key> failedRecords;
148 };
149
150 static TraversalResult isolatedCopy(TraversalResult&& result)
151 {
152     HashMap<String, Vector<RecordInformation>> isolatedRecords;
153     for (auto& keyValue : result.records) {
154         auto& recordVector = keyValue.value;
155         for (size_t cptr = 0; cptr < recordVector.size(); cptr++)
156             recordVector[cptr] = isolatedCopy(recordVector[cptr]);
157
158         isolatedRecords.set(keyValue.key.isolatedCopy(), WTFMove(recordVector));
159     }
160
161     // No need to isolate keys since they are isolated through the copy constructor
162     return TraversalResult { result.cacheIdentifier, WTFMove(isolatedRecords), WTFMove(result.failedRecords) };
163 }
164
165 void Cache::open(CompletionCallback&& callback)
166 {
167     if (m_state == State::Open) {
168         callback(std::nullopt);
169         return;
170     }
171     if (m_state == State::Opening) {
172         m_pendingOpeningCallbacks.append(WTFMove(callback));
173         return;
174     }
175     m_state = State::Opening;
176     TraversalResult traversalResult { m_identifier, { }, { } };
177     m_caches.readRecordsList(*this, [caches = makeRef(m_caches), callback = WTFMove(callback), traversalResult = WTFMove(traversalResult)](const auto* storageRecord, const auto&) mutable {
178         if (!storageRecord) {
179             RunLoop::main().dispatch([caches = WTFMove(caches), callback = WTFMove(callback), traversalResult = isolatedCopy(WTFMove(traversalResult)) ]() mutable {
180                 for (auto& key : traversalResult.failedRecords)
181                     caches->removeCacheEntry(key);
182
183                 auto* cache = caches->find(traversalResult.cacheIdentifier);
184                 if (!cache) {
185                     callback(Error::Internal);
186                     return;
187                 }
188                 cache->m_records = WTFMove(traversalResult.records);
189                 cache->finishOpening(WTFMove(callback), std::nullopt);
190             });
191             return;
192         }
193
194         auto decoded = decodeRecordHeader(*storageRecord);
195         if (!decoded) {
196             traversalResult.failedRecords.append(storageRecord->key);
197             return;
198         }
199
200         auto& record = decoded->record;
201         auto insertionTime = decoded->insertionTime;
202
203         RecordInformation recordInformation { storageRecord->key, insertionTime, 0, 0, record.responseBodySize, record.request.url(), false, { } };
204         updateVaryInformation(recordInformation, record.request, record.response);
205
206         auto& sameURLRecords = traversalResult.records.ensure(computeKeyURL(recordInformation.url), [] { return Vector<RecordInformation> { }; }).iterator->value;
207         sameURLRecords.append(WTFMove(recordInformation));
208     });
209 }
210
211 void Cache::finishOpening(CompletionCallback&& callback, std::optional<Error>&& error)
212 {
213     Vector<std::reference_wrapper<RecordInformation>> records;
214     for (auto& value : m_records.values()) {
215         for (auto& record : value)
216             records.append(record);
217     }
218     std::sort(records.begin(), records.end(), [&](const auto& a, const auto& b) {
219         return a.get().insertionTime < b.get().insertionTime;
220     });
221     for (auto& record : records)
222         record.get().identifier = ++m_nextRecordIdentifier;
223
224     if (error) {
225         m_state = State::Uninitialized;
226         callback(error.value());
227         auto callbacks = WTFMove(m_pendingOpeningCallbacks);
228         for (auto& callback : callbacks)
229             callback(error.value());
230         return;
231     }
232     m_state = State::Open;
233
234     callback(std::nullopt);
235     auto callbacks = WTFMove(m_pendingOpeningCallbacks);
236     for (auto& callback : callbacks)
237         callback(std::nullopt);
238 }
239
240 class ReadRecordTaskCounter : public RefCounted<ReadRecordTaskCounter> {
241 public:
242     using ReadRecordsCallback = WTF::Function<void(Vector<Record>&&, Vector<uint64_t>&&)>;
243     static Ref<ReadRecordTaskCounter> create(ReadRecordsCallback&& callback) { return adoptRef(*new ReadRecordTaskCounter(WTFMove(callback))); }
244
245     ~ReadRecordTaskCounter()
246     {
247         ASSERT(RunLoop::isMain());
248         if (!m_callback)
249             return;
250         std::sort(m_records.begin(), m_records.end(), [&] (const auto& a, const auto& b) {
251             return a.identifier < b.identifier;
252         });
253         m_callback(WTFMove(m_records), WTFMove(m_failedRecords));
254     }
255
256     void appendRecord(Expected<Record, Error>&& result, uint64_t recordIdentifier, uint64_t updateCounter)
257     {
258         ASSERT(RunLoop::isMain());
259         if (!result.has_value()) {
260             m_failedRecords.append(recordIdentifier);
261             return;
262         }
263         result.value().identifier = recordIdentifier;
264         result.value().updateResponseCounter = updateCounter;
265         m_records.append(WTFMove(result.value()));
266     }
267
268 private:
269     explicit ReadRecordTaskCounter(ReadRecordsCallback&& callback)
270         : m_callback(WTFMove(callback))
271     {
272     }
273
274     ReadRecordsCallback m_callback;
275     Vector<Record> m_records;
276     Vector<uint64_t> m_failedRecords;
277 };
278
279 void Cache::retrieveRecord(const RecordInformation& record, Ref<ReadRecordTaskCounter>&& taskCounter)
280 {
281     readRecordFromDisk(record, [caches = makeRef(m_caches), identifier = m_identifier, recordIdentifier = record.identifier, updateCounter = record.updateResponseCounter, taskCounter = WTFMove(taskCounter)](Expected<Record, Error>&& result) mutable {
282         auto* cache = caches->find(identifier);
283         if (!cache)
284             return;
285         taskCounter->appendRecord(WTFMove(result), recordIdentifier, updateCounter);
286     });
287 }
288
289 void Cache::retrieveRecords(const URL& url, RecordsCallback&& callback)
290 {
291     ASSERT(m_state == State::Open);
292
293     auto taskCounter = ReadRecordTaskCounter::create([caches = makeRef(m_caches), identifier = m_identifier, callback = WTFMove(callback)](Vector<Record>&& records, Vector<uint64_t>&& failedRecordIdentifiers) mutable {
294         auto* cache = caches->find(identifier);
295         if (cache)
296             cache->removeFromRecordList(failedRecordIdentifiers);
297         callback(WTFMove(records));
298     });
299
300     if (url.isNull()) {
301         for (auto& records : m_records.values()) {
302             for (auto& record : records)
303                 retrieveRecord(record, taskCounter.copyRef());
304         }
305         return;
306     }
307
308     auto* records = recordsFromURL(url);
309     if (!records)
310         return;
311
312     for (auto& record : *records)
313         retrieveRecord(record, taskCounter.copyRef());
314 }
315
316 RecordInformation& Cache::addRecord(Vector<RecordInformation>* records, const Record& record)
317 {
318     if (!records) {
319         auto key = computeKeyURL(record.request.url());
320         ASSERT(!m_records.contains(key));
321         records = &m_records.set(key, Vector<RecordInformation> { }).iterator->value;
322     }
323     records->append(toRecordInformation(record));
324     return records->last();
325 }
326
327 Vector<RecordInformation>* Cache::recordsFromURL(const URL& url)
328 {
329     auto iterator = m_records.find(computeKeyURL(url));
330     if (iterator == m_records.end())
331         return nullptr;
332     return &iterator->value;
333 }
334
335 const Vector<RecordInformation>* Cache::recordsFromURL(const URL& url) const
336 {
337     auto iterator = m_records.find(computeKeyURL(url));
338     if (iterator == m_records.end())
339         return nullptr;
340     return &iterator->value;
341 }
342
343 class AsynchronousPutTaskCounter : public RefCounted<AsynchronousPutTaskCounter> {
344 public:
345     static Ref<AsynchronousPutTaskCounter> create(RecordIdentifiersCallback&& callback) { return adoptRef(*new AsynchronousPutTaskCounter(WTFMove(callback))); }
346     ~AsynchronousPutTaskCounter()
347     {
348         ASSERT(RunLoop::isMain());
349         if (!m_callback)
350             return;
351         if (m_error) {
352             m_callback(makeUnexpected(m_error.value()));
353             return;
354         }
355         m_callback(WTFMove(m_recordIdentifiers));
356     }
357
358     void setError(Error error)
359     {
360         ASSERT(RunLoop::isMain());
361         if (m_error)
362             return;
363
364         m_error = error;
365     }
366
367     void addRecordIdentifier(uint64_t identifier)
368     {
369         m_recordIdentifiers.append(identifier);
370     }
371
372 private:
373     explicit AsynchronousPutTaskCounter(RecordIdentifiersCallback&& callback)
374         : m_callback(WTFMove(callback))
375     {
376     }
377
378     std::optional<Error> m_error;
379     RecordIdentifiersCallback m_callback;
380     Vector<uint64_t> m_recordIdentifiers;
381 };
382
383 void Cache::storeRecords(Vector<Record>&& records, RecordIdentifiersCallback&& callback)
384 {
385     auto taskCounter = AsynchronousPutTaskCounter::create(WTFMove(callback));
386
387     WebCore::CacheQueryOptions options;
388     for (auto& record : records) {
389         auto* sameURLRecords = recordsFromURL(record.request.url());
390         auto matchingRecords = queryCache(sameURLRecords, record.request, options);
391
392         auto position = !matchingRecords.isEmpty() ? sameURLRecords->findMatching([&](const auto& item) { return item.identifier == matchingRecords[0]; }) : notFound;
393
394         if (position == notFound) {
395             record.identifier = ++m_nextRecordIdentifier;
396             taskCounter->addRecordIdentifier(record.identifier);
397
398             auto& recordToWrite = addRecord(sameURLRecords, record);
399             writeRecordToDisk(recordToWrite, WTFMove(record), taskCounter.copyRef(), 0);
400         } else {
401             auto& existingRecord = sameURLRecords->at(position);
402             taskCounter->addRecordIdentifier(existingRecord.identifier);
403             updateRecordToDisk(existingRecord, WTFMove(record), taskCounter.copyRef());
404         }
405     }
406 }
407
408 void Cache::put(Vector<Record>&& records, RecordIdentifiersCallback&& callback)
409 {
410     ASSERT(m_state == State::Open);
411
412     WebCore::CacheQueryOptions options;
413     uint64_t spaceRequired = 0;
414
415     for (auto& record : records) {
416         auto* sameURLRecords = recordsFromURL(record.request.url());
417         auto matchingRecords = queryCache(sameURLRecords, record.request, options);
418
419         auto position = (sameURLRecords && !matchingRecords.isEmpty()) ? sameURLRecords->findMatching([&](const auto& item) { return item.identifier == matchingRecords[0]; }) : notFound;
420
421         spaceRequired += record.responseBodySize;
422         if (position != notFound)
423             spaceRequired -= sameURLRecords->at(position).size;
424     }
425
426     if (m_caches.hasEnoughSpace(spaceRequired)) {
427         storeRecords(WTFMove(records), WTFMove(callback));
428         return;
429     }
430
431     m_caches.requestSpace(spaceRequired, [caches = makeRef(m_caches), identifier = m_identifier, records = WTFMove(records), callback = WTFMove(callback)](std::optional<DOMCacheEngine::Error>&& error) mutable {
432         if (error) {
433             callback(makeUnexpected(error.value()));
434             return;
435         }
436         auto* cache = caches->find(identifier);
437         if (!cache)
438             return;
439
440         cache->storeRecords(WTFMove(records), WTFMove(callback));
441     });
442 }
443
444 void Cache::remove(WebCore::ResourceRequest&& request, WebCore::CacheQueryOptions&& options, RecordIdentifiersCallback&& callback)
445 {
446     ASSERT(m_state == State::Open);
447
448     auto* records = recordsFromURL(request.url());
449     auto recordIdentifiers = queryCache(records, request, options);
450     if (recordIdentifiers.isEmpty()) {
451         callback({ });
452         return;
453     }
454
455     records->removeAllMatching([this, &recordIdentifiers](auto& item) {
456         bool shouldRemove = recordIdentifiers.findMatching([&item](auto identifier) { return identifier == item.identifier; }) != notFound;
457         if (shouldRemove)
458             this->removeRecordFromDisk(item);
459         return shouldRemove;
460     });
461
462     callback(WTFMove(recordIdentifiers));
463 }
464
465 void Cache::removeFromRecordList(const Vector<uint64_t>& recordIdentifiers)
466 {
467     if (recordIdentifiers.isEmpty())
468         return;
469
470     for (auto& records : m_records.values()) {
471         auto* cache = this;
472         records.removeAllMatching([cache, &recordIdentifiers](const auto& item) {
473             return notFound != recordIdentifiers.findMatching([cache, &item](const auto& identifier) {
474                 if (item.identifier != identifier)
475                     return false;
476                 cache->removeRecordFromDisk(item);
477                 return true;
478             });
479         });
480     }
481 }
482
483 void Cache::writeRecordToDisk(const RecordInformation& recordInformation, Record&& record, Ref<AsynchronousPutTaskCounter>&& taskCounter, uint64_t previousRecordSize)
484 {
485     m_caches.writeRecord(*this, recordInformation, WTFMove(record), previousRecordSize, [taskCounter = WTFMove(taskCounter)](std::optional<Error>&& error) {
486         if (error)
487             taskCounter->setError(error.value());
488     });
489 }
490
491 void Cache::updateRecordToDisk(RecordInformation& existingRecord, Record&& record, Ref<AsynchronousPutTaskCounter>&& taskCounter)
492 {
493     ++existingRecord.updateResponseCounter;
494     readRecordFromDisk(existingRecord, [caches = makeRef(m_caches), identifier = m_identifier, recordIdentifier = existingRecord.identifier, record = WTFMove(record), taskCounter = WTFMove(taskCounter)](Expected<Record, Error>&& result) mutable {
495         if (!result.has_value())
496             return;
497
498         auto* cache = caches->find(identifier);
499         if (!cache)
500             return;
501
502         auto* sameURLRecords = cache->recordsFromURL(result.value().request.url());
503         if (!sameURLRecords)
504             return;
505
506         auto position = sameURLRecords->findMatching([&] (const auto& item) { return item.identifier == recordIdentifier; });
507         if (position == notFound)
508             return;
509         auto& recordInfo = sameURLRecords->at(position);
510         auto previousSize = recordInfo.size;
511         recordInfo.size = record.responseBodySize;
512
513         auto& recordFromDisk = result.value();
514         record.requestHeadersGuard = recordFromDisk.requestHeadersGuard;
515         record.request = WTFMove(recordFromDisk.request);
516         record.options = WTFMove(recordFromDisk.options);
517         record.referrer = WTFMove(recordFromDisk.referrer);
518
519         updateVaryInformation(recordInfo, record.request, record.response);
520
521         cache->writeRecordToDisk(recordInfo, WTFMove(record), WTFMove(taskCounter), previousSize);
522     });
523 }
524
525 void Cache::readRecordFromDisk(const RecordInformation& record, WTF::Function<void(Expected<Record, Error>&&)>&& callback)
526 {
527     m_caches.readRecord(record.key, WTFMove(callback));
528 }
529
530 void Cache::removeRecordFromDisk(const RecordInformation& record)
531 {
532     m_caches.removeRecord(record);
533 }
534
535 Storage::Record Cache::encode(const RecordInformation& recordInformation, const Record& record)
536 {
537     WTF::Persistence::Encoder encoder;
538     encoder << recordInformation.insertionTime;
539     encoder << recordInformation.size;
540     encoder << record.requestHeadersGuard;
541     record.request.encodeWithoutPlatformData(encoder);
542     record.options.encodePersistent(encoder);
543     encoder << record.referrer;
544
545     encoder << record.responseHeadersGuard;
546     encoder << record.response;
547     encoder << record.responseBodySize;
548
549     encoder.encodeChecksum();
550
551     Data header(encoder.buffer(), encoder.bufferSize());
552     Data body;
553     WTF::switchOn(record.responseBody, [](const Ref<WebCore::FormData>& formData) {
554         // FIXME: Store form data body.
555     }, [&](const Ref<WebCore::SharedBuffer>& buffer) {
556         body = { reinterpret_cast<const uint8_t*>(buffer->data()), buffer->size() };
557     }, [](const std::nullptr_t&) {
558     });
559
560     return { recordInformation.key, { }, header, body, { } };
561 }
562
563 std::optional<Cache::DecodedRecord> Cache::decodeRecordHeader(const Storage::Record& storage)
564 {
565     WTF::Persistence::Decoder decoder(storage.header.data(), storage.header.size());
566
567     Record record;
568
569     double insertionTime;
570     if (!decoder.decode(insertionTime))
571         return std::nullopt;
572
573     uint64_t size;
574     if (!decoder.decode(size))
575         return std::nullopt;
576
577     if (!decoder.decode(record.requestHeadersGuard))
578         return std::nullopt;
579
580     if (!record.request.decodeWithoutPlatformData(decoder))
581         return std::nullopt;
582
583     if (!FetchOptions::decodePersistent(decoder, record.options))
584         return std::nullopt;
585
586     if (!decoder.decode(record.referrer))
587         return std::nullopt;
588
589     if (!decoder.decode(record.responseHeadersGuard))
590         return std::nullopt;
591
592     if (!decoder.decode(record.response))
593         return std::nullopt;
594
595     if (!decoder.decode(record.responseBodySize))
596         return std::nullopt;
597
598     if (!decoder.verifyChecksum())
599         return std::nullopt;
600
601     return DecodedRecord { insertionTime, size, WTFMove(record) };
602 }
603
604 std::optional<Record> Cache::decode(const Storage::Record& storage)
605 {
606     auto result = decodeRecordHeader(storage);
607
608     if (!result)
609         return std::nullopt;
610
611     auto record = WTFMove(result->record);
612     record.responseBody = WebCore::SharedBuffer::create(storage.body.data(), storage.body.size());
613
614     return WTFMove(record);
615 }
616
617 Vector<Key> Cache::keys() const
618 {
619     Vector<Key> keys;
620     for (auto& records : m_records.values()) {
621         for (auto& record : records)
622             keys.append(record.key);
623     }
624     return keys;
625 }
626
627 } // namespace CacheStorage
628
629 } // namespace WebKit