Shrink various loading-related enums to shrink CachedResource
[WebKit-https.git] / Source / WebCore / inspector / agents / InspectorNetworkAgent.cpp
1 /*
2  * Copyright (C) 2011 Google Inc. All rights reserved.
3  * Copyright (C) 2015-2018 Apple Inc. All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are
7  * met:
8  *
9  *     * Redistributions of source code must retain the above copyright
10  * notice, this list of conditions and the following disclaimer.
11  *     * Redistributions in binary form must reproduce the above
12  * copyright notice, this list of conditions and the following disclaimer
13  * in the documentation and/or other materials provided with the
14  * distribution.
15  *     * Neither the name of Google Inc. nor the names of its
16  * contributors may be used to endorse or promote products derived from
17  * this software without specific prior written permission.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30  */
31
32 #include "config.h"
33 #include "InspectorNetworkAgent.h"
34
35 #include "CachedCSSStyleSheet.h"
36 #include "CachedRawResource.h"
37 #include "CachedResource.h"
38 #include "CachedResourceLoader.h"
39 #include "CachedResourceRequestInitiators.h"
40 #include "CachedScript.h"
41 #include "Document.h"
42 #include "DocumentLoader.h"
43 #include "DocumentThreadableLoader.h"
44 #include "Frame.h"
45 #include "FrameLoader.h"
46 #include "HTTPHeaderMap.h"
47 #include "HTTPHeaderNames.h"
48 #include "InspectorTimelineAgent.h"
49 #include "InstrumentingAgents.h"
50 #include "JSMainThreadExecState.h"
51 #include "JSWebSocket.h"
52 #include "LoaderStrategy.h"
53 #include "MIMETypeRegistry.h"
54 #include "MemoryCache.h"
55 #include "NetworkResourcesData.h"
56 #include "Page.h"
57 #include "PlatformStrategies.h"
58 #include "ProgressTracker.h"
59 #include "ResourceError.h"
60 #include "ResourceLoader.h"
61 #include "ResourceRequest.h"
62 #include "ResourceResponse.h"
63 #include "RuntimeEnabledFeatures.h"
64 #include "ScriptState.h"
65 #include "ScriptableDocumentParser.h"
66 #include "SubresourceLoader.h"
67 #include "TextResourceDecoder.h"
68 #include "ThreadableLoaderClient.h"
69 #include "URL.h"
70 #include "WebSocket.h"
71 #include "WebSocketChannel.h"
72 #include "WebSocketFrame.h"
73 #include <JavaScriptCore/ContentSearchUtilities.h>
74 #include <JavaScriptCore/IdentifiersFactory.h>
75 #include <JavaScriptCore/InjectedScript.h>
76 #include <JavaScriptCore/InjectedScriptManager.h>
77 #include <JavaScriptCore/JSCInlines.h>
78 #include <JavaScriptCore/ScriptCallStack.h>
79 #include <JavaScriptCore/ScriptCallStackFactory.h>
80 #include <wtf/JSONValues.h>
81 #include <wtf/Lock.h>
82 #include <wtf/RefPtr.h>
83 #include <wtf/Stopwatch.h>
84 #include <wtf/text/Base64.h>
85 #include <wtf/text/StringBuilder.h>
86
87 typedef Inspector::NetworkBackendDispatcherHandler::LoadResourceCallback LoadResourceCallback;
88
89 namespace WebCore {
90
91 using namespace Inspector;
92
93 namespace {
94
95 class InspectorThreadableLoaderClient final : public ThreadableLoaderClient {
96     WTF_MAKE_NONCOPYABLE(InspectorThreadableLoaderClient);
97 public:
98     InspectorThreadableLoaderClient(RefPtr<LoadResourceCallback>&& callback)
99         : m_callback(WTFMove(callback))
100     {
101     }
102
103     virtual ~InspectorThreadableLoaderClient() = default;
104
105     void didReceiveResponse(unsigned long, const ResourceResponse& response) override
106     {
107         m_mimeType = response.mimeType();
108         m_statusCode = response.httpStatusCode();
109
110         // FIXME: This assumes text only responses. We should support non-text responses as well.
111         TextEncoding textEncoding(response.textEncodingName());
112         bool useDetector = false;
113         if (!textEncoding.isValid()) {
114             textEncoding = UTF8Encoding();
115             useDetector = true;
116         }
117
118         m_decoder = TextResourceDecoder::create("text/plain"_s, textEncoding, useDetector);
119     }
120
121     void didReceiveData(const char* data, int dataLength) override
122     {
123         if (!dataLength)
124             return;
125
126         if (dataLength == -1)
127             dataLength = strlen(data);
128
129         m_responseText.append(m_decoder->decode(data, dataLength));
130     }
131
132     void didFinishLoading(unsigned long) override
133     {
134         if (m_decoder)
135             m_responseText.append(m_decoder->flush());
136
137         m_callback->sendSuccess(m_responseText.toString(), m_mimeType, m_statusCode);
138         dispose();
139     }
140
141     void didFail(const ResourceError& error) override
142     {
143         m_callback->sendFailure(error.isAccessControl() ? "Loading resource for inspector failed access control check"_s : "Loading resource for inspector failed"_s);
144         dispose();
145     }
146
147     void setLoader(RefPtr<ThreadableLoader>&& loader)
148     {
149         m_loader = WTFMove(loader);
150     }
151
152 private:
153     void dispose()
154     {
155         m_loader = nullptr;
156         delete this;
157     }
158
159     RefPtr<LoadResourceCallback> m_callback;
160     RefPtr<ThreadableLoader> m_loader;
161     RefPtr<TextResourceDecoder> m_decoder;
162     String m_mimeType;
163     StringBuilder m_responseText;
164     int m_statusCode;
165 };
166
167 } // namespace
168
169 InspectorNetworkAgent::InspectorNetworkAgent(WebAgentContext& context)
170     : InspectorAgentBase("Network"_s, context)
171     , m_frontendDispatcher(std::make_unique<Inspector::NetworkFrontendDispatcher>(context.frontendRouter))
172     , m_backendDispatcher(Inspector::NetworkBackendDispatcher::create(context.backendDispatcher, this))
173     , m_injectedScriptManager(context.injectedScriptManager)
174     , m_resourcesData(std::make_unique<NetworkResourcesData>())
175 {
176 }
177
178 void InspectorNetworkAgent::didCreateFrontendAndBackend(Inspector::FrontendRouter*, Inspector::BackendDispatcher*)
179 {
180 }
181
182 void InspectorNetworkAgent::willDestroyFrontendAndBackend(Inspector::DisconnectReason)
183 {
184     ErrorString unused;
185     disable(unused);
186 }
187
188 static Ref<JSON::Object> buildObjectForHeaders(const HTTPHeaderMap& headers)
189 {
190     Ref<JSON::Object> headersObject = JSON::Object::create();
191
192     for (const auto& header : headers)
193         headersObject->setString(header.key, header.value);
194     return headersObject;
195 }
196
197 Ref<Inspector::Protocol::Network::ResourceTiming> InspectorNetworkAgent::buildObjectForTiming(const NetworkLoadMetrics& timing, ResourceLoader& resourceLoader)
198 {
199     MonotonicTime startTime = resourceLoader.loadTiming().startTime();
200     Seconds startTimeInInspector = m_environment.executionStopwatch()->elapsedTimeSince(startTime);
201
202     return Inspector::Protocol::Network::ResourceTiming::create()
203         .setStartTime(startTimeInInspector.seconds())
204         .setDomainLookupStart(timing.domainLookupStart.milliseconds())
205         .setDomainLookupEnd(timing.domainLookupEnd.milliseconds())
206         .setConnectStart(timing.connectStart.milliseconds())
207         .setConnectEnd(timing.connectEnd.milliseconds())
208         .setSecureConnectionStart(timing.secureConnectionStart.milliseconds())
209         .setRequestStart(timing.requestStart.milliseconds())
210         .setResponseStart(timing.responseStart.milliseconds())
211         .release();
212 }
213
214 static Inspector::Protocol::Network::Metrics::Priority toProtocol(NetworkLoadPriority priority)
215 {
216     switch (priority) {
217     case NetworkLoadPriority::Low:
218         return Inspector::Protocol::Network::Metrics::Priority::Low;
219     case NetworkLoadPriority::Medium:
220         return Inspector::Protocol::Network::Metrics::Priority::Medium;
221     case NetworkLoadPriority::High:
222         return Inspector::Protocol::Network::Metrics::Priority::High;
223     }
224
225     ASSERT_NOT_REACHED();
226     return Inspector::Protocol::Network::Metrics::Priority::Medium;
227 }
228
229 Ref<Inspector::Protocol::Network::Metrics> InspectorNetworkAgent::buildObjectForMetrics(const NetworkLoadMetrics& networkLoadMetrics)
230 {
231     auto metrics = Inspector::Protocol::Network::Metrics::create().release();
232
233     if (!networkLoadMetrics.protocol.isNull())
234         metrics->setProtocol(networkLoadMetrics.protocol);
235     if (networkLoadMetrics.priority)
236         metrics->setPriority(toProtocol(*networkLoadMetrics.priority));
237     if (networkLoadMetrics.remoteAddress)
238         metrics->setRemoteAddress(*networkLoadMetrics.remoteAddress);
239     if (networkLoadMetrics.connectionIdentifier)
240         metrics->setConnectionIdentifier(*networkLoadMetrics.connectionIdentifier);
241     if (networkLoadMetrics.requestHeaders)
242         metrics->setRequestHeaders(buildObjectForHeaders(*networkLoadMetrics.requestHeaders));
243
244     if (networkLoadMetrics.requestHeaderBytesSent)
245         metrics->setRequestHeaderBytesSent(*networkLoadMetrics.requestHeaderBytesSent);
246     if (networkLoadMetrics.requestBodyBytesSent)
247         metrics->setRequestBodyBytesSent(*networkLoadMetrics.requestBodyBytesSent);
248     if (networkLoadMetrics.responseHeaderBytesReceived)
249         metrics->setResponseHeaderBytesReceived(*networkLoadMetrics.responseHeaderBytesReceived);
250     if (networkLoadMetrics.responseBodyBytesReceived)
251         metrics->setResponseBodyBytesReceived(*networkLoadMetrics.responseBodyBytesReceived);
252     if (networkLoadMetrics.responseBodyDecodedSize)
253         metrics->setResponseBodyDecodedSize(*networkLoadMetrics.responseBodyDecodedSize);
254
255     return metrics;
256 }
257
258 static Ref<Inspector::Protocol::Network::Request> buildObjectForResourceRequest(const ResourceRequest& request)
259 {
260     auto requestObject = Inspector::Protocol::Network::Request::create()
261         .setUrl(request.url().string())
262         .setMethod(request.httpMethod())
263         .setHeaders(buildObjectForHeaders(request.httpHeaderFields()))
264         .release();
265     if (request.httpBody() && !request.httpBody()->isEmpty()) {
266         auto bytes = request.httpBody()->flatten();
267         requestObject->setPostData(String::fromUTF8WithLatin1Fallback(bytes.data(), bytes.size()));
268     }
269     return requestObject;
270 }
271
272 static Inspector::Protocol::Network::Response::Source responseSource(ResourceResponse::Source source)
273 {
274     switch (source) {
275     case ResourceResponse::Source::ApplicationCache:
276         // FIXME: Add support for ApplicationCache in inspector.
277     case ResourceResponse::Source::Unknown:
278         return Inspector::Protocol::Network::Response::Source::Unknown;
279     case ResourceResponse::Source::Network:
280         return Inspector::Protocol::Network::Response::Source::Network;
281     case ResourceResponse::Source::MemoryCache:
282     case ResourceResponse::Source::MemoryCacheAfterValidation:
283         return Inspector::Protocol::Network::Response::Source::MemoryCache;
284     case ResourceResponse::Source::DiskCache:
285     case ResourceResponse::Source::DiskCacheAfterValidation:
286         return Inspector::Protocol::Network::Response::Source::DiskCache;
287     case ResourceResponse::Source::ServiceWorker:
288         return Inspector::Protocol::Network::Response::Source::ServiceWorker;
289     }
290
291     ASSERT_NOT_REACHED();
292     return Inspector::Protocol::Network::Response::Source::Unknown;
293 }
294
295 RefPtr<Inspector::Protocol::Network::Response> InspectorNetworkAgent::buildObjectForResourceResponse(const ResourceResponse& response, ResourceLoader* resourceLoader)
296 {
297     if (response.isNull())
298         return nullptr;
299
300     Ref<JSON::Object> headers = buildObjectForHeaders(response.httpHeaderFields());
301
302     auto responseObject = Inspector::Protocol::Network::Response::create()
303         .setUrl(response.url().string())
304         .setStatus(response.httpStatusCode())
305         .setStatusText(response.httpStatusText())
306         .setHeaders(WTFMove(headers))
307         .setMimeType(response.mimeType())
308         .setSource(responseSource(response.source()))
309         .release();
310
311     if (resourceLoader)
312         responseObject->setTiming(buildObjectForTiming(response.deprecatedNetworkLoadMetrics(), *resourceLoader));
313
314     return WTFMove(responseObject);
315 }
316
317 Ref<Inspector::Protocol::Network::CachedResource> InspectorNetworkAgent::buildObjectForCachedResource(CachedResource* cachedResource)
318 {
319     auto resourceObject = Inspector::Protocol::Network::CachedResource::create()
320         .setUrl(cachedResource->url())
321         .setType(InspectorPageAgent::cachedResourceTypeJSON(*cachedResource))
322         .setBodySize(cachedResource->encodedSize())
323         .release();
324
325     auto resourceResponse = buildObjectForResourceResponse(cachedResource->response(), cachedResource->loader());
326     resourceObject->setResponse(WTFMove(resourceResponse));
327
328     String sourceMappingURL = InspectorPageAgent::sourceMapURLForResource(cachedResource);
329     if (!sourceMappingURL.isEmpty())
330         resourceObject->setSourceMapURL(sourceMappingURL);
331
332     return resourceObject;
333 }
334
335 InspectorNetworkAgent::~InspectorNetworkAgent()
336 {
337     if (m_enabled) {
338         ErrorString unused;
339         disable(unused);
340     }
341     ASSERT(!m_instrumentingAgents.inspectorNetworkAgent());
342 }
343
344 double InspectorNetworkAgent::timestamp()
345 {
346     return m_environment.executionStopwatch()->elapsedTime().seconds();
347 }
348
349 void InspectorNetworkAgent::willSendRequest(unsigned long identifier, DocumentLoader* loader, ResourceRequest& request, const ResourceResponse& redirectResponse, InspectorPageAgent::ResourceType type)
350 {
351     if (request.hiddenFromInspector()) {
352         m_hiddenRequestIdentifiers.add(identifier);
353         return;
354     }
355
356     double sendTimestamp = timestamp();
357     WallTime walltime = WallTime::now();
358
359     String requestId = IdentifiersFactory::requestId(identifier);
360     String frameId = frameIdentifier(loader);
361     String loaderId = loaderIdentifier(loader);
362     String targetId = request.initiatorIdentifier();
363
364     if (type == InspectorPageAgent::OtherResource) {
365         if (m_loadingXHRSynchronously)
366             type = InspectorPageAgent::XHRResource;
367         else if (loader && equalIgnoringFragmentIdentifier(request.url(), loader->url()) && !loader->isCommitted())
368             type = InspectorPageAgent::DocumentResource;
369         else if (loader) {
370             for (auto& linkIcon : loader->linkIcons()) {
371                 if (equalIgnoringFragmentIdentifier(request.url(), linkIcon.url)) {
372                     type = InspectorPageAgent::ImageResource;
373                     break;
374                 }
375             }
376         }
377     }
378
379     m_resourcesData->resourceCreated(requestId, loaderId, type);
380
381     for (auto& entry : m_extraRequestHeaders)
382         request.setHTTPHeaderField(entry.key, entry.value);
383
384     auto protocolResourceType = InspectorPageAgent::resourceTypeJSON(type);
385
386     Document* document = loader && loader->frame() ? loader->frame()->document() : nullptr;
387     auto initiatorObject = buildInitiatorObject(document);
388
389     String url = loader ? loader->url().string() : request.url();
390     m_frontendDispatcher->requestWillBeSent(requestId, frameId, loaderId, url, buildObjectForResourceRequest(request), sendTimestamp, walltime.secondsSinceEpoch().seconds(), initiatorObject, buildObjectForResourceResponse(redirectResponse, nullptr), type != InspectorPageAgent::OtherResource ? &protocolResourceType : nullptr, targetId.isEmpty() ? nullptr : &targetId);
391 }
392
393 static InspectorPageAgent::ResourceType resourceTypeForCachedResource(CachedResource* resource)
394 {
395     if (resource)
396         return InspectorPageAgent::inspectorResourceType(*resource);
397     return InspectorPageAgent::OtherResource;
398 }
399
400 static InspectorPageAgent::ResourceType resourceTypeForLoadType(InspectorInstrumentation::LoadType loadType)
401 {
402     switch (loadType) {
403     case InspectorInstrumentation::LoadType::Ping:
404         return InspectorPageAgent::PingResource;
405     case InspectorInstrumentation::LoadType::Beacon:
406         return InspectorPageAgent::BeaconResource;
407     }
408
409     ASSERT_NOT_REACHED();
410     return InspectorPageAgent::OtherResource;
411 }
412
413 void InspectorNetworkAgent::willSendRequest(unsigned long identifier, DocumentLoader* loader, ResourceRequest& request, const ResourceResponse& redirectResponse)
414 {
415     auto* cachedResource = loader ? InspectorPageAgent::cachedResource(loader->frame(), request.url()) : nullptr;
416     willSendRequest(identifier, loader, request, redirectResponse, resourceTypeForCachedResource(cachedResource));
417 }
418
419 void InspectorNetworkAgent::willSendRequestOfType(unsigned long identifier, DocumentLoader* loader, ResourceRequest& request, InspectorInstrumentation::LoadType loadType)
420 {
421     willSendRequest(identifier, loader, request, ResourceResponse(), resourceTypeForLoadType(loadType));
422 }
423
424 void InspectorNetworkAgent::didReceiveResponse(unsigned long identifier, DocumentLoader* loader, const ResourceResponse& response, ResourceLoader* resourceLoader)
425 {
426     if (m_hiddenRequestIdentifiers.contains(identifier))
427         return;
428
429     String requestId = IdentifiersFactory::requestId(identifier);
430
431     std::optional<ResourceResponse> realResponse;
432     if (platformStrategies()->loaderStrategy()->havePerformedSecurityChecks(response)) {
433         callOnMainThreadAndWait([&] {
434             // We do not need to isolate response since it comes straight from IPC, but we might want to isolate it for extra safety.
435             auto response = platformStrategies()->loaderStrategy()->responseFromResourceLoadIdentifier(identifier);
436             if (!response.isNull())
437                 realResponse = WTFMove(response);
438         });
439     }
440
441     RefPtr<Inspector::Protocol::Network::Response> resourceResponse = buildObjectForResourceResponse(realResponse ? *realResponse : response, resourceLoader);
442
443     bool isNotModified = response.httpStatusCode() == 304;
444
445     CachedResource* cachedResource = nullptr;
446     if (resourceLoader && resourceLoader->isSubresourceLoader() && !isNotModified)
447         cachedResource = static_cast<SubresourceLoader*>(resourceLoader)->cachedResource();
448     if (!cachedResource && loader)
449         cachedResource = InspectorPageAgent::cachedResource(loader->frame(), response.url());
450
451     if (cachedResource) {
452         // Use mime type from cached resource in case the one in response is empty.
453         if (resourceResponse && response.mimeType().isEmpty())
454             resourceResponse->setString(Inspector::Protocol::Network::Response::MimeType, cachedResource->response().mimeType());
455         m_resourcesData->addCachedResource(requestId, cachedResource);
456     }
457
458     InspectorPageAgent::ResourceType type = m_resourcesData->resourceType(requestId);
459     InspectorPageAgent::ResourceType newType = cachedResource ? InspectorPageAgent::inspectorResourceType(*cachedResource) : type;
460
461     // FIXME: XHRResource is returned for CachedResource::Type::RawResource, it should be OtherResource unless it truly is an XHR.
462     // RawResource is used for loading worker scripts, and those should stay as ScriptResource and not change to XHRResource.
463     if (type != newType && newType != InspectorPageAgent::XHRResource && newType != InspectorPageAgent::OtherResource)
464         type = newType;
465
466     String frameId = frameIdentifier(loader);
467     String loaderId = loaderIdentifier(loader);
468
469     m_resourcesData->responseReceived(requestId, frameId, response, type, shouldForceBufferingNetworkResourceData());
470
471     m_frontendDispatcher->responseReceived(requestId, frameId, loaderId, timestamp(), InspectorPageAgent::resourceTypeJSON(type), resourceResponse);
472
473     // If we revalidated the resource and got Not modified, send content length following didReceiveResponse
474     // as there will be no calls to didReceiveData from the network stack.
475     if (isNotModified && cachedResource && cachedResource->encodedSize())
476         didReceiveData(identifier, nullptr, cachedResource->encodedSize(), 0);
477 }
478
479 void InspectorNetworkAgent::didReceiveData(unsigned long identifier, const char* data, int dataLength, int encodedDataLength)
480 {
481     if (m_hiddenRequestIdentifiers.contains(identifier))
482         return;
483
484     String requestId = IdentifiersFactory::requestId(identifier);
485
486     if (data) {
487         NetworkResourcesData::ResourceData const* resourceData = m_resourcesData->data(requestId);
488         if (resourceData && !m_loadingXHRSynchronously)
489             m_resourcesData->maybeAddResourceData(requestId, data, dataLength);
490     }
491
492     m_frontendDispatcher->dataReceived(requestId, timestamp(), dataLength, encodedDataLength);
493 }
494
495 void InspectorNetworkAgent::didFinishLoading(unsigned long identifier, DocumentLoader* loader, const NetworkLoadMetrics& networkLoadMetrics, ResourceLoader* resourceLoader)
496 {
497     if (m_hiddenRequestIdentifiers.remove(identifier))
498         return;
499
500     double elapsedFinishTime;
501     if (resourceLoader && networkLoadMetrics.isComplete()) {
502         MonotonicTime startTime = resourceLoader->loadTiming().startTime();
503         Seconds startTimeInInspector = m_environment.executionStopwatch()->elapsedTimeSince(startTime);
504         elapsedFinishTime = (startTimeInInspector + networkLoadMetrics.responseEnd).seconds();
505     } else
506         elapsedFinishTime = timestamp();
507
508     String requestId = IdentifiersFactory::requestId(identifier);
509     if (loader && m_resourcesData->resourceType(requestId) == InspectorPageAgent::DocumentResource)
510         m_resourcesData->addResourceSharedBuffer(requestId, loader->frameLoader()->documentLoader()->mainResourceData(), loader->frame()->document()->encoding());
511
512     m_resourcesData->maybeDecodeDataToContent(requestId);
513
514     String sourceMappingURL;
515     NetworkResourcesData::ResourceData const* resourceData = m_resourcesData->data(requestId);
516     if (resourceData && resourceData->cachedResource())
517         sourceMappingURL = InspectorPageAgent::sourceMapURLForResource(resourceData->cachedResource());
518
519     std::optional<NetworkLoadMetrics> realMetrics;
520     if (platformStrategies()->loaderStrategy()->shouldPerformSecurityChecks() && !networkLoadMetrics.isComplete()) {
521         callOnMainThreadAndWait([&] {
522             realMetrics = platformStrategies()->loaderStrategy()->networkMetricsFromResourceLoadIdentifier(identifier).isolatedCopy();
523         });
524     }
525     RefPtr<Inspector::Protocol::Network::Metrics> metrics = buildObjectForMetrics(realMetrics ? *realMetrics : networkLoadMetrics);
526
527     m_frontendDispatcher->loadingFinished(requestId, elapsedFinishTime, !sourceMappingURL.isEmpty() ? &sourceMappingURL : nullptr, metrics);
528 }
529
530 void InspectorNetworkAgent::didFailLoading(unsigned long identifier, DocumentLoader* loader, const ResourceError& error)
531 {
532     if (m_hiddenRequestIdentifiers.remove(identifier))
533         return;
534
535     String requestId = IdentifiersFactory::requestId(identifier);
536
537     if (loader && m_resourcesData->resourceType(requestId) == InspectorPageAgent::DocumentResource) {
538         Frame* frame = loader->frame();
539         if (frame && frame->loader().documentLoader() && frame->document()) {
540             m_resourcesData->addResourceSharedBuffer(requestId,
541                 frame->loader().documentLoader()->mainResourceData(),
542                 frame->document()->encoding());
543         }
544     }
545
546     bool canceled = error.isCancellation();
547     m_frontendDispatcher->loadingFailed(requestId, timestamp(), error.localizedDescription(), canceled ? &canceled : nullptr);
548 }
549
550 void InspectorNetworkAgent::didLoadResourceFromMemoryCache(DocumentLoader* loader, CachedResource& resource)
551 {
552     ASSERT(loader);
553     if (!loader)
554         return;
555
556     unsigned long identifier = loader->frame()->page()->progress().createUniqueIdentifier();
557     String requestId = IdentifiersFactory::requestId(identifier);
558     String loaderId = loaderIdentifier(loader);
559     String frameId = frameIdentifier(loader);
560
561     m_resourcesData->resourceCreated(requestId, loaderId, resource);
562
563     RefPtr<Inspector::Protocol::Network::Initiator> initiatorObject = buildInitiatorObject(loader->frame() ? loader->frame()->document() : nullptr);
564
565     // FIXME: It would be ideal to generate the Network.Response with the MemoryCache source
566     // instead of whatever ResourceResponse::Source the CachedResources's response has.
567     // The frontend already knows for certain that this was served from the memory cache.
568
569     m_frontendDispatcher->requestServedFromMemoryCache(requestId, frameId, loaderId, loader->url().string(), timestamp(), initiatorObject, buildObjectForCachedResource(&resource));
570 }
571
572 void InspectorNetworkAgent::setInitialScriptContent(unsigned long identifier, const String& sourceString)
573 {
574     m_resourcesData->setResourceContent(IdentifiersFactory::requestId(identifier), sourceString);
575 }
576
577 void InspectorNetworkAgent::didReceiveScriptResponse(unsigned long identifier)
578 {
579     m_resourcesData->setResourceType(IdentifiersFactory::requestId(identifier), InspectorPageAgent::ScriptResource);
580 }
581
582 void InspectorNetworkAgent::didReceiveThreadableLoaderResponse(unsigned long identifier, DocumentThreadableLoader& documentThreadableLoader)
583 {
584     String initiator = documentThreadableLoader.options().initiator;
585     if (initiator == cachedResourceRequestInitiators().fetch)
586         m_resourcesData->setResourceType(IdentifiersFactory::requestId(identifier), InspectorPageAgent::FetchResource);
587     else if (initiator == cachedResourceRequestInitiators().xmlhttprequest)
588         m_resourcesData->setResourceType(IdentifiersFactory::requestId(identifier), InspectorPageAgent::XHRResource);
589 }
590
591 void InspectorNetworkAgent::willLoadXHRSynchronously()
592 {
593     m_loadingXHRSynchronously = true;
594 }
595
596 void InspectorNetworkAgent::didLoadXHRSynchronously()
597 {
598     m_loadingXHRSynchronously = false;
599 }
600
601 void InspectorNetworkAgent::willDestroyCachedResource(CachedResource& cachedResource)
602 {
603     Vector<String> requestIds = m_resourcesData->removeCachedResource(&cachedResource);
604     if (!requestIds.size())
605         return;
606
607     String content;
608     bool base64Encoded;
609     if (!InspectorNetworkAgent::cachedResourceContent(cachedResource, &content, &base64Encoded))
610         return;
611
612     for (auto& id : requestIds)
613         m_resourcesData->setResourceContent(id, content, base64Encoded);
614 }
615
616 void InspectorNetworkAgent::willRecalculateStyle()
617 {
618     m_isRecalculatingStyle = true;
619 }
620
621 void InspectorNetworkAgent::didRecalculateStyle()
622 {
623     m_isRecalculatingStyle = false;
624     m_styleRecalculationInitiator = nullptr;
625 }
626
627 void InspectorNetworkAgent::didScheduleStyleRecalculation(Document& document)
628 {
629     if (!m_styleRecalculationInitiator)
630         m_styleRecalculationInitiator = buildInitiatorObject(&document);
631 }
632
633 RefPtr<Inspector::Protocol::Network::Initiator> InspectorNetworkAgent::buildInitiatorObject(Document* document)
634 {
635     // FIXME: Worker support.
636     if (!isMainThread()) {
637         return Inspector::Protocol::Network::Initiator::create()
638             .setType(Inspector::Protocol::Network::Initiator::Type::Other)
639             .release();
640     }
641
642     Ref<ScriptCallStack> stackTrace = createScriptCallStack(JSMainThreadExecState::currentState());
643     if (stackTrace->size() > 0) {
644         auto initiatorObject = Inspector::Protocol::Network::Initiator::create()
645             .setType(Inspector::Protocol::Network::Initiator::Type::Script)
646             .release();
647         initiatorObject->setStackTrace(stackTrace->buildInspectorArray());
648         return WTFMove(initiatorObject);
649     }
650
651     if (document && document->scriptableDocumentParser()) {
652         auto initiatorObject = Inspector::Protocol::Network::Initiator::create()
653             .setType(Inspector::Protocol::Network::Initiator::Type::Parser)
654             .release();
655         initiatorObject->setUrl(document->url().string());
656         initiatorObject->setLineNumber(document->scriptableDocumentParser()->textPosition().m_line.oneBasedInt());
657         return WTFMove(initiatorObject);
658     }
659
660     if (m_isRecalculatingStyle && m_styleRecalculationInitiator)
661         return m_styleRecalculationInitiator;
662
663     return Inspector::Protocol::Network::Initiator::create()
664         .setType(Inspector::Protocol::Network::Initiator::Type::Other)
665         .release();
666 }
667
668 void InspectorNetworkAgent::didCreateWebSocket(unsigned long identifier, const URL& requestURL)
669 {
670     m_frontendDispatcher->webSocketCreated(IdentifiersFactory::requestId(identifier), requestURL.string());
671 }
672
673 void InspectorNetworkAgent::willSendWebSocketHandshakeRequest(unsigned long identifier, const ResourceRequest& request)
674 {
675     auto requestObject = Inspector::Protocol::Network::WebSocketRequest::create()
676         .setHeaders(buildObjectForHeaders(request.httpHeaderFields()))
677         .release();
678     m_frontendDispatcher->webSocketWillSendHandshakeRequest(IdentifiersFactory::requestId(identifier), timestamp(), WallTime::now().secondsSinceEpoch().seconds(), WTFMove(requestObject));
679 }
680
681 void InspectorNetworkAgent::didReceiveWebSocketHandshakeResponse(unsigned long identifier, const ResourceResponse& response)
682 {
683     auto responseObject = Inspector::Protocol::Network::WebSocketResponse::create()
684         .setStatus(response.httpStatusCode())
685         .setStatusText(response.httpStatusText())
686         .setHeaders(buildObjectForHeaders(response.httpHeaderFields()))
687         .release();
688     m_frontendDispatcher->webSocketHandshakeResponseReceived(IdentifiersFactory::requestId(identifier), timestamp(), WTFMove(responseObject));
689 }
690
691 void InspectorNetworkAgent::didCloseWebSocket(unsigned long identifier)
692 {
693     m_frontendDispatcher->webSocketClosed(IdentifiersFactory::requestId(identifier), timestamp());
694 }
695
696 void InspectorNetworkAgent::didReceiveWebSocketFrame(unsigned long identifier, const WebSocketFrame& frame)
697 {
698     auto frameObject = Inspector::Protocol::Network::WebSocketFrame::create()
699         .setOpcode(frame.opCode)
700         .setMask(frame.masked)
701         .setPayloadData(String::fromUTF8WithLatin1Fallback(frame.payload, frame.payloadLength))
702         .setPayloadLength(frame.payloadLength)
703         .release();
704     m_frontendDispatcher->webSocketFrameReceived(IdentifiersFactory::requestId(identifier), timestamp(), WTFMove(frameObject));
705 }
706
707 void InspectorNetworkAgent::didSendWebSocketFrame(unsigned long identifier, const WebSocketFrame& frame)
708 {
709     auto frameObject = Inspector::Protocol::Network::WebSocketFrame::create()
710         .setOpcode(frame.opCode)
711         .setMask(frame.masked)
712         .setPayloadData(String::fromUTF8WithLatin1Fallback(frame.payload, frame.payloadLength))
713         .setPayloadLength(frame.payloadLength)
714         .release();
715     m_frontendDispatcher->webSocketFrameSent(IdentifiersFactory::requestId(identifier), timestamp(), WTFMove(frameObject));
716 }
717
718 void InspectorNetworkAgent::didReceiveWebSocketFrameError(unsigned long identifier, const String& errorMessage)
719 {
720     m_frontendDispatcher->webSocketFrameError(IdentifiersFactory::requestId(identifier), timestamp(), errorMessage);
721 }
722
723 void InspectorNetworkAgent::enable(ErrorString&)
724 {
725     enable();
726 }
727
728 void InspectorNetworkAgent::enable()
729 {
730     m_enabled = true;
731     m_instrumentingAgents.setInspectorNetworkAgent(this);
732
733     {
734         LockHolder lock(WebSocket::allActiveWebSocketsMutex());
735
736         for (WebSocket* webSocket : activeWebSockets(lock)) {
737             ASSERT(is<WebSocketChannel>(webSocket->channel().get()));
738             WebSocketChannel* channel = downcast<WebSocketChannel>(webSocket->channel().get());
739
740             unsigned identifier = channel->identifier();
741             didCreateWebSocket(identifier, webSocket->url());
742             willSendWebSocketHandshakeRequest(identifier, channel->clientHandshakeRequest());
743
744             if (channel->handshakeMode() == WebSocketHandshake::Connected)
745                 didReceiveWebSocketHandshakeResponse(identifier, channel->serverHandshakeResponse());
746
747             if (webSocket->readyState() == WebSocket::CLOSED)
748                 didCloseWebSocket(identifier);
749         }
750     }
751 }
752
753 void InspectorNetworkAgent::disable(ErrorString&)
754 {
755     m_enabled = false;
756     m_instrumentingAgents.setInspectorNetworkAgent(nullptr);
757     m_resourcesData->clear();
758     m_extraRequestHeaders.clear();
759
760     setResourceCachingDisabled(false);
761 }
762
763 void InspectorNetworkAgent::setExtraHTTPHeaders(ErrorString&, const JSON::Object& headers)
764 {
765     for (auto& entry : headers) {
766         String stringValue;
767         if (entry.value->asString(stringValue))
768             m_extraRequestHeaders.set(entry.key, stringValue);
769     }
770 }
771
772 void InspectorNetworkAgent::getResponseBody(ErrorString& errorString, const String& requestId, String* content, bool* base64Encoded)
773 {
774     NetworkResourcesData::ResourceData const* resourceData = m_resourcesData->data(requestId);
775     if (!resourceData) {
776         errorString = "No resource with given identifier found"_s;
777         return;
778     }
779
780     if (resourceData->hasContent()) {
781         *base64Encoded = resourceData->base64Encoded();
782         *content = resourceData->content();
783         return;
784     }
785
786     if (resourceData->isContentEvicted()) {
787         errorString = "Request content was evicted from inspector cache"_s;
788         return;
789     }
790
791     if (resourceData->buffer() && !resourceData->textEncodingName().isNull()) {
792         *base64Encoded = false;
793         if (InspectorPageAgent::sharedBufferContent(resourceData->buffer(), resourceData->textEncodingName(), *base64Encoded, content))
794             return;
795     }
796
797     if (resourceData->cachedResource()) {
798         if (InspectorNetworkAgent::cachedResourceContent(*resourceData->cachedResource(), content, base64Encoded))
799             return;
800     }
801
802     errorString = "No data found for resource with given identifier"_s;
803 }
804
805 void InspectorNetworkAgent::setResourceCachingDisabled(ErrorString&, bool disabled)
806 {
807     setResourceCachingDisabled(disabled);
808 }
809
810 void InspectorNetworkAgent::loadResource(const String& frameId, const String& urlString, Ref<LoadResourceCallback>&& callback)
811 {
812     ErrorString error;
813     auto* context = scriptExecutionContext(error, frameId);
814     if (!context) {
815         callback->sendFailure(error);
816         return;
817     }
818
819     URL url = context->completeURL(urlString);
820     ResourceRequest request(url);
821     request.setHTTPMethod("GET"_s);
822     request.setHiddenFromInspector(true);
823
824     ThreadableLoaderOptions options;
825     options.sendLoadCallbacks = SendCallbackPolicy::SendCallbacks; // So we remove this from m_hiddenRequestIdentifiers on completion.
826     options.defersLoadingPolicy = DefersLoadingPolicy::DisallowDefersLoading; // So the request is never deferred.
827     options.mode = FetchOptions::Mode::NoCors;
828     options.credentials = FetchOptions::Credentials::SameOrigin;
829     options.contentSecurityPolicyEnforcement = ContentSecurityPolicyEnforcement::DoNotEnforce;
830
831     // InspectorThreadableLoaderClient deletes itself when the load completes or fails.
832     InspectorThreadableLoaderClient* inspectorThreadableLoaderClient = new InspectorThreadableLoaderClient(callback.copyRef());
833     auto loader = ThreadableLoader::create(*context, *inspectorThreadableLoaderClient, WTFMove(request), options);
834     if (!loader) {
835         callback->sendFailure("Could not load requested resource."_s);
836         return;
837     }
838
839     // If the load already completed, inspectorThreadableLoaderClient will have been deleted and we will have already called the callback.
840     if (!callback->isActive())
841         return;
842
843     inspectorThreadableLoaderClient->setLoader(WTFMove(loader));
844 }
845
846 WebSocket* InspectorNetworkAgent::webSocketForRequestId(const String& requestId)
847 {
848     LockHolder lock(WebSocket::allActiveWebSocketsMutex());
849
850     for (WebSocket* webSocket : activeWebSockets(lock)) {
851         ASSERT(is<WebSocketChannel>(webSocket->channel().get()));
852         WebSocketChannel* channel = downcast<WebSocketChannel>(webSocket->channel().get());
853         if (IdentifiersFactory::requestId(channel->identifier()) == requestId)
854             return webSocket;
855     }
856
857     return nullptr;
858 }
859
860 static JSC::JSValue webSocketAsScriptValue(JSC::ExecState& state, WebSocket* webSocket)
861 {
862     JSC::JSLockHolder lock(&state);
863     return toJS(&state, deprecatedGlobalObjectForPrototype(&state), webSocket);
864 }
865
866 void InspectorNetworkAgent::resolveWebSocket(ErrorString& errorString, const String& requestId, const String* objectGroup, RefPtr<Inspector::Protocol::Runtime::RemoteObject>& result)
867 {
868     WebSocket* webSocket = webSocketForRequestId(requestId);
869     if (!webSocket) {
870         errorString = "WebSocket not found"_s;
871         return;
872     }
873
874     // FIXME: <https://webkit.org/b/168475> Web Inspector: Correctly display iframe's and worker's WebSockets
875     if (!is<Document>(webSocket->scriptExecutionContext()))
876         return;
877
878     auto* document = downcast<Document>(webSocket->scriptExecutionContext());
879     auto* frame = document->frame();
880     if (!frame) {
881         errorString = "WebSocket belongs to document without a frame"_s;
882         return;
883     }
884
885     auto& state = *mainWorldExecState(frame);
886     auto injectedScript = m_injectedScriptManager.injectedScriptFor(&state);
887     ASSERT(!injectedScript.hasNoValue());
888
889     String objectGroupName = objectGroup ? *objectGroup : String();
890     result = injectedScript.wrapObject(webSocketAsScriptValue(state, webSocket), objectGroupName);
891 }
892
893 bool InspectorNetworkAgent::shouldTreatAsText(const String& mimeType)
894 {
895     return startsWithLettersIgnoringASCIICase(mimeType, "text/")
896         || MIMETypeRegistry::isSupportedJavaScriptMIMEType(mimeType)
897         || MIMETypeRegistry::isSupportedJSONMIMEType(mimeType)
898         || MIMETypeRegistry::isXMLMIMEType(mimeType);
899 }
900
901 Ref<TextResourceDecoder> InspectorNetworkAgent::createTextDecoder(const String& mimeType, const String& textEncodingName)
902 {
903     if (!textEncodingName.isEmpty())
904         return TextResourceDecoder::create("text/plain"_s, textEncodingName);
905
906     if (MIMETypeRegistry::isTextMIMEType(mimeType))
907         return TextResourceDecoder::create(mimeType, "UTF-8");
908     if (MIMETypeRegistry::isXMLMIMEType(mimeType)) {
909         auto decoder = TextResourceDecoder::create("application/xml"_s);
910         decoder->useLenientXMLDecoding();
911         return decoder;
912     }
913
914     return TextResourceDecoder::create("text/plain"_s, "UTF-8");
915 }
916
917 std::optional<String> InspectorNetworkAgent::textContentForCachedResource(CachedResource& cachedResource)
918 {
919     if (!InspectorNetworkAgent::shouldTreatAsText(cachedResource.mimeType()))
920         return std::nullopt;
921
922     String result;
923     bool base64Encoded;
924     if (InspectorNetworkAgent::cachedResourceContent(cachedResource, &result, &base64Encoded)) {
925         ASSERT(!base64Encoded);
926         return result;
927     }
928
929     return std::nullopt;
930 }
931
932 bool InspectorNetworkAgent::cachedResourceContent(CachedResource& resource, String* result, bool* base64Encoded)
933 {
934     ASSERT(result);
935     ASSERT(base64Encoded);
936
937     if (!resource.encodedSize()) {
938         *base64Encoded = false;
939         *result = String();
940         return true;
941     }
942
943     switch (resource.type()) {
944     case CachedResource::Type::CSSStyleSheet:
945         *base64Encoded = false;
946         *result = downcast<CachedCSSStyleSheet>(resource).sheetText();
947         // The above can return a null String if the MIME type is invalid.
948         return !result->isNull();
949     case CachedResource::Type::Script:
950         *base64Encoded = false;
951         *result = downcast<CachedScript>(resource).script().toString();
952         return true;
953     default:
954         auto* buffer = resource.resourceBuffer();
955         if (!buffer)
956             return false;
957
958         if (InspectorNetworkAgent::shouldTreatAsText(resource.mimeType())) {
959             auto decoder = InspectorNetworkAgent::createTextDecoder(resource.mimeType(), resource.response().textEncodingName());
960             *base64Encoded = false;
961             *result = decoder->decodeAndFlush(buffer->data(), buffer->size());
962             return true;
963         }
964
965         *base64Encoded = true;
966         *result = base64Encode(buffer->data(), buffer->size());
967         return true;
968     }
969 }
970
971 static Ref<Inspector::Protocol::Page::SearchResult> buildObjectForSearchResult(const String& requestId, const String& frameId, const String& url, int matchesCount)
972 {
973     auto searchResult = Inspector::Protocol::Page::SearchResult::create()
974         .setUrl(url)
975         .setFrameId(frameId)
976         .setMatchesCount(matchesCount)
977         .release();
978     searchResult->setRequestId(requestId);
979     return searchResult;
980 }
981
982 static std::optional<String> textContentForResourceData(const NetworkResourcesData::ResourceData& resourceData)
983 {
984     if (resourceData.hasContent() && !resourceData.base64Encoded())
985         return resourceData.content();
986
987     if (resourceData.cachedResource())
988         return InspectorNetworkAgent::textContentForCachedResource(*resourceData.cachedResource());
989
990     return std::nullopt;
991 }
992
993 void InspectorNetworkAgent::searchOtherRequests(const JSC::Yarr::RegularExpression& regex, RefPtr<JSON::ArrayOf<Inspector::Protocol::Page::SearchResult>>& result)
994 {
995     Vector<NetworkResourcesData::ResourceData*> resources = m_resourcesData->resources();
996     for (auto* resourceData : resources) {
997         if (auto textContent = textContentForResourceData(*resourceData)) {
998             int matchesCount = ContentSearchUtilities::countRegularExpressionMatches(regex, resourceData->content());
999             if (matchesCount)
1000                 result->addItem(buildObjectForSearchResult(resourceData->requestId(), resourceData->frameId(), resourceData->url(), matchesCount));
1001         }
1002     }
1003 }
1004
1005 void InspectorNetworkAgent::searchInRequest(ErrorString& errorString, const String& requestId, const String& query, bool caseSensitive, bool isRegex, RefPtr<JSON::ArrayOf<Inspector::Protocol::GenericTypes::SearchMatch>>& results)
1006 {
1007     NetworkResourcesData::ResourceData const* resourceData = m_resourcesData->data(requestId);
1008     if (!resourceData) {
1009         errorString = "No resource with given identifier found"_s;
1010         return;
1011     }
1012
1013     if (!resourceData->hasContent()) {
1014         errorString = "No resource content"_s;
1015         return;
1016     }
1017
1018     results = ContentSearchUtilities::searchInTextByLines(resourceData->content(), query, caseSensitive, isRegex);
1019 }
1020
1021 void InspectorNetworkAgent::mainFrameNavigated(DocumentLoader& loader)
1022 {
1023     m_resourcesData->clear(loaderIdentifier(&loader));
1024 }
1025
1026 } // namespace WebCore