a383897d4ed18a0805acaa75fa4bcbe43158e495
[WebKit-https.git] / Source / WebCore / inspector / InspectorNetworkAgent.cpp
1 /*
2  * Copyright (C) 2011 Google Inc. All rights reserved.
3  * Copyright (C) 2015 Apple Inc. All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are
7  * met:
8  *
9  *     * Redistributions of source code must retain the above copyright
10  * notice, this list of conditions and the following disclaimer.
11  *     * Redistributions in binary form must reproduce the above
12  * copyright notice, this list of conditions and the following disclaimer
13  * in the documentation and/or other materials provided with the
14  * distribution.
15  *     * Neither the name of Google Inc. nor the names of its
16  * contributors may be used to endorse or promote products derived from
17  * this software without specific prior written permission.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30  */
31
32 #include "config.h"
33 #include "InspectorNetworkAgent.h"
34
35 #include "CachedRawResource.h"
36 #include "CachedResource.h"
37 #include "CachedResourceLoader.h"
38 #include "CachedResourceRequestInitiators.h"
39 #include "Document.h"
40 #include "DocumentLoader.h"
41 #include "DocumentThreadableLoader.h"
42 #include "Frame.h"
43 #include "FrameLoader.h"
44 #include "HTTPHeaderMap.h"
45 #include "HTTPHeaderNames.h"
46 #include "InspectorPageAgent.h"
47 #include "InspectorTimelineAgent.h"
48 #include "InstrumentingAgents.h"
49 #include "JSMainThreadExecState.h"
50 #include "JSWebSocket.h"
51 #include "MemoryCache.h"
52 #include "NetworkResourcesData.h"
53 #include "Page.h"
54 #include "ProgressTracker.h"
55 #include "ResourceError.h"
56 #include "ResourceLoader.h"
57 #include "ResourceRequest.h"
58 #include "ResourceResponse.h"
59 #include "ScriptState.h"
60 #include "ScriptableDocumentParser.h"
61 #include "SubresourceLoader.h"
62 #include "ThreadableLoaderClient.h"
63 #include "URL.h"
64 #include "WebSocket.h"
65 #include "WebSocketChannel.h"
66 #include "WebSocketFrame.h"
67 #include <inspector/ContentSearchUtilities.h>
68 #include <inspector/IdentifiersFactory.h>
69 #include <inspector/InjectedScript.h>
70 #include <inspector/InjectedScriptManager.h>
71 #include <inspector/InspectorFrontendRouter.h>
72 #include <inspector/InspectorValues.h>
73 #include <inspector/ScriptCallStack.h>
74 #include <inspector/ScriptCallStackFactory.h>
75 #include <runtime/JSCInlines.h>
76 #include <wtf/Lock.h>
77 #include <wtf/RefPtr.h>
78 #include <wtf/Stopwatch.h>
79 #include <wtf/text/StringBuilder.h>
80
81 using namespace Inspector;
82
83 typedef Inspector::NetworkBackendDispatcherHandler::LoadResourceCallback LoadResourceCallback;
84
85 namespace WebCore {
86
87 namespace {
88
89 class InspectorThreadableLoaderClient final : public ThreadableLoaderClient {
90     WTF_MAKE_NONCOPYABLE(InspectorThreadableLoaderClient);
91 public:
92     InspectorThreadableLoaderClient(RefPtr<LoadResourceCallback>&& callback)
93         : m_callback(WTFMove(callback)) { }
94
95     virtual ~InspectorThreadableLoaderClient() { }
96
97     void didReceiveResponse(unsigned long, const ResourceResponse& response) override
98     {
99         m_mimeType = response.mimeType();
100         m_statusCode = response.httpStatusCode();
101
102         // FIXME: This assumes text only responses. We should support non-text responses as well.
103         TextEncoding textEncoding(response.textEncodingName());
104         bool useDetector = false;
105         if (!textEncoding.isValid()) {
106             textEncoding = UTF8Encoding();
107             useDetector = true;
108         }
109
110         m_decoder = TextResourceDecoder::create(ASCIILiteral("text/plain"), textEncoding, useDetector);
111     }
112
113     void didReceiveData(const char* data, int dataLength) override
114     {
115         if (!dataLength)
116             return;
117
118         if (dataLength == -1)
119             dataLength = strlen(data);
120
121         m_responseText.append(m_decoder->decode(data, dataLength));
122     }
123
124     void didFinishLoading(unsigned long) override
125     {
126         if (m_decoder)
127             m_responseText.append(m_decoder->flush());
128
129         m_callback->sendSuccess(m_responseText.toString(), m_mimeType, m_statusCode);
130         dispose();
131     }
132
133     void didFail(const ResourceError& error) override
134     {
135         m_callback->sendFailure(error.isAccessControl() ? ASCIILiteral("Loading resource for inspector failed access control check") : ASCIILiteral("Loading resource for inspector failed"));
136         dispose();
137     }
138
139     void setLoader(RefPtr<ThreadableLoader>&& loader)
140     {
141         m_loader = WTFMove(loader);
142     }
143
144 private:
145     void dispose()
146     {
147         m_loader = nullptr;
148         delete this;
149     }
150
151     RefPtr<LoadResourceCallback> m_callback;
152     RefPtr<ThreadableLoader> m_loader;
153     RefPtr<TextResourceDecoder> m_decoder;
154     String m_mimeType;
155     StringBuilder m_responseText;
156     int m_statusCode;
157 };
158
159 } // namespace
160
161 InspectorNetworkAgent::InspectorNetworkAgent(WebAgentContext& context, InspectorPageAgent* pageAgent)
162     : InspectorAgentBase(ASCIILiteral("Network"), context)
163     , m_frontendDispatcher(std::make_unique<Inspector::NetworkFrontendDispatcher>(context.frontendRouter))
164     , m_backendDispatcher(Inspector::NetworkBackendDispatcher::create(context.backendDispatcher, this))
165     , m_injectedScriptManager(context.injectedScriptManager)
166     , m_pageAgent(pageAgent)
167     , m_resourcesData(std::make_unique<NetworkResourcesData>())
168 {
169 }
170
171 void InspectorNetworkAgent::didCreateFrontendAndBackend(Inspector::FrontendRouter*, Inspector::BackendDispatcher*)
172 {
173 }
174
175 void InspectorNetworkAgent::willDestroyFrontendAndBackend(Inspector::DisconnectReason)
176 {
177     ErrorString unused;
178     disable(unused);
179 }
180
181 static Ref<InspectorObject> buildObjectForHeaders(const HTTPHeaderMap& headers)
182 {
183     Ref<InspectorObject> headersObject = InspectorObject::create();
184     
185     for (const auto& header : headers)
186         headersObject->setString(header.key, header.value);
187     return headersObject;
188 }
189
190 Ref<Inspector::Protocol::Network::ResourceTiming> InspectorNetworkAgent::buildObjectForTiming(const NetworkLoadMetrics& timing, ResourceLoader& resourceLoader)
191 {
192     MonotonicTime startTime = resourceLoader.loadTiming().startTime();
193     double startTimeInInspector = m_environment.executionStopwatch()->elapsedTimeSince(startTime);
194
195     return Inspector::Protocol::Network::ResourceTiming::create()
196         .setStartTime(startTimeInInspector)
197         .setDomainLookupStart(timing.domainLookupStart.milliseconds())
198         .setDomainLookupEnd(timing.domainLookupEnd.milliseconds())
199         .setConnectStart(timing.connectStart.milliseconds())
200         .setConnectEnd(timing.connectEnd.milliseconds())
201         .setSecureConnectionStart(timing.secureConnectionStart.milliseconds())
202         .setRequestStart(timing.requestStart.milliseconds())
203         .setResponseStart(timing.responseStart.milliseconds())
204         .release();
205 }
206
207 static Inspector::Protocol::Network::Metrics::Priority toProtocol(NetworkLoadPriority priority)
208 {
209     switch (priority) {
210     case NetworkLoadPriority::Low:
211         return Inspector::Protocol::Network::Metrics::Priority::Low;
212     case NetworkLoadPriority::Medium:
213         return Inspector::Protocol::Network::Metrics::Priority::Medium;
214     case NetworkLoadPriority::High:
215         return Inspector::Protocol::Network::Metrics::Priority::High;
216     }
217
218     ASSERT_NOT_REACHED();
219     return Inspector::Protocol::Network::Metrics::Priority::Medium;
220 }
221
222 Ref<Inspector::Protocol::Network::Metrics> InspectorNetworkAgent::buildObjectForMetrics(const NetworkLoadMetrics& networkLoadMetrics)
223 {
224     auto metrics = Inspector::Protocol::Network::Metrics::create().release();
225
226     if (!networkLoadMetrics.protocol.isNull())
227         metrics->setProtocol(networkLoadMetrics.protocol);
228     if (networkLoadMetrics.priority)
229         metrics->setPriority(toProtocol(*networkLoadMetrics.priority));
230     if (networkLoadMetrics.remoteAddress)
231         metrics->setRemoteAddress(*networkLoadMetrics.remoteAddress);
232     if (networkLoadMetrics.connectionIdentifier)
233         metrics->setConnectionIdentifier(*networkLoadMetrics.connectionIdentifier);
234     if (networkLoadMetrics.requestHeaders)
235         metrics->setRequestHeaders(buildObjectForHeaders(*networkLoadMetrics.requestHeaders));
236
237     if (networkLoadMetrics.requestHeaderBytesSent)
238         metrics->setRequestHeaderBytesSent(*networkLoadMetrics.requestHeaderBytesSent);
239     if (networkLoadMetrics.requestBodyBytesSent)
240         metrics->setRequestBodyBytesSent(*networkLoadMetrics.requestBodyBytesSent);
241     if (networkLoadMetrics.responseHeaderBytesReceived)
242         metrics->setResponseHeaderBytesReceived(*networkLoadMetrics.responseHeaderBytesReceived);
243     if (networkLoadMetrics.responseBodyBytesReceived)
244         metrics->setResponseBodyBytesReceived(*networkLoadMetrics.responseBodyBytesReceived);
245     if (networkLoadMetrics.responseBodyDecodedSize)
246         metrics->setResponseBodyDecodedSize(*networkLoadMetrics.responseBodyDecodedSize);
247
248     return metrics;
249 }
250
251 static Ref<Inspector::Protocol::Network::Request> buildObjectForResourceRequest(const ResourceRequest& request)
252 {
253     auto requestObject = Inspector::Protocol::Network::Request::create()
254         .setUrl(request.url().string())
255         .setMethod(request.httpMethod())
256         .setHeaders(buildObjectForHeaders(request.httpHeaderFields()))
257         .release();
258     if (request.httpBody() && !request.httpBody()->isEmpty()) {
259         auto bytes = request.httpBody()->flatten();
260         requestObject->setPostData(String::fromUTF8WithLatin1Fallback(bytes.data(), bytes.size()));
261     }
262     return requestObject;
263 }
264
265 static Inspector::Protocol::Network::Response::Source responseSource(ResourceResponse::Source source)
266 {
267     switch (source) {
268     case ResourceResponse::Source::Unknown:
269         return Inspector::Protocol::Network::Response::Source::Unknown;
270     case ResourceResponse::Source::Network:
271         return Inspector::Protocol::Network::Response::Source::Network;
272     case ResourceResponse::Source::MemoryCache:
273     case ResourceResponse::Source::MemoryCacheAfterValidation:
274         return Inspector::Protocol::Network::Response::Source::MemoryCache;
275     case ResourceResponse::Source::DiskCache:
276     case ResourceResponse::Source::DiskCacheAfterValidation:
277         return Inspector::Protocol::Network::Response::Source::DiskCache;
278     }
279
280     ASSERT_NOT_REACHED();
281     return Inspector::Protocol::Network::Response::Source::Unknown;
282 }
283
284 RefPtr<Inspector::Protocol::Network::Response> InspectorNetworkAgent::buildObjectForResourceResponse(const ResourceResponse& response, ResourceLoader* resourceLoader)
285 {
286     if (response.isNull())
287         return nullptr;
288
289     double status = response.httpStatusCode();
290     Ref<InspectorObject> headers = buildObjectForHeaders(response.httpHeaderFields());
291
292     auto responseObject = Inspector::Protocol::Network::Response::create()
293         .setUrl(response.url().string())
294         .setStatus(status)
295         .setStatusText(response.httpStatusText())
296         .setHeaders(WTFMove(headers))
297         .setMimeType(response.mimeType())
298         .setSource(responseSource(response.source()))
299         .release();
300
301     if (resourceLoader)
302         responseObject->setTiming(buildObjectForTiming(response.deprecatedNetworkLoadMetrics(), *resourceLoader));
303
304     return WTFMove(responseObject);
305 }
306
307 Ref<Inspector::Protocol::Network::CachedResource> InspectorNetworkAgent::buildObjectForCachedResource(CachedResource* cachedResource)
308 {
309     auto resourceObject = Inspector::Protocol::Network::CachedResource::create()
310         .setUrl(cachedResource->url())
311         .setType(InspectorPageAgent::cachedResourceTypeJson(*cachedResource))
312         .setBodySize(cachedResource->encodedSize())
313         .release();
314
315     auto resourceResponse = buildObjectForResourceResponse(cachedResource->response(), cachedResource->loader());
316     resourceObject->setResponse(WTFMove(resourceResponse));
317
318     String sourceMappingURL = InspectorPageAgent::sourceMapURLForResource(cachedResource);
319     if (!sourceMappingURL.isEmpty())
320         resourceObject->setSourceMapURL(sourceMappingURL);
321
322     return resourceObject;
323 }
324
325 InspectorNetworkAgent::~InspectorNetworkAgent()
326 {
327     if (m_enabled) {
328         ErrorString unused;
329         disable(unused);
330     }
331     ASSERT(!m_instrumentingAgents.inspectorNetworkAgent());
332 }
333
334 double InspectorNetworkAgent::timestamp()
335 {
336     return m_environment.executionStopwatch()->elapsedTime();
337 }
338
339 void InspectorNetworkAgent::willSendRequest(unsigned long identifier, DocumentLoader& loader, ResourceRequest& request, const ResourceResponse& redirectResponse)
340 {
341     if (request.hiddenFromInspector()) {
342         m_hiddenRequestIdentifiers.add(identifier);
343         return;
344     }
345
346     String requestId = IdentifiersFactory::requestId(identifier);
347     m_resourcesData->resourceCreated(requestId, m_pageAgent->loaderId(&loader));
348
349     CachedResource* cachedResource = InspectorPageAgent::cachedResource(loader.frame(), request.url());
350     InspectorPageAgent::ResourceType type = cachedResource ? InspectorPageAgent::cachedResourceType(*cachedResource) : m_resourcesData->resourceType(requestId);
351     if (type == InspectorPageAgent::OtherResource) {
352         if (m_loadingXHRSynchronously)
353             type = InspectorPageAgent::XHRResource;
354         else if (equalIgnoringFragmentIdentifier(request.url(), loader.url()) && !loader.isCommitted())
355             type = InspectorPageAgent::DocumentResource;
356         else {
357             for (auto& linkIcon : loader.linkIcons()) {
358                 if (equalIgnoringFragmentIdentifier(request.url(), linkIcon.url)) {
359                     type = InspectorPageAgent::ImageResource;
360                     break;
361                 }
362             }
363         }
364     }
365
366     m_resourcesData->setResourceType(requestId, type);
367
368     for (auto& entry : m_extraRequestHeaders)
369         request.setHTTPHeaderField(entry.key, entry.value);
370
371     Inspector::Protocol::Page::ResourceType resourceType = InspectorPageAgent::resourceTypeJson(type);
372
373     RefPtr<Inspector::Protocol::Network::Initiator> initiatorObject = buildInitiatorObject(loader.frame() ? loader.frame()->document() : nullptr);
374     String targetId = request.initiatorIdentifier();
375
376     m_frontendDispatcher->requestWillBeSent(requestId, m_pageAgent->frameId(loader.frame()), m_pageAgent->loaderId(&loader), loader.url().string(), buildObjectForResourceRequest(request), timestamp(), initiatorObject, buildObjectForResourceResponse(redirectResponse, nullptr), type != InspectorPageAgent::OtherResource ? &resourceType : nullptr, targetId.isEmpty() ? nullptr : &targetId);
377 }
378
379 void InspectorNetworkAgent::didReceiveResponse(unsigned long identifier, DocumentLoader& loader, const ResourceResponse& response, ResourceLoader* resourceLoader)
380 {
381     if (m_hiddenRequestIdentifiers.contains(identifier))
382         return;
383
384     String requestId = IdentifiersFactory::requestId(identifier);
385     RefPtr<Inspector::Protocol::Network::Response> resourceResponse = buildObjectForResourceResponse(response, resourceLoader);
386
387     bool isNotModified = response.httpStatusCode() == 304;
388
389     CachedResource* cachedResource = nullptr;
390     if (resourceLoader && resourceLoader->isSubresourceLoader() && !isNotModified)
391         cachedResource = static_cast<SubresourceLoader*>(resourceLoader)->cachedResource();
392     if (!cachedResource)
393         cachedResource = InspectorPageAgent::cachedResource(loader.frame(), response.url());
394
395     if (cachedResource) {
396         // Use mime type from cached resource in case the one in response is empty.
397         if (resourceResponse && response.mimeType().isEmpty())
398             resourceResponse->setString(Inspector::Protocol::Network::Response::MimeType, cachedResource->response().mimeType());
399         m_resourcesData->addCachedResource(requestId, cachedResource);
400     }
401
402     InspectorPageAgent::ResourceType type = m_resourcesData->resourceType(requestId);
403     InspectorPageAgent::ResourceType newType = cachedResource ? InspectorPageAgent::cachedResourceType(*cachedResource) : type;
404
405     // FIXME: XHRResource is returned for CachedResource::RawResource, it should be OtherResource unless it truly is an XHR.
406     // RawResource is used for loading worker scripts, and those should stay as ScriptResource and not change to XHRResource.
407     if (type != newType && newType != InspectorPageAgent::XHRResource && newType != InspectorPageAgent::OtherResource)
408         type = newType;
409
410     m_resourcesData->responseReceived(requestId, m_pageAgent->frameId(loader.frame()), response);
411     m_resourcesData->setResourceType(requestId, type);
412
413     m_frontendDispatcher->responseReceived(requestId, m_pageAgent->frameId(loader.frame()), m_pageAgent->loaderId(&loader), timestamp(), InspectorPageAgent::resourceTypeJson(type), resourceResponse);
414
415     // If we revalidated the resource and got Not modified, send content length following didReceiveResponse
416     // as there will be no calls to didReceiveData from the network stack.
417     if (isNotModified && cachedResource && cachedResource->encodedSize())
418         didReceiveData(identifier, nullptr, cachedResource->encodedSize(), 0);
419 }
420
421 static bool isErrorStatusCode(int statusCode)
422 {
423     return statusCode >= 400;
424 }
425
426 void InspectorNetworkAgent::didReceiveData(unsigned long identifier, const char* data, int dataLength, int encodedDataLength)
427 {
428     if (m_hiddenRequestIdentifiers.contains(identifier))
429         return;
430
431     String requestId = IdentifiersFactory::requestId(identifier);
432
433     if (data) {
434         NetworkResourcesData::ResourceData const* resourceData = m_resourcesData->data(requestId);
435         if (resourceData && !m_loadingXHRSynchronously && (!resourceData->cachedResource() || resourceData->cachedResource()->dataBufferingPolicy() == DoNotBufferData || isErrorStatusCode(resourceData->httpStatusCode())))
436             m_resourcesData->maybeAddResourceData(requestId, data, dataLength);
437     }
438
439     m_frontendDispatcher->dataReceived(requestId, timestamp(), dataLength, encodedDataLength);
440 }
441
442 void InspectorNetworkAgent::didFinishLoading(unsigned long identifier, DocumentLoader& loader, const NetworkLoadMetrics& networkLoadMetrics, ResourceLoader* resourceLoader)
443 {
444     if (m_hiddenRequestIdentifiers.remove(identifier))
445         return;
446
447     double elapsedFinishTime;
448     if (resourceLoader && networkLoadMetrics.isComplete()) {
449         MonotonicTime startTime = resourceLoader->loadTiming().startTime();
450         double startTimeInInspector = m_environment.executionStopwatch()->elapsedTimeSince(startTime);
451         elapsedFinishTime = startTimeInInspector + networkLoadMetrics.responseEnd.seconds();
452     } else
453         elapsedFinishTime = timestamp();
454
455     String requestId = IdentifiersFactory::requestId(identifier);
456     if (m_resourcesData->resourceType(requestId) == InspectorPageAgent::DocumentResource)
457         m_resourcesData->addResourceSharedBuffer(requestId, loader.frameLoader()->documentLoader()->mainResourceData(), loader.frame()->document()->encoding());
458
459     m_resourcesData->maybeDecodeDataToContent(requestId);
460
461     String sourceMappingURL;
462     NetworkResourcesData::ResourceData const* resourceData = m_resourcesData->data(requestId);
463     if (resourceData && resourceData->cachedResource())
464         sourceMappingURL = InspectorPageAgent::sourceMapURLForResource(resourceData->cachedResource());
465
466     RefPtr<Inspector::Protocol::Network::Metrics> metrics = buildObjectForMetrics(networkLoadMetrics);
467
468     m_frontendDispatcher->loadingFinished(requestId, elapsedFinishTime, !sourceMappingURL.isEmpty() ? &sourceMappingURL : nullptr, metrics);
469 }
470
471 void InspectorNetworkAgent::didFailLoading(unsigned long identifier, DocumentLoader& loader, const ResourceError& error)
472 {
473     if (m_hiddenRequestIdentifiers.remove(identifier))
474         return;
475
476     String requestId = IdentifiersFactory::requestId(identifier);
477
478     if (m_resourcesData->resourceType(requestId) == InspectorPageAgent::DocumentResource) {
479         Frame* frame = loader.frame();
480         if (frame && frame->loader().documentLoader() && frame->document()) {
481             m_resourcesData->addResourceSharedBuffer(requestId,
482                 frame->loader().documentLoader()->mainResourceData(),
483                 frame->document()->encoding());
484         }
485     }
486
487     bool canceled = error.isCancellation();
488     m_frontendDispatcher->loadingFailed(requestId, timestamp(), error.localizedDescription(), canceled ? &canceled : nullptr);
489 }
490
491 void InspectorNetworkAgent::didLoadResourceFromMemoryCache(DocumentLoader& loader, CachedResource& resource)
492 {
493     String loaderId = m_pageAgent->loaderId(&loader);
494     String frameId = m_pageAgent->frameId(loader.frame());
495     unsigned long identifier = loader.frame()->page()->progress().createUniqueIdentifier();
496     String requestId = IdentifiersFactory::requestId(identifier);
497
498     m_resourcesData->resourceCreated(requestId, loaderId);
499     m_resourcesData->addCachedResource(requestId, &resource);
500
501     RefPtr<Inspector::Protocol::Network::Initiator> initiatorObject = buildInitiatorObject(loader.frame() ? loader.frame()->document() : nullptr);
502
503     // FIXME: It would be ideal to generate the Network.Response with the MemoryCache source
504     // instead of whatever ResourceResponse::Source the CachedResources's response has.
505     // The frontend already knows for certain that this was served from the memory cache.
506
507     m_frontendDispatcher->requestServedFromMemoryCache(requestId, frameId, loaderId, loader.url().string(), timestamp(), initiatorObject, buildObjectForCachedResource(&resource));
508 }
509
510 void InspectorNetworkAgent::setInitialScriptContent(unsigned long identifier, const String& sourceString)
511 {
512     m_resourcesData->setResourceContent(IdentifiersFactory::requestId(identifier), sourceString);
513 }
514
515 void InspectorNetworkAgent::didReceiveScriptResponse(unsigned long identifier)
516 {
517     m_resourcesData->setResourceType(IdentifiersFactory::requestId(identifier), InspectorPageAgent::ScriptResource);
518 }
519
520 void InspectorNetworkAgent::didReceiveThreadableLoaderResponse(unsigned long identifier, DocumentThreadableLoader& documentThreadableLoader)
521 {
522     String initiator = documentThreadableLoader.options().initiator;
523     if (initiator == cachedResourceRequestInitiators().fetch)
524         m_resourcesData->setResourceType(IdentifiersFactory::requestId(identifier), InspectorPageAgent::FetchResource);
525     else if (initiator == cachedResourceRequestInitiators().xmlhttprequest)
526         m_resourcesData->setResourceType(IdentifiersFactory::requestId(identifier), InspectorPageAgent::XHRResource);
527 }
528
529 void InspectorNetworkAgent::didFinishXHRLoading(unsigned long identifier, const String& decodedText)
530 {
531     m_resourcesData->setResourceContent(IdentifiersFactory::requestId(identifier), decodedText);
532 }
533
534 void InspectorNetworkAgent::willLoadXHRSynchronously()
535 {
536     m_loadingXHRSynchronously = true;
537 }
538
539 void InspectorNetworkAgent::didLoadXHRSynchronously()
540 {
541     m_loadingXHRSynchronously = false;
542 }
543
544 void InspectorNetworkAgent::willDestroyCachedResource(CachedResource& cachedResource)
545 {
546     Vector<String> requestIds = m_resourcesData->removeCachedResource(&cachedResource);
547     if (!requestIds.size())
548         return;
549
550     String content;
551     bool base64Encoded;
552     if (!InspectorPageAgent::cachedResourceContent(&cachedResource, &content, &base64Encoded))
553         return;
554     for (auto& id : requestIds)
555         m_resourcesData->setResourceContent(id, content, base64Encoded);
556 }
557
558 void InspectorNetworkAgent::willRecalculateStyle()
559 {
560     m_isRecalculatingStyle = true;
561 }
562
563 void InspectorNetworkAgent::didRecalculateStyle()
564 {
565     m_isRecalculatingStyle = false;
566     m_styleRecalculationInitiator = nullptr;
567 }
568
569 void InspectorNetworkAgent::didScheduleStyleRecalculation(Document& document)
570 {
571     if (!m_styleRecalculationInitiator)
572         m_styleRecalculationInitiator = buildInitiatorObject(&document);
573 }
574
575 RefPtr<Inspector::Protocol::Network::Initiator> InspectorNetworkAgent::buildInitiatorObject(Document* document)
576 {
577     Ref<ScriptCallStack> stackTrace = createScriptCallStack(JSMainThreadExecState::currentState(), ScriptCallStack::maxCallStackSizeToCapture);
578     if (stackTrace->size() > 0) {
579         auto initiatorObject = Inspector::Protocol::Network::Initiator::create()
580             .setType(Inspector::Protocol::Network::Initiator::Type::Script)
581             .release();
582         initiatorObject->setStackTrace(stackTrace->buildInspectorArray());
583         return WTFMove(initiatorObject);
584     }
585
586     if (document && document->scriptableDocumentParser()) {
587         auto initiatorObject = Inspector::Protocol::Network::Initiator::create()
588             .setType(Inspector::Protocol::Network::Initiator::Type::Parser)
589             .release();
590         initiatorObject->setUrl(document->url().string());
591         initiatorObject->setLineNumber(document->scriptableDocumentParser()->textPosition().m_line.oneBasedInt());
592         return WTFMove(initiatorObject);
593     }
594
595     if (m_isRecalculatingStyle && m_styleRecalculationInitiator)
596         return m_styleRecalculationInitiator;
597
598     return Inspector::Protocol::Network::Initiator::create()
599         .setType(Inspector::Protocol::Network::Initiator::Type::Other)
600         .release();
601 }
602
603 void InspectorNetworkAgent::didCreateWebSocket(unsigned long identifier, const URL& requestURL)
604 {
605     m_frontendDispatcher->webSocketCreated(IdentifiersFactory::requestId(identifier), requestURL.string());
606 }
607
608 void InspectorNetworkAgent::willSendWebSocketHandshakeRequest(unsigned long identifier, const ResourceRequest& request)
609 {
610     auto requestObject = Inspector::Protocol::Network::WebSocketRequest::create()
611         .setHeaders(buildObjectForHeaders(request.httpHeaderFields()))
612         .release();
613     m_frontendDispatcher->webSocketWillSendHandshakeRequest(IdentifiersFactory::requestId(identifier), timestamp(), currentTime(), WTFMove(requestObject));
614 }
615
616 void InspectorNetworkAgent::didReceiveWebSocketHandshakeResponse(unsigned long identifier, const ResourceResponse& response)
617 {
618     auto responseObject = Inspector::Protocol::Network::WebSocketResponse::create()
619         .setStatus(response.httpStatusCode())
620         .setStatusText(response.httpStatusText())
621         .setHeaders(buildObjectForHeaders(response.httpHeaderFields()))
622         .release();
623     m_frontendDispatcher->webSocketHandshakeResponseReceived(IdentifiersFactory::requestId(identifier), timestamp(), WTFMove(responseObject));
624 }
625
626 void InspectorNetworkAgent::didCloseWebSocket(unsigned long identifier)
627 {
628     m_frontendDispatcher->webSocketClosed(IdentifiersFactory::requestId(identifier), timestamp());
629 }
630
631 void InspectorNetworkAgent::didReceiveWebSocketFrame(unsigned long identifier, const WebSocketFrame& frame)
632 {
633     auto frameObject = Inspector::Protocol::Network::WebSocketFrame::create()
634         .setOpcode(frame.opCode)
635         .setMask(frame.masked)
636         .setPayloadData(String::fromUTF8WithLatin1Fallback(frame.payload, frame.payloadLength))
637         .setPayloadLength(frame.payloadLength)
638         .release();
639     m_frontendDispatcher->webSocketFrameReceived(IdentifiersFactory::requestId(identifier), timestamp(), WTFMove(frameObject));
640 }
641
642 void InspectorNetworkAgent::didSendWebSocketFrame(unsigned long identifier, const WebSocketFrame& frame)
643 {
644     auto frameObject = Inspector::Protocol::Network::WebSocketFrame::create()
645         .setOpcode(frame.opCode)
646         .setMask(frame.masked)
647         .setPayloadData(String::fromUTF8WithLatin1Fallback(frame.payload, frame.payloadLength))
648         .setPayloadLength(frame.payloadLength)
649         .release();
650     m_frontendDispatcher->webSocketFrameSent(IdentifiersFactory::requestId(identifier), timestamp(), WTFMove(frameObject));
651 }
652
653 void InspectorNetworkAgent::didReceiveWebSocketFrameError(unsigned long identifier, const String& errorMessage)
654 {
655     m_frontendDispatcher->webSocketFrameError(IdentifiersFactory::requestId(identifier), timestamp(), errorMessage);
656 }
657
658 void InspectorNetworkAgent::enable(ErrorString&)
659 {
660     enable();
661 }
662
663 void InspectorNetworkAgent::enable()
664 {
665     m_enabled = true;
666     m_instrumentingAgents.setInspectorNetworkAgent(this);
667
668     LockHolder lock(WebSocket::allActiveWebSocketsMutex());
669
670     for (WebSocket* webSocket : WebSocket::allActiveWebSockets(lock)) {
671         if (!is<Document>(webSocket->scriptExecutionContext()) || !is<WebSocketChannel>(webSocket->channel().get()))
672             continue;
673
674         Document* document = downcast<Document>(webSocket->scriptExecutionContext());
675         if (document->page() != &m_pageAgent->page())
676             continue;
677
678         WebSocketChannel* channel = downcast<WebSocketChannel>(webSocket->channel().get());
679         if (!channel)
680             continue;
681
682         unsigned identifier = channel->identifier();
683         didCreateWebSocket(identifier, webSocket->url());
684         willSendWebSocketHandshakeRequest(identifier, channel->clientHandshakeRequest());
685
686         if (channel->handshakeMode() == WebSocketHandshake::Connected)
687             didReceiveWebSocketHandshakeResponse(identifier, channel->serverHandshakeResponse());
688
689         if (webSocket->readyState() == WebSocket::CLOSED)
690             didCloseWebSocket(identifier);
691     }
692 }
693
694 void InspectorNetworkAgent::disable(ErrorString&)
695 {
696     m_enabled = false;
697     m_instrumentingAgents.setInspectorNetworkAgent(nullptr);
698     m_resourcesData->clear();
699     m_extraRequestHeaders.clear();
700
701     m_pageAgent->page().setResourceCachingDisabledOverride(false);
702 }
703
704 void InspectorNetworkAgent::setExtraHTTPHeaders(ErrorString&, const InspectorObject& headers)
705 {
706     for (auto& entry : headers) {
707         String stringValue;
708         if (entry.value->asString(stringValue))
709             m_extraRequestHeaders.set(entry.key, stringValue);
710     }
711 }
712
713 void InspectorNetworkAgent::getResponseBody(ErrorString& errorString, const String& requestId, String* content, bool* base64Encoded)
714 {
715     NetworkResourcesData::ResourceData const* resourceData = m_resourcesData->data(requestId);
716     if (!resourceData) {
717         errorString = ASCIILiteral("No resource with given identifier found");
718         return;
719     }
720
721     if (resourceData->hasContent()) {
722         *base64Encoded = resourceData->base64Encoded();
723         *content = resourceData->content();
724         return;
725     }
726
727     if (resourceData->isContentEvicted()) {
728         errorString = ASCIILiteral("Request content was evicted from inspector cache");
729         return;
730     }
731
732     if (resourceData->buffer() && !resourceData->textEncodingName().isNull()) {
733         *base64Encoded = false;
734         if (InspectorPageAgent::sharedBufferContent(resourceData->buffer(), resourceData->textEncodingName(), *base64Encoded, content))
735             return;
736     }
737
738     if (resourceData->cachedResource()) {
739         if (InspectorPageAgent::cachedResourceContent(resourceData->cachedResource(), content, base64Encoded))
740             return;
741     }
742
743     errorString = ASCIILiteral("No data found for resource with given identifier");
744 }
745
746 void InspectorNetworkAgent::setResourceCachingDisabled(ErrorString&, bool disabled)
747 {
748     m_pageAgent->page().setResourceCachingDisabledOverride(disabled);
749 }
750
751 void InspectorNetworkAgent::loadResource(ErrorString& errorString, const String& frameId, const String& urlString, Ref<LoadResourceCallback>&& callback)
752 {
753     Frame* frame = m_pageAgent->assertFrame(errorString, frameId);
754     if (!frame)
755         return;
756
757     Document* document = frame->document();
758     if (!document) {
759         errorString = ASCIILiteral("No Document instance for the specified frame");
760         return;
761     }
762
763     URL url = document->completeURL(urlString);
764     ResourceRequest request(url);
765     request.setHTTPMethod(ASCIILiteral("GET"));
766     request.setHiddenFromInspector(true);
767
768     ThreadableLoaderOptions options;
769     options.sendLoadCallbacks = SendCallbacks; // So we remove this from m_hiddenRequestIdentifiers on completion.
770     options.defersLoadingPolicy = DefersLoadingPolicy::DisallowDefersLoading; // So the request is never deferred.
771     options.mode = FetchOptions::Mode::NoCors;
772     options.credentials = FetchOptions::Credentials::SameOrigin;
773     options.contentSecurityPolicyEnforcement = ContentSecurityPolicyEnforcement::DoNotEnforce;
774
775     // InspectorThreadableLoaderClient deletes itself when the load completes or fails.
776     InspectorThreadableLoaderClient* inspectorThreadableLoaderClient = new InspectorThreadableLoaderClient(callback.copyRef());
777     auto loader = DocumentThreadableLoader::create(*document, *inspectorThreadableLoaderClient, WTFMove(request), options);
778     if (!loader)
779         return;
780
781     // If the load already completed, inspectorThreadableLoaderClient will have been deleted and we will have already called the callback.
782     if (!callback->isActive())
783         return;
784
785     inspectorThreadableLoaderClient->setLoader(WTFMove(loader));
786 }
787
788 WebSocket* InspectorNetworkAgent::webSocketForRequestId(const String& requestId)
789 {
790     LockHolder lock(WebSocket::allActiveWebSocketsMutex());
791
792     for (WebSocket* webSocket : WebSocket::allActiveWebSockets(lock)) {
793         if (!is<WebSocketChannel>(webSocket->channel().get()))
794             continue;
795
796         WebSocketChannel* channel = downcast<WebSocketChannel>(webSocket->channel().get());
797         if (!channel)
798             continue;
799
800         if (IdentifiersFactory::requestId(channel->identifier()) != requestId)
801             continue;
802
803         // FIXME: <webkit.org/b/168475> Web Inspector: Correctly display iframe's and worker's WebSockets
804         if (!is<Document>(webSocket->scriptExecutionContext()))
805             continue;
806
807         Document* document = downcast<Document>(webSocket->scriptExecutionContext());
808         if (document->page() != &m_pageAgent->page())
809             continue;
810
811         return webSocket;
812     }
813
814     return nullptr;
815 }
816
817 static JSC::JSValue webSocketAsScriptValue(JSC::ExecState& state, WebSocket* webSocket)
818 {
819     JSC::JSLockHolder lock(&state);
820     return toJS(&state, deprecatedGlobalObjectForPrototype(&state), webSocket);
821 }
822
823 void InspectorNetworkAgent::resolveWebSocket(ErrorString& errorString, const String& requestId, const String* const objectGroup, RefPtr<Inspector::Protocol::Runtime::RemoteObject>& result)
824 {
825     WebSocket* webSocket = webSocketForRequestId(requestId);
826     if (!webSocket) {
827         errorString = ASCIILiteral("WebSocket not found");
828         return;
829     }
830
831     // FIXME: <webkit.org/b/168475> Web Inspector: Correctly display iframe's and worker's WebSockets
832     Document* document = downcast<Document>(webSocket->scriptExecutionContext());
833     Frame* frame = document->frame();
834     if (!frame) {
835         errorString = ASCIILiteral("WebSocket belongs to document without a frame");
836         return;
837     }
838
839     auto& state = *mainWorldExecState(frame);
840     auto injectedScript = m_injectedScriptManager.injectedScriptFor(&state);
841     ASSERT(!injectedScript.hasNoValue());
842
843     String objectGroupName = objectGroup ? *objectGroup : String();
844     result = injectedScript.wrapObject(webSocketAsScriptValue(state, webSocket), objectGroupName);
845 }
846
847 static Ref<Inspector::Protocol::Page::SearchResult> buildObjectForSearchResult(const String& requestId, const String& frameId, const String& url, int matchesCount)
848 {
849     auto searchResult = Inspector::Protocol::Page::SearchResult::create()
850         .setUrl(url)
851         .setFrameId(frameId)
852         .setMatchesCount(matchesCount)
853         .release();
854     searchResult->setRequestId(requestId);
855     return searchResult;
856 }
857
858 void InspectorNetworkAgent::searchOtherRequests(const JSC::Yarr::RegularExpression& regex, RefPtr<Inspector::Protocol::Array<Inspector::Protocol::Page::SearchResult>>& result)
859 {
860     Vector<NetworkResourcesData::ResourceData*> resources = m_resourcesData->resources();
861     for (auto* resourceData : resources) {
862         if (resourceData->hasContent()) {
863             int matchesCount = ContentSearchUtilities::countRegularExpressionMatches(regex, resourceData->content());
864             if (matchesCount)
865                 result->addItem(buildObjectForSearchResult(resourceData->requestId(), resourceData->frameId(), resourceData->url(), matchesCount));
866         }
867     }
868 }
869
870 void InspectorNetworkAgent::searchInRequest(ErrorString& errorString, const String& requestId, const String& query, bool caseSensitive, bool isRegex, RefPtr<Inspector::Protocol::Array<Inspector::Protocol::GenericTypes::SearchMatch>>& results)
871 {
872     NetworkResourcesData::ResourceData const* resourceData = m_resourcesData->data(requestId);
873     if (!resourceData) {
874         errorString = ASCIILiteral("No resource with given identifier found");
875         return;
876     }
877
878     if (!resourceData->hasContent()) {
879         errorString = ASCIILiteral("No resource content");
880         return;
881     }
882
883     results = ContentSearchUtilities::searchInTextByLines(resourceData->content(), query, caseSensitive, isRegex);
884 }
885
886 void InspectorNetworkAgent::mainFrameNavigated(DocumentLoader& loader)
887 {
888     m_resourcesData->clear(m_pageAgent->loaderId(&loader));
889 }
890
891 } // namespace WebCore