Source/WebCore/inspector/InspectorNetworkAgent.cpp

   1 /*
   2  * Copyright (C) 2011 Google Inc. All rights reserved.
   3  * Copyright (C) 2015 Apple Inc. All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are
   7  * met:
   8  *
   9  *     * Redistributions of source code must retain the above copyright
  10  * notice, this list of conditions and the following disclaimer.
  11  *     * Redistributions in binary form must reproduce the above
  12  * copyright notice, this list of conditions and the following disclaimer
  13  * in the documentation and/or other materials provided with the
  14  * distribution.
  15  *     * Neither the name of Google Inc. nor the names of its
  16  * contributors may be used to endorse or promote products derived from
  17  * this software without specific prior written permission.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30  */
  31
  32 #include "config.h"
  33 #include "InspectorNetworkAgent.h"
  34
  35 #include "CachedRawResource.h"
  36 #include "CachedResource.h"
  37 #include "CachedResourceLoader.h"
  38 #include "CachedResourceRequestInitiators.h"
  39 #include "Document.h"
  40 #include "DocumentLoader.h"
  41 #include "DocumentThreadableLoader.h"
  42 #include "Frame.h"
  43 #include "FrameLoader.h"
  44 #include "HTTPHeaderMap.h"
  45 #include "HTTPHeaderNames.h"
  46 #include "InspectorPageAgent.h"
  47 #include "InspectorTimelineAgent.h"
  48 #include "InstrumentingAgents.h"
  49 #include "JSMainThreadExecState.h"
  50 #include "JSWebSocket.h"
  51 #include "MemoryCache.h"
  52 #include "NetworkResourcesData.h"
  53 #include "Page.h"
  54 #include "ProgressTracker.h"
  55 #include "ResourceError.h"
  56 #include "ResourceLoader.h"
  57 #include "ResourceRequest.h"
  58 #include "ResourceResponse.h"
  59 #include "ScriptState.h"
  60 #include "ScriptableDocumentParser.h"
  61 #include "SubresourceLoader.h"
  62 #include "ThreadableLoaderClient.h"
  63 #include "URL.h"
  64 #include "WebSocket.h"
  65 #include "WebSocketChannel.h"
  66 #include "WebSocketFrame.h"
  67 #include <inspector/ContentSearchUtilities.h>
  68 #include <inspector/IdentifiersFactory.h>
  69 #include <inspector/InjectedScript.h>
  70 #include <inspector/InjectedScriptManager.h>
  71 #include <inspector/InspectorFrontendRouter.h>
  72 #include <inspector/InspectorValues.h>
  73 #include <inspector/ScriptCallStack.h>
  74 #include <inspector/ScriptCallStackFactory.h>
  75 #include <runtime/JSCInlines.h>
  76 #include <wtf/Lock.h>
  77 #include <wtf/RefPtr.h>
  78 #include <wtf/Stopwatch.h>
  79 #include <wtf/text/StringBuilder.h>
  80
  81 using namespace Inspector;
  82
  83 typedef Inspector::NetworkBackendDispatcherHandler::LoadResourceCallback LoadResourceCallback;
  84
  85 namespace WebCore {
  86
  87 namespace {
  88
  89 class InspectorThreadableLoaderClient final : public ThreadableLoaderClient {
  90     WTF_MAKE_NONCOPYABLE(InspectorThreadableLoaderClient);
  91 public:
  92     InspectorThreadableLoaderClient(RefPtr<LoadResourceCallback>&& callback)
  93         : m_callback(WTFMove(callback)) { }
  94
  95     virtual ~InspectorThreadableLoaderClient() { }
  96
  97     void didReceiveResponse(unsigned long, const ResourceResponse& response) override
  98     {
  99         m_mimeType = response.mimeType();
 100         m_statusCode = response.httpStatusCode();
 101
 102         // FIXME: This assumes text only responses. We should support non-text responses as well.
 103         TextEncoding textEncoding(response.textEncodingName());
 104         bool useDetector = false;
 105         if (!textEncoding.isValid()) {
 106             textEncoding = UTF8Encoding();
 107             useDetector = true;
 108         }
 109
 110         m_decoder = TextResourceDecoder::create(ASCIILiteral("text/plain"), textEncoding, useDetector);
 111     }
 112
 113     void didReceiveData(const char* data, int dataLength) override
 114     {
 115         if (!dataLength)
 116             return;
 117
 118         if (dataLength == -1)
 119             dataLength = strlen(data);
 120
 121         m_responseText.append(m_decoder->decode(data, dataLength));
 122     }
 123
 124     void didFinishLoading(unsigned long) override
 125     {
 126         if (m_decoder)
 127             m_responseText.append(m_decoder->flush());
 128
 129         m_callback->sendSuccess(m_responseText.toString(), m_mimeType, m_statusCode);
 130         dispose();
 131     }
 132
 133     void didFail(const ResourceError& error) override
 134     {
 135         m_callback->sendFailure(error.isAccessControl() ? ASCIILiteral("Loading resource for inspector failed access control check") : ASCIILiteral("Loading resource for inspector failed"));
 136         dispose();
 137     }
 138
 139     void setLoader(RefPtr<ThreadableLoader>&& loader)
 140     {
 141         m_loader = WTFMove(loader);
 142     }
 143
 144 private:
 145     void dispose()
 146     {
 147         m_loader = nullptr;
 148         delete this;
 149     }
 150
 151     RefPtr<LoadResourceCallback> m_callback;
 152     RefPtr<ThreadableLoader> m_loader;
 153     RefPtr<TextResourceDecoder> m_decoder;
 154     String m_mimeType;
 155     StringBuilder m_responseText;
 156     int m_statusCode;
 157 };
 158
 159 } // namespace
 160
 161 InspectorNetworkAgent::InspectorNetworkAgent(WebAgentContext& context, InspectorPageAgent* pageAgent)
 162     : InspectorAgentBase(ASCIILiteral("Network"), context)
 163     , m_frontendDispatcher(std::make_unique<Inspector::NetworkFrontendDispatcher>(context.frontendRouter))
 164     , m_backendDispatcher(Inspector::NetworkBackendDispatcher::create(context.backendDispatcher, this))
 165     , m_injectedScriptManager(context.injectedScriptManager)
 166     , m_pageAgent(pageAgent)
 167     , m_resourcesData(std::make_unique<NetworkResourcesData>())
 168 {
 169 }
 170
 171 void InspectorNetworkAgent::didCreateFrontendAndBackend(Inspector::FrontendRouter*, Inspector::BackendDispatcher*)
 172 {
 173 }
 174
 175 void InspectorNetworkAgent::willDestroyFrontendAndBackend(Inspector::DisconnectReason)
 176 {
 177     ErrorString unused;
 178     disable(unused);
 179 }
 180
 181 static Ref<InspectorObject> buildObjectForHeaders(const HTTPHeaderMap& headers)
 182 {
 183     Ref<InspectorObject> headersObject = InspectorObject::create();
 184
 185     for (const auto& header : headers)
 186         headersObject->setString(header.key, header.value);
 187     return headersObject;
 188 }
 189
 190 Ref<Inspector::Protocol::Network::ResourceTiming> InspectorNetworkAgent::buildObjectForTiming(const NetworkLoadMetrics& timing, ResourceLoader& resourceLoader)
 191 {
 192     MonotonicTime startTime = resourceLoader.loadTiming().startTime();
 193     double startTimeInInspector = m_environment.executionStopwatch()->elapsedTimeSince(startTime);
 194
 195     return Inspector::Protocol::Network::ResourceTiming::create()
 196         .setStartTime(startTimeInInspector)
 197         .setDomainLookupStart(timing.domainLookupStart.milliseconds())
 198         .setDomainLookupEnd(timing.domainLookupEnd.milliseconds())
 199         .setConnectStart(timing.connectStart.milliseconds())
 200         .setConnectEnd(timing.connectEnd.milliseconds())
 201         .setSecureConnectionStart(timing.secureConnectionStart.milliseconds())
 202         .setRequestStart(timing.requestStart.milliseconds())
 203         .setResponseStart(timing.responseStart.milliseconds())
 204         .release();
 205 }
 206
 207 static Inspector::Protocol::Network::Metrics::Priority toProtocol(NetworkLoadPriority priority)
 208 {
 209     switch (priority) {
 210     case NetworkLoadPriority::Low:
 211         return Inspector::Protocol::Network::Metrics::Priority::Low;
 212     case NetworkLoadPriority::Medium:
 213         return Inspector::Protocol::Network::Metrics::Priority::Medium;
 214     case NetworkLoadPriority::High:
 215         return Inspector::Protocol::Network::Metrics::Priority::High;
 216     }
 217
 218     ASSERT_NOT_REACHED();
 219     return Inspector::Protocol::Network::Metrics::Priority::Medium;
 220 }
 221
 222 Ref<Inspector::Protocol::Network::Metrics> InspectorNetworkAgent::buildObjectForMetrics(const NetworkLoadMetrics& networkLoadMetrics)
 223 {
 224     auto metrics = Inspector::Protocol::Network::Metrics::create().release();
 225
 226     if (!networkLoadMetrics.protocol.isNull())
 227         metrics->setProtocol(networkLoadMetrics.protocol);
 228     if (networkLoadMetrics.priority)
 229         metrics->setPriority(toProtocol(*networkLoadMetrics.priority));
 230     if (networkLoadMetrics.remoteAddress)
 231         metrics->setRemoteAddress(*networkLoadMetrics.remoteAddress);
 232     if (networkLoadMetrics.connectionIdentifier)
 233         metrics->setConnectionIdentifier(*networkLoadMetrics.connectionIdentifier);
 234     if (networkLoadMetrics.requestHeaders)
 235         metrics->setRequestHeaders(buildObjectForHeaders(*networkLoadMetrics.requestHeaders));
 236
 237     if (networkLoadMetrics.requestHeaderBytesSent)
 238         metrics->setRequestHeaderBytesSent(*networkLoadMetrics.requestHeaderBytesSent);
 239     if (networkLoadMetrics.requestBodyBytesSent)
 240         metrics->setRequestBodyBytesSent(*networkLoadMetrics.requestBodyBytesSent);
 241     if (networkLoadMetrics.responseHeaderBytesReceived)
 242         metrics->setResponseHeaderBytesReceived(*networkLoadMetrics.responseHeaderBytesReceived);
 243     if (networkLoadMetrics.responseBodyBytesReceived)
 244         metrics->setResponseBodyBytesReceived(*networkLoadMetrics.responseBodyBytesReceived);
 245     if (networkLoadMetrics.responseBodyDecodedSize)
 246         metrics->setResponseBodyDecodedSize(*networkLoadMetrics.responseBodyDecodedSize);
 247
 248     return metrics;
 249 }
 250
 251 static Ref<Inspector::Protocol::Network::Request> buildObjectForResourceRequest(const ResourceRequest& request)
 252 {
 253     auto requestObject = Inspector::Protocol::Network::Request::create()
 254         .setUrl(request.url().string())
 255         .setMethod(request.httpMethod())
 256         .setHeaders(buildObjectForHeaders(request.httpHeaderFields()))
 257         .release();
 258     if (request.httpBody() && !request.httpBody()->isEmpty()) {
 259         auto bytes = request.httpBody()->flatten();
 260         requestObject->setPostData(String::fromUTF8WithLatin1Fallback(bytes.data(), bytes.size()));
 261     }
 262     return requestObject;
 263 }
 264
 265 static Inspector::Protocol::Network::Response::Source responseSource(ResourceResponse::Source source)
 266 {
 267     switch (source) {
 268     case ResourceResponse::Source::Unknown:
 269         return Inspector::Protocol::Network::Response::Source::Unknown;
 270     case ResourceResponse::Source::Network:
 271         return Inspector::Protocol::Network::Response::Source::Network;
 272     case ResourceResponse::Source::MemoryCache:
 273     case ResourceResponse::Source::MemoryCacheAfterValidation:
 274         return Inspector::Protocol::Network::Response::Source::MemoryCache;
 275     case ResourceResponse::Source::DiskCache:
 276     case ResourceResponse::Source::DiskCacheAfterValidation:
 277         return Inspector::Protocol::Network::Response::Source::DiskCache;
 278     }
 279
 280     ASSERT_NOT_REACHED();
 281     return Inspector::Protocol::Network::Response::Source::Unknown;
 282 }
 283
 284 RefPtr<Inspector::Protocol::Network::Response> InspectorNetworkAgent::buildObjectForResourceResponse(const ResourceResponse& response, ResourceLoader* resourceLoader)
 285 {
 286     if (response.isNull())
 287         return nullptr;
 288
 289     double status = response.httpStatusCode();
 290     Ref<InspectorObject> headers = buildObjectForHeaders(response.httpHeaderFields());
 291
 292     auto responseObject = Inspector::Protocol::Network::Response::create()
 293         .setUrl(response.url().string())
 294         .setStatus(status)
 295         .setStatusText(response.httpStatusText())
 296         .setHeaders(WTFMove(headers))
 297         .setMimeType(response.mimeType())
 298         .setSource(responseSource(response.source()))
 299         .release();
 300
 301     if (resourceLoader)
 302         responseObject->setTiming(buildObjectForTiming(response.deprecatedNetworkLoadMetrics(), *resourceLoader));
 303
 304     return WTFMove(responseObject);
 305 }
 306
 307 Ref<Inspector::Protocol::Network::CachedResource> InspectorNetworkAgent::buildObjectForCachedResource(CachedResource* cachedResource)
 308 {
 309     auto resourceObject = Inspector::Protocol::Network::CachedResource::create()
 310         .setUrl(cachedResource->url())
 311         .setType(InspectorPageAgent::cachedResourceTypeJson(*cachedResource))
 312         .setBodySize(cachedResource->encodedSize())
 313         .release();
 314
 315     auto resourceResponse = buildObjectForResourceResponse(cachedResource->response(), cachedResource->loader());
 316     resourceObject->setResponse(WTFMove(resourceResponse));
 317
 318     String sourceMappingURL = InspectorPageAgent::sourceMapURLForResource(cachedResource);
 319     if (!sourceMappingURL.isEmpty())
 320         resourceObject->setSourceMapURL(sourceMappingURL);
 321
 322     return resourceObject;
 323 }
 324
 325 InspectorNetworkAgent::~InspectorNetworkAgent()
 326 {
 327     if (m_enabled) {
 328         ErrorString unused;
 329         disable(unused);
 330     }
 331     ASSERT(!m_instrumentingAgents.inspectorNetworkAgent());
 332 }
 333
 334 double InspectorNetworkAgent::timestamp()
 335 {
 336     return m_environment.executionStopwatch()->elapsedTime();
 337 }
 338
 339 void InspectorNetworkAgent::willSendRequest(unsigned long identifier, DocumentLoader& loader, ResourceRequest& request, const ResourceResponse& redirectResponse)
 340 {
 341     if (request.hiddenFromInspector()) {
 342         m_hiddenRequestIdentifiers.add(identifier);
 343         return;
 344     }
 345
 346     String requestId = IdentifiersFactory::requestId(identifier);
 347     m_resourcesData->resourceCreated(requestId, m_pageAgent->loaderId(&loader));
 348
 349     CachedResource* cachedResource = InspectorPageAgent::cachedResource(loader.frame(), request.url());
 350     InspectorPageAgent::ResourceType type = cachedResource ? InspectorPageAgent::cachedResourceType(*cachedResource) : m_resourcesData->resourceType(requestId);
 351     if (type == InspectorPageAgent::OtherResource) {
 352         if (m_loadingXHRSynchronously)
 353             type = InspectorPageAgent::XHRResource;
 354         else if (equalIgnoringFragmentIdentifier(request.url(), loader.url()) && !loader.isCommitted())
 355             type = InspectorPageAgent::DocumentResource;
 356         else {
 357             for (auto& linkIcon : loader.linkIcons()) {
 358                 if (equalIgnoringFragmentIdentifier(request.url(), linkIcon.url)) {
 359                     type = InspectorPageAgent::ImageResource;
 360                     break;
 361                 }
 362             }
 363         }
 364     }
 365
 366     m_resourcesData->setResourceType(requestId, type);
 367
 368     for (auto& entry : m_extraRequestHeaders)
 369         request.setHTTPHeaderField(entry.key, entry.value);
 370
 371     Inspector::Protocol::Page::ResourceType resourceType = InspectorPageAgent::resourceTypeJson(type);
 372
 373     RefPtr<Inspector::Protocol::Network::Initiator> initiatorObject = buildInitiatorObject(loader.frame() ? loader.frame()->document() : nullptr);
 374     String targetId = request.initiatorIdentifier();
 375
 376     m_frontendDispatcher->requestWillBeSent(requestId, m_pageAgent->frameId(loader.frame()), m_pageAgent->loaderId(&loader), loader.url().string(), buildObjectForResourceRequest(request), timestamp(), initiatorObject, buildObjectForResourceResponse(redirectResponse, nullptr), type != InspectorPageAgent::OtherResource ? &resourceType : nullptr, targetId.isEmpty() ? nullptr : &targetId);
 377 }
 378
 379 void InspectorNetworkAgent::didReceiveResponse(unsigned long identifier, DocumentLoader& loader, const ResourceResponse& response, ResourceLoader* resourceLoader)
 380 {
 381     if (m_hiddenRequestIdentifiers.contains(identifier))
 382         return;
 383
 384     String requestId = IdentifiersFactory::requestId(identifier);
 385     RefPtr<Inspector::Protocol::Network::Response> resourceResponse = buildObjectForResourceResponse(response, resourceLoader);
 386
 387     bool isNotModified = response.httpStatusCode() == 304;
 388
 389     CachedResource* cachedResource = nullptr;
 390     if (resourceLoader && resourceLoader->isSubresourceLoader() && !isNotModified)
 391         cachedResource = static_cast<SubresourceLoader*>(resourceLoader)->cachedResource();
 392     if (!cachedResource)
 393         cachedResource = InspectorPageAgent::cachedResource(loader.frame(), response.url());
 394
 395     if (cachedResource) {
 396         // Use mime type from cached resource in case the one in response is empty.
 397         if (resourceResponse && response.mimeType().isEmpty())
 398             resourceResponse->setString(Inspector::Protocol::Network::Response::MimeType, cachedResource->response().mimeType());
 399         m_resourcesData->addCachedResource(requestId, cachedResource);
 400     }
 401
 402     InspectorPageAgent::ResourceType type = m_resourcesData->resourceType(requestId);
 403     InspectorPageAgent::ResourceType newType = cachedResource ? InspectorPageAgent::cachedResourceType(*cachedResource) : type;
 404
 405     // FIXME: XHRResource is returned for CachedResource::RawResource, it should be OtherResource unless it truly is an XHR.
 406     // RawResource is used for loading worker scripts, and those should stay as ScriptResource and not change to XHRResource.
 407     if (type != newType && newType != InspectorPageAgent::XHRResource && newType != InspectorPageAgent::OtherResource)
 408         type = newType;
 409
 410     m_resourcesData->responseReceived(requestId, m_pageAgent->frameId(loader.frame()), response);
 411     m_resourcesData->setResourceType(requestId, type);
 412
 413     m_frontendDispatcher->responseReceived(requestId, m_pageAgent->frameId(loader.frame()), m_pageAgent->loaderId(&loader), timestamp(), InspectorPageAgent::resourceTypeJson(type), resourceResponse);
 414
 415     // If we revalidated the resource and got Not modified, send content length following didReceiveResponse
 416     // as there will be no calls to didReceiveData from the network stack.
 417     if (isNotModified && cachedResource && cachedResource->encodedSize())
 418         didReceiveData(identifier, nullptr, cachedResource->encodedSize(), 0);
 419 }
 420
 421 static bool isErrorStatusCode(int statusCode)
 422 {
 423     return statusCode >= 400;
 424 }
 425
 426 void InspectorNetworkAgent::didReceiveData(unsigned long identifier, const char* data, int dataLength, int encodedDataLength)
 427 {
 428     if (m_hiddenRequestIdentifiers.contains(identifier))
 429         return;
 430
 431     String requestId = IdentifiersFactory::requestId(identifier);
 432
 433     if (data) {
 434         NetworkResourcesData::ResourceData const* resourceData = m_resourcesData->data(requestId);
 435         if (resourceData && !m_loadingXHRSynchronously && (!resourceData->cachedResource() || resourceData->cachedResource()->dataBufferingPolicy() == DoNotBufferData || isErrorStatusCode(resourceData->httpStatusCode())))
 436             m_resourcesData->maybeAddResourceData(requestId, data, dataLength);
 437     }
 438
 439     m_frontendDispatcher->dataReceived(requestId, timestamp(), dataLength, encodedDataLength);
 440 }
 441
 442 void InspectorNetworkAgent::didFinishLoading(unsigned long identifier, DocumentLoader& loader, const NetworkLoadMetrics& networkLoadMetrics, ResourceLoader* resourceLoader)
 443 {
 444     if (m_hiddenRequestIdentifiers.remove(identifier))
 445         return;
 446
 447     double elapsedFinishTime;
 448     if (resourceLoader && networkLoadMetrics.isComplete()) {
 449         MonotonicTime startTime = resourceLoader->loadTiming().startTime();
 450         double startTimeInInspector = m_environment.executionStopwatch()->elapsedTimeSince(startTime);
 451         elapsedFinishTime = startTimeInInspector + networkLoadMetrics.responseEnd.seconds();
 452     } else
 453         elapsedFinishTime = timestamp();
 454
 455     String requestId = IdentifiersFactory::requestId(identifier);
 456     if (m_resourcesData->resourceType(requestId) == InspectorPageAgent::DocumentResource)
 457         m_resourcesData->addResourceSharedBuffer(requestId, loader.frameLoader()->documentLoader()->mainResourceData(), loader.frame()->document()->encoding());
 458
 459     m_resourcesData->maybeDecodeDataToContent(requestId);
 460
 461     String sourceMappingURL;
 462     NetworkResourcesData::ResourceData const* resourceData = m_resourcesData->data(requestId);
 463     if (resourceData && resourceData->cachedResource())
 464         sourceMappingURL = InspectorPageAgent::sourceMapURLForResource(resourceData->cachedResource());
 465
 466     RefPtr<Inspector::Protocol::Network::Metrics> metrics = buildObjectForMetrics(networkLoadMetrics);
 467
 468     m_frontendDispatcher->loadingFinished(requestId, elapsedFinishTime, !sourceMappingURL.isEmpty() ? &sourceMappingURL : nullptr, metrics);
 469 }
 470
 471 void InspectorNetworkAgent::didFailLoading(unsigned long identifier, DocumentLoader& loader, const ResourceError& error)
 472 {
 473     if (m_hiddenRequestIdentifiers.remove(identifier))
 474         return;
 475
 476     String requestId = IdentifiersFactory::requestId(identifier);
 477
 478     if (m_resourcesData->resourceType(requestId) == InspectorPageAgent::DocumentResource) {
 479         Frame* frame = loader.frame();
 480         if (frame && frame->loader().documentLoader() && frame->document()) {
 481             m_resourcesData->addResourceSharedBuffer(requestId,
 482                 frame->loader().documentLoader()->mainResourceData(),
 483                 frame->document()->encoding());
 484         }
 485     }
 486
 487     bool canceled = error.isCancellation();
 488     m_frontendDispatcher->loadingFailed(requestId, timestamp(), error.localizedDescription(), canceled ? &canceled : nullptr);
 489 }
 490
 491 void InspectorNetworkAgent::didLoadResourceFromMemoryCache(DocumentLoader& loader, CachedResource& resource)
 492 {
 493     String loaderId = m_pageAgent->loaderId(&loader);
 494     String frameId = m_pageAgent->frameId(loader.frame());
 495     unsigned long identifier = loader.frame()->page()->progress().createUniqueIdentifier();
 496     String requestId = IdentifiersFactory::requestId(identifier);
 497
 498     m_resourcesData->resourceCreated(requestId, loaderId);
 499     m_resourcesData->addCachedResource(requestId, &resource);
 500
 501     RefPtr<Inspector::Protocol::Network::Initiator> initiatorObject = buildInitiatorObject(loader.frame() ? loader.frame()->document() : nullptr);
 502
 503     // FIXME: It would be ideal to generate the Network.Response with the MemoryCache source
 504     // instead of whatever ResourceResponse::Source the CachedResources's response has.
 505     // The frontend already knows for certain that this was served from the memory cache.
 506
 507     m_frontendDispatcher->requestServedFromMemoryCache(requestId, frameId, loaderId, loader.url().string(), timestamp(), initiatorObject, buildObjectForCachedResource(&resource));
 508 }
 509
 510 void InspectorNetworkAgent::setInitialScriptContent(unsigned long identifier, const String& sourceString)
 511 {
 512     m_resourcesData->setResourceContent(IdentifiersFactory::requestId(identifier), sourceString);
 513 }
 514
 515 void InspectorNetworkAgent::didReceiveScriptResponse(unsigned long identifier)
 516 {
 517     m_resourcesData->setResourceType(IdentifiersFactory::requestId(identifier), InspectorPageAgent::ScriptResource);
 518 }
 519
 520 void InspectorNetworkAgent::didReceiveThreadableLoaderResponse(unsigned long identifier, DocumentThreadableLoader& documentThreadableLoader)
 521 {
 522     String initiator = documentThreadableLoader.options().initiator;
 523     if (initiator == cachedResourceRequestInitiators().fetch)
 524         m_resourcesData->setResourceType(IdentifiersFactory::requestId(identifier), InspectorPageAgent::FetchResource);
 525     else if (initiator == cachedResourceRequestInitiators().xmlhttprequest)
 526         m_resourcesData->setResourceType(IdentifiersFactory::requestId(identifier), InspectorPageAgent::XHRResource);
 527 }
 528
 529 void InspectorNetworkAgent::didFinishXHRLoading(unsigned long identifier, const String& decodedText)
 530 {
 531     m_resourcesData->setResourceContent(IdentifiersFactory::requestId(identifier), decodedText);
 532 }
 533
 534 void InspectorNetworkAgent::willLoadXHRSynchronously()
 535 {
 536     m_loadingXHRSynchronously = true;
 537 }
 538
 539 void InspectorNetworkAgent::didLoadXHRSynchronously()
 540 {
 541     m_loadingXHRSynchronously = false;
 542 }
 543
 544 void InspectorNetworkAgent::willDestroyCachedResource(CachedResource& cachedResource)
 545 {
 546     Vector<String> requestIds = m_resourcesData->removeCachedResource(&cachedResource);
 547     if (!requestIds.size())
 548         return;
 549
 550     String content;
 551     bool base64Encoded;
 552     if (!InspectorPageAgent::cachedResourceContent(&cachedResource, &content, &base64Encoded))
 553         return;
 554     for (auto& id : requestIds)
 555         m_resourcesData->setResourceContent(id, content, base64Encoded);
 556 }
 557
 558 void InspectorNetworkAgent::willRecalculateStyle()
 559 {
 560     m_isRecalculatingStyle = true;
 561 }
 562
 563 void InspectorNetworkAgent::didRecalculateStyle()
 564 {
 565     m_isRecalculatingStyle = false;
 566     m_styleRecalculationInitiator = nullptr;
 567 }
 568
 569 void InspectorNetworkAgent::didScheduleStyleRecalculation(Document& document)
 570 {
 571     if (!m_styleRecalculationInitiator)
 572         m_styleRecalculationInitiator = buildInitiatorObject(&document);
 573 }
 574
 575 RefPtr<Inspector::Protocol::Network::Initiator> InspectorNetworkAgent::buildInitiatorObject(Document* document)
 576 {
 577     Ref<ScriptCallStack> stackTrace = createScriptCallStack(JSMainThreadExecState::currentState(), ScriptCallStack::maxCallStackSizeToCapture);
 578     if (stackTrace->size() > 0) {
 579         auto initiatorObject = Inspector::Protocol::Network::Initiator::create()
 580             .setType(Inspector::Protocol::Network::Initiator::Type::Script)
 581             .release();
 582         initiatorObject->setStackTrace(stackTrace->buildInspectorArray());
 583         return WTFMove(initiatorObject);
 584     }
 585
 586     if (document && document->scriptableDocumentParser()) {
 587         auto initiatorObject = Inspector::Protocol::Network::Initiator::create()
 588             .setType(Inspector::Protocol::Network::Initiator::Type::Parser)
 589             .release();
 590         initiatorObject->setUrl(document->url().string());
 591         initiatorObject->setLineNumber(document->scriptableDocumentParser()->textPosition().m_line.oneBasedInt());
 592         return WTFMove(initiatorObject);
 593     }
 594
 595     if (m_isRecalculatingStyle && m_styleRecalculationInitiator)
 596         return m_styleRecalculationInitiator;
 597
 598     return Inspector::Protocol::Network::Initiator::create()
 599         .setType(Inspector::Protocol::Network::Initiator::Type::Other)
 600         .release();
 601 }
 602
 603 void InspectorNetworkAgent::didCreateWebSocket(unsigned long identifier, const URL& requestURL)
 604 {
 605     m_frontendDispatcher->webSocketCreated(IdentifiersFactory::requestId(identifier), requestURL.string());
 606 }
 607
 608 void InspectorNetworkAgent::willSendWebSocketHandshakeRequest(unsigned long identifier, const ResourceRequest& request)
 609 {
 610     auto requestObject = Inspector::Protocol::Network::WebSocketRequest::create()
 611         .setHeaders(buildObjectForHeaders(request.httpHeaderFields()))
 612         .release();
 613     m_frontendDispatcher->webSocketWillSendHandshakeRequest(IdentifiersFactory::requestId(identifier), timestamp(), currentTime(), WTFMove(requestObject));
 614 }
 615
 616 void InspectorNetworkAgent::didReceiveWebSocketHandshakeResponse(unsigned long identifier, const ResourceResponse& response)
 617 {
 618     auto responseObject = Inspector::Protocol::Network::WebSocketResponse::create()
 619         .setStatus(response.httpStatusCode())
 620         .setStatusText(response.httpStatusText())
 621         .setHeaders(buildObjectForHeaders(response.httpHeaderFields()))
 622         .release();
 623     m_frontendDispatcher->webSocketHandshakeResponseReceived(IdentifiersFactory::requestId(identifier), timestamp(), WTFMove(responseObject));
 624 }
 625
 626 void InspectorNetworkAgent::didCloseWebSocket(unsigned long identifier)
 627 {
 628     m_frontendDispatcher->webSocketClosed(IdentifiersFactory::requestId(identifier), timestamp());
 629 }
 630
 631 void InspectorNetworkAgent::didReceiveWebSocketFrame(unsigned long identifier, const WebSocketFrame& frame)
 632 {
 633     auto frameObject = Inspector::Protocol::Network::WebSocketFrame::create()
 634         .setOpcode(frame.opCode)
 635         .setMask(frame.masked)
 636         .setPayloadData(String::fromUTF8WithLatin1Fallback(frame.payload, frame.payloadLength))
 637         .setPayloadLength(frame.payloadLength)
 638         .release();
 639     m_frontendDispatcher->webSocketFrameReceived(IdentifiersFactory::requestId(identifier), timestamp(), WTFMove(frameObject));
 640 }
 641
 642 void InspectorNetworkAgent::didSendWebSocketFrame(unsigned long identifier, const WebSocketFrame& frame)
 643 {
 644     auto frameObject = Inspector::Protocol::Network::WebSocketFrame::create()
 645         .setOpcode(frame.opCode)
 646         .setMask(frame.masked)
 647         .setPayloadData(String::fromUTF8WithLatin1Fallback(frame.payload, frame.payloadLength))
 648         .setPayloadLength(frame.payloadLength)
 649         .release();
 650     m_frontendDispatcher->webSocketFrameSent(IdentifiersFactory::requestId(identifier), timestamp(), WTFMove(frameObject));
 651 }
 652
 653 void InspectorNetworkAgent::didReceiveWebSocketFrameError(unsigned long identifier, const String& errorMessage)
 654 {
 655     m_frontendDispatcher->webSocketFrameError(IdentifiersFactory::requestId(identifier), timestamp(), errorMessage);
 656 }
 657
 658 void InspectorNetworkAgent::enable(ErrorString&)
 659 {
 660     enable();
 661 }
 662
 663 void InspectorNetworkAgent::enable()
 664 {
 665     m_enabled = true;
 666     m_instrumentingAgents.setInspectorNetworkAgent(this);
 667
 668     LockHolder lock(WebSocket::allActiveWebSocketsMutex());
 669
 670     for (WebSocket* webSocket : WebSocket::allActiveWebSockets(lock)) {
 671         if (!is<Document>(webSocket->scriptExecutionContext()) || !is<WebSocketChannel>(webSocket->channel().get()))
 672             continue;
 673
 674         Document* document = downcast<Document>(webSocket->scriptExecutionContext());
 675         if (document->page() != &m_pageAgent->page())
 676             continue;
 677
 678         WebSocketChannel* channel = downcast<WebSocketChannel>(webSocket->channel().get());
 679         if (!channel)
 680             continue;
 681
 682         unsigned identifier = channel->identifier();
 683         didCreateWebSocket(identifier, webSocket->url());
 684         willSendWebSocketHandshakeRequest(identifier, channel->clientHandshakeRequest());
 685
 686         if (channel->handshakeMode() == WebSocketHandshake::Connected)
 687             didReceiveWebSocketHandshakeResponse(identifier, channel->serverHandshakeResponse());
 688
 689         if (webSocket->readyState() == WebSocket::CLOSED)
 690             didCloseWebSocket(identifier);
 691     }
 692 }
 693
 694 void InspectorNetworkAgent::disable(ErrorString&)
 695 {
 696     m_enabled = false;
 697     m_instrumentingAgents.setInspectorNetworkAgent(nullptr);
 698     m_resourcesData->clear();
 699     m_extraRequestHeaders.clear();
 700
 701     m_pageAgent->page().setResourceCachingDisabledOverride(false);
 702 }
 703
 704 void InspectorNetworkAgent::setExtraHTTPHeaders(ErrorString&, const InspectorObject& headers)
 705 {
 706     for (auto& entry : headers) {
 707         String stringValue;
 708         if (entry.value->asString(stringValue))
 709             m_extraRequestHeaders.set(entry.key, stringValue);
 710     }
 711 }
 712
 713 void InspectorNetworkAgent::getResponseBody(ErrorString& errorString, const String& requestId, String* content, bool* base64Encoded)
 714 {
 715     NetworkResourcesData::ResourceData const* resourceData = m_resourcesData->data(requestId);
 716     if (!resourceData) {
 717         errorString = ASCIILiteral("No resource with given identifier found");
 718         return;
 719     }
 720
 721     if (resourceData->hasContent()) {
 722         *base64Encoded = resourceData->base64Encoded();
 723         *content = resourceData->content();
 724         return;
 725     }
 726
 727     if (resourceData->isContentEvicted()) {
 728         errorString = ASCIILiteral("Request content was evicted from inspector cache");
 729         return;
 730     }
 731
 732     if (resourceData->buffer() && !resourceData->textEncodingName().isNull()) {
 733         *base64Encoded = false;
 734         if (InspectorPageAgent::sharedBufferContent(resourceData->buffer(), resourceData->textEncodingName(), *base64Encoded, content))
 735             return;
 736     }
 737
 738     if (resourceData->cachedResource()) {
 739         if (InspectorPageAgent::cachedResourceContent(resourceData->cachedResource(), content, base64Encoded))
 740             return;
 741     }
 742
 743     errorString = ASCIILiteral("No data found for resource with given identifier");
 744 }
 745
 746 void InspectorNetworkAgent::setResourceCachingDisabled(ErrorString&, bool disabled)
 747 {
 748     m_pageAgent->page().setResourceCachingDisabledOverride(disabled);
 749 }
 750
 751 void InspectorNetworkAgent::loadResource(ErrorString& errorString, const String& frameId, const String& urlString, Ref<LoadResourceCallback>&& callback)
 752 {
 753     Frame* frame = m_pageAgent->assertFrame(errorString, frameId);
 754     if (!frame)
 755         return;
 756
 757     Document* document = frame->document();
 758     if (!document) {
 759         errorString = ASCIILiteral("No Document instance for the specified frame");
 760         return;
 761     }
 762
 763     URL url = document->completeURL(urlString);
 764     ResourceRequest request(url);
 765     request.setHTTPMethod(ASCIILiteral("GET"));
 766     request.setHiddenFromInspector(true);
 767
 768     ThreadableLoaderOptions options;
 769     options.sendLoadCallbacks = SendCallbacks; // So we remove this from m_hiddenRequestIdentifiers on completion.
 770     options.defersLoadingPolicy = DefersLoadingPolicy::DisallowDefersLoading; // So the request is never deferred.
 771     options.mode = FetchOptions::Mode::NoCors;
 772     options.credentials = FetchOptions::Credentials::SameOrigin;
 773     options.contentSecurityPolicyEnforcement = ContentSecurityPolicyEnforcement::DoNotEnforce;
 774
 775     // InspectorThreadableLoaderClient deletes itself when the load completes or fails.
 776     InspectorThreadableLoaderClient* inspectorThreadableLoaderClient = new InspectorThreadableLoaderClient(callback.copyRef());
 777     auto loader = DocumentThreadableLoader::create(*document, *inspectorThreadableLoaderClient, WTFMove(request), options);
 778     if (!loader)
 779         return;
 780
 781     // If the load already completed, inspectorThreadableLoaderClient will have been deleted and we will have already called the callback.
 782     if (!callback->isActive())
 783         return;
 784
 785     inspectorThreadableLoaderClient->setLoader(WTFMove(loader));
 786 }
 787
 788 WebSocket* InspectorNetworkAgent::webSocketForRequestId(const String& requestId)
 789 {
 790     LockHolder lock(WebSocket::allActiveWebSocketsMutex());
 791
 792     for (WebSocket* webSocket : WebSocket::allActiveWebSockets(lock)) {
 793         if (!is<WebSocketChannel>(webSocket->channel().get()))
 794             continue;
 795
 796         WebSocketChannel* channel = downcast<WebSocketChannel>(webSocket->channel().get());
 797         if (!channel)
 798             continue;
 799
 800         if (IdentifiersFactory::requestId(channel->identifier()) != requestId)
 801             continue;
 802
 803         // FIXME: <webkit.org/b/168475> Web Inspector: Correctly display iframe's and worker's WebSockets
 804         if (!is<Document>(webSocket->scriptExecutionContext()))
 805             continue;
 806
 807         Document* document = downcast<Document>(webSocket->scriptExecutionContext());
 808         if (document->page() != &m_pageAgent->page())
 809             continue;
 810
 811         return webSocket;
 812     }
 813
 814     return nullptr;
 815 }
 816
 817 static JSC::JSValue webSocketAsScriptValue(JSC::ExecState& state, WebSocket* webSocket)
 818 {
 819     JSC::JSLockHolder lock(&state);
 820     return toJS(&state, deprecatedGlobalObjectForPrototype(&state), webSocket);
 821 }
 822
 823 void InspectorNetworkAgent::resolveWebSocket(ErrorString& errorString, const String& requestId, const String* const objectGroup, RefPtr<Inspector::Protocol::Runtime::RemoteObject>& result)
 824 {
 825     WebSocket* webSocket = webSocketForRequestId(requestId);
 826     if (!webSocket) {
 827         errorString = ASCIILiteral("WebSocket not found");
 828         return;
 829     }
 830
 831     // FIXME: <webkit.org/b/168475> Web Inspector: Correctly display iframe's and worker's WebSockets
 832     Document* document = downcast<Document>(webSocket->scriptExecutionContext());
 833     Frame* frame = document->frame();
 834     if (!frame) {
 835         errorString = ASCIILiteral("WebSocket belongs to document without a frame");
 836         return;
 837     }
 838
 839     auto& state = *mainWorldExecState(frame);
 840     auto injectedScript = m_injectedScriptManager.injectedScriptFor(&state);
 841     ASSERT(!injectedScript.hasNoValue());
 842
 843     String objectGroupName = objectGroup ? *objectGroup : String();
 844     result = injectedScript.wrapObject(webSocketAsScriptValue(state, webSocket), objectGroupName);
 845 }
 846
 847 static Ref<Inspector::Protocol::Page::SearchResult> buildObjectForSearchResult(const String& requestId, const String& frameId, const String& url, int matchesCount)
 848 {
 849     auto searchResult = Inspector::Protocol::Page::SearchResult::create()
 850         .setUrl(url)
 851         .setFrameId(frameId)
 852         .setMatchesCount(matchesCount)
 853         .release();
 854     searchResult->setRequestId(requestId);
 855     return searchResult;
 856 }
 857
 858 void InspectorNetworkAgent::searchOtherRequests(const JSC::Yarr::RegularExpression& regex, RefPtr<Inspector::Protocol::Array<Inspector::Protocol::Page::SearchResult>>& result)
 859 {
 860     Vector<NetworkResourcesData::ResourceData*> resources = m_resourcesData->resources();
 861     for (auto* resourceData : resources) {
 862         if (resourceData->hasContent()) {
 863             int matchesCount = ContentSearchUtilities::countRegularExpressionMatches(regex, resourceData->content());
 864             if (matchesCount)
 865                 result->addItem(buildObjectForSearchResult(resourceData->requestId(), resourceData->frameId(), resourceData->url(), matchesCount));
 866         }
 867     }
 868 }
 869
 870 void InspectorNetworkAgent::searchInRequest(ErrorString& errorString, const String& requestId, const String& query, bool caseSensitive, bool isRegex, RefPtr<Inspector::Protocol::Array<Inspector::Protocol::GenericTypes::SearchMatch>>& results)
 871 {
 872     NetworkResourcesData::ResourceData const* resourceData = m_resourcesData->data(requestId);
 873     if (!resourceData) {
 874         errorString = ASCIILiteral("No resource with given identifier found");
 875         return;
 876     }
 877
 878     if (!resourceData->hasContent()) {
 879         errorString = ASCIILiteral("No resource content");
 880         return;
 881     }
 882
 883     results = ContentSearchUtilities::searchInTextByLines(resourceData->content(), query, caseSensitive, isRegex);
 884 }
 885
 886 void InspectorNetworkAgent::mainFrameNavigated(DocumentLoader& loader)
 887 {
 888     m_resourcesData->clear(m_pageAgent->loaderId(&loader));
 889 }
 890
 891 } // namespace WebCore