Source/WebInspectorUI/UserInterface/Models/Recording.js

   1 /*
   2  * Copyright (C) 2017 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
  14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
  17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  23  * THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 WI.Recording = class Recording extends WI.Object
  27 {
  28     constructor(version, type, initialState, frames, data)
  29     {
  30         super();
  31
  32         this._version = version;
  33         this._type = type;
  34         this._initialState = initialState;
  35         this._frames = frames;
  36         this._data = data;
  37         this._displayName = WI.UIString("Recording");
  38
  39         this._swizzle = [];
  40         this._actions = [new WI.RecordingInitialStateAction].concat(...this._frames.map((frame) => frame.actions));
  41         this._visualActionIndexes = [];
  42         this._source = null;
  43
  44         this._processContext = null;
  45         this._processStates = [];
  46         this._processing = false;
  47     }
  48
  49     static fromPayload(payload, frames)
  50     {
  51         if (typeof payload !== "object" || payload === null)
  52             return null;
  53
  54         if (isNaN(payload.version) || payload.version <= 0)
  55             return null;
  56
  57         let type = null;
  58         switch (payload.type) {
  59         case RecordingAgent.Type.Canvas2D:
  60             type = WI.Recording.Type.Canvas2D;
  61             break;
  62         case RecordingAgent.Type.CanvasBitmapRenderer:
  63             type = WI.Recording.Type.CanvasBitmapRenderer;
  64             break;
  65         case RecordingAgent.Type.CanvasWebGL:
  66             type = WI.Recording.Type.CanvasWebGL;
  67             break;
  68         default:
  69             type = String(payload.type);
  70             break;
  71         }
  72
  73         if (typeof payload.initialState !== "object" || payload.initialState === null)
  74             payload.initialState = {};
  75         if (typeof payload.initialState.attributes !== "object" || payload.initialState.attributes === null)
  76             payload.initialState.attributes = {};
  77         if (!Array.isArray(payload.initialState.states) || payload.initialState.states.some((item) => typeof item !== "object" || item === null)) {
  78             payload.initialState.states = [];
  79
  80             // COMPATIBILITY (iOS 12.0): Recording.InitialState.states did not exist yet
  81             if (!isEmptyObject(payload.initialState.attributes)) {
  82                 let {width, height, ...state} = payload.initialState.attributes;
  83                 if (!isEmptyObject(state))
  84                     payload.initialState.states.push(state);
  85             }
  86         }
  87         if (!Array.isArray(payload.initialState.parameters))
  88             payload.initialState.parameters = [];
  89         if (typeof payload.initialState.content !== "string")
  90             payload.initialState.content = "";
  91
  92         if (!Array.isArray(payload.frames))
  93             payload.frames = [];
  94
  95         if (!Array.isArray(payload.data))
  96             payload.data = [];
  97
  98         if (!frames)
  99             frames = payload.frames.map(WI.RecordingFrame.fromPayload)
 100
 101         return new WI.Recording(payload.version, type, payload.initialState, frames, payload.data);
 102     }
 103
 104     static displayNameForSwizzleType(swizzleType)
 105     {
 106         switch (swizzleType) {
 107         case WI.Recording.Swizzle.None:
 108             return WI.unlocalizedString("None");
 109         case WI.Recording.Swizzle.Number:
 110             return WI.unlocalizedString("Number");
 111         case WI.Recording.Swizzle.Boolean:
 112             return WI.unlocalizedString("Boolean");
 113         case WI.Recording.Swizzle.String:
 114             return WI.unlocalizedString("String");
 115         case WI.Recording.Swizzle.Array:
 116             return WI.unlocalizedString("Array");
 117         case WI.Recording.Swizzle.TypedArray:
 118             return WI.unlocalizedString("TypedArray");
 119         case WI.Recording.Swizzle.Image:
 120             return WI.unlocalizedString("Image");
 121         case WI.Recording.Swizzle.ImageData:
 122             return WI.unlocalizedString("ImageData");
 123         case WI.Recording.Swizzle.DOMMatrix:
 124             return WI.unlocalizedString("DOMMatrix");
 125         case WI.Recording.Swizzle.Path2D:
 126             return WI.unlocalizedString("Path2D");
 127         case WI.Recording.Swizzle.CanvasGradient:
 128             return WI.unlocalizedString("CanvasGradient");
 129         case WI.Recording.Swizzle.CanvasPattern:
 130             return WI.unlocalizedString("CanvasPattern");
 131         case WI.Recording.Swizzle.WebGLBuffer:
 132             return WI.unlocalizedString("WebGLBuffer");
 133         case WI.Recording.Swizzle.WebGLFramebuffer:
 134             return WI.unlocalizedString("WebGLFramebuffer");
 135         case WI.Recording.Swizzle.WebGLRenderbuffer:
 136             return WI.unlocalizedString("WebGLRenderbuffer");
 137         case WI.Recording.Swizzle.WebGLTexture:
 138             return WI.unlocalizedString("WebGLTexture");
 139         case WI.Recording.Swizzle.WebGLShader:
 140             return WI.unlocalizedString("WebGLShader");
 141         case WI.Recording.Swizzle.WebGLProgram:
 142             return WI.unlocalizedString("WebGLProgram");
 143         case WI.Recording.Swizzle.WebGLUniformLocation:
 144             return WI.unlocalizedString("WebGLUniformLocation");
 145         case WI.Recording.Swizzle.ImageBitmap:
 146             return WI.unlocalizedString("ImageBitmap");
 147         default:
 148             console.error("Unknown swizzle type", swizzleType);
 149             return null;
 150         }
 151     }
 152
 153     static synthesizeError(message)
 154     {
 155         const target = WI.mainTarget;
 156         const source = WI.ConsoleMessage.MessageSource.Other;
 157         const level = WI.ConsoleMessage.MessageLevel.Error;
 158         let consoleMessage = new WI.ConsoleMessage(target, source, level, WI.UIString("Recording error: %s").format(message));
 159         consoleMessage.shouldRevealConsole = true;
 160
 161         WI.consoleLogViewController.appendConsoleMessage(consoleMessage);
 162     }
 163
 164     // Public
 165
 166     get displayName() { return this._displayName; }
 167     get type() { return this._type; }
 168     get initialState() { return this._initialState; }
 169     get frames() { return this._frames; }
 170     get data() { return this._data; }
 171     get actions() { return this._actions; }
 172     get visualActionIndexes() { return this._visualActionIndexes; }
 173
 174     get source() { return this._source; }
 175     set source(source) { this._source = source; }
 176
 177     get processing() { return this._processing; }
 178
 179     get ready()
 180     {
 181         return this._actions.lastValue.ready;
 182     }
 183
 184     startProcessing()
 185     {
 186         console.assert(!this._processing, "Cannot start an already started process().");
 187         console.assert(!this.ready, "Cannot start a completed process().");
 188         if (this._processing || this.ready)
 189             return;
 190
 191         this._processing = true;
 192
 193         this._process();
 194     }
 195
 196     stopProcessing()
 197     {
 198         console.assert(this._processing, "Cannot stop an already stopped process().");
 199         console.assert(!this.ready, "Cannot stop a completed process().");
 200         if (!this._processing || this.ready)
 201             return;
 202
 203         this._processing = false;
 204     }
 205
 206     createDisplayName(suggestedName)
 207     {
 208         let recordingNameSet;
 209         if (this._source) {
 210             recordingNameSet = this._source[WI.Recording.CanvasRecordingNamesSymbol];
 211             if (!recordingNameSet)
 212                 this._source[WI.Recording.CanvasRecordingNamesSymbol] = recordingNameSet = new Set;
 213         } else
 214             recordingNameSet = WI.Recording._importedRecordingNameSet;
 215
 216         let name;
 217         if (suggestedName) {
 218             name = suggestedName;
 219             let duplicateNumber = 2;
 220             while (recordingNameSet.has(name))
 221                 name = `${suggestedName} (${duplicateNumber++})`;
 222         } else {
 223             let recordingNumber = 1;
 224             do {
 225                 name = WI.UIString("Recording %d").format(recordingNumber++);
 226             } while (recordingNameSet.has(name));
 227         }
 228
 229         recordingNameSet.add(name);
 230         this._displayName = name;
 231     }
 232
 233     async swizzle(index, type)
 234     {
 235         if (typeof this._swizzle[index] !== "object")
 236             this._swizzle[index] = {};
 237
 238         if (type === WI.Recording.Swizzle.Number)
 239             return parseFloat(index);
 240
 241         if (type === WI.Recording.Swizzle.Boolean)
 242             return !!index;
 243
 244         if (type === WI.Recording.Swizzle.Array)
 245             return Array.isArray(index) ? index : [];
 246
 247         if (type === WI.Recording.Swizzle.DOMMatrix)
 248             return new DOMMatrix(index);
 249
 250         // FIXME: <https://webkit.org/b/176009> Web Inspector: send data for WebGL objects during a recording instead of a placeholder string
 251         if (type === WI.Recording.Swizzle.TypedArray
 252             || type === WI.Recording.Swizzle.WebGLBuffer
 253             || type === WI.Recording.Swizzle.WebGLFramebuffer
 254             || type === WI.Recording.Swizzle.WebGLRenderbuffer
 255             || type === WI.Recording.Swizzle.WebGLTexture
 256             || type === WI.Recording.Swizzle.WebGLShader
 257             || type === WI.Recording.Swizzle.WebGLProgram
 258             || type === WI.Recording.Swizzle.WebGLUniformLocation) {
 259             return index;
 260         }
 261
 262         if (!(type in this._swizzle[index])) {
 263             try {
 264                 let data = this._data[index];
 265                 switch (type) {
 266                 case WI.Recording.Swizzle.None:
 267                     this._swizzle[index][type] = data;
 268                     break;
 269
 270                 case WI.Recording.Swizzle.String:
 271                     this._swizzle[index][type] = String(data);
 272                     break;
 273
 274                 case WI.Recording.Swizzle.Image:
 275                     this._swizzle[index][type] = await WI.ImageUtilities.promisifyLoad(data);
 276                     break;
 277
 278                 case WI.Recording.Swizzle.ImageData:
 279                     this._swizzle[index][type] = new ImageData(new Uint8ClampedArray(data[0]), parseInt(data[1]), parseInt(data[2]));
 280                     break;
 281
 282                 case WI.Recording.Swizzle.Path2D:
 283                     this._swizzle[index][type] = new Path2D(data);
 284                     break;
 285
 286                 case WI.Recording.Swizzle.CanvasGradient:
 287                     var gradientType = await this.swizzle(data[0], WI.Recording.Swizzle.String);
 288
 289                     WI.ImageUtilities.scratchCanvasContext2D((context) => {
 290                         this._swizzle[index][type] = gradientType === "radial-gradient" ? context.createRadialGradient(...data[1]) : context.createLinearGradient(...data[1]);
 291                     });
 292
 293                     for (let stop of data[2]) {
 294                         let color = await this.swizzle(stop[1], WI.Recording.Swizzle.String);
 295                         this._swizzle[index][type].addColorStop(stop[0], color);
 296                     }
 297                     break;
 298
 299                 case WI.Recording.Swizzle.CanvasPattern:
 300                     var [image, repeat] = await Promise.all([
 301                         this.swizzle(data[0], WI.Recording.Swizzle.Image),
 302                         this.swizzle(data[1], WI.Recording.Swizzle.String),
 303                     ]);
 304
 305                     WI.ImageUtilities.scratchCanvasContext2D((context) => {
 306                         this._swizzle[index][type] = context.createPattern(image, repeat);
 307                         this._swizzle[index][type].__image = image;
 308                     });
 309                     break;
 310
 311                 case WI.Recording.Swizzle.ImageBitmap:
 312                     var image = await this.swizzle(index, WI.Recording.Swizzle.Image);
 313                     this._swizzle[index][type] = await createImageBitmap(image);
 314                     break;
 315
 316                 case WI.Recording.Swizzle.CallStack: {
 317                     let array = await this.swizzle(data, WI.Recording.Swizzle.Array);
 318                     this._swizzle[index][type] = await Promise.all(array.map((item) => this.swizzle(item, WI.Recording.Swizzle.CallFrame)));
 319                     break;
 320                 }
 321
 322                 case WI.Recording.Swizzle.CallFrame: {
 323                     let array = await this.swizzle(data, WI.Recording.Swizzle.Array);
 324                     let [functionName, url] = await Promise.all([
 325                         this.swizzle(array[0], WI.Recording.Swizzle.String),
 326                         this.swizzle(array[1], WI.Recording.Swizzle.String),
 327                     ]);
 328                     this._swizzle[index][type] = WI.CallFrame.fromPayload(WI.assumingMainTarget(), {
 329                         functionName,
 330                         url,
 331                         lineNumber: array[2],
 332                         columnNumber: array[3],
 333                     });
 334                     break;
 335                 }
 336                 }
 337             } catch { }
 338         }
 339
 340         return this._swizzle[index][type];
 341     }
 342
 343     createContext()
 344     {
 345         let createCanvasContext = (type) => {
 346             let canvas = document.createElement("canvas");
 347             if ("width" in this._initialState.attributes)
 348                 canvas.width = this._initialState.attributes.width;
 349             if ("height" in this._initialState.attributes)
 350                 canvas.height = this._initialState.attributes.height;
 351             return canvas.getContext(type, ...this._initialState.parameters);
 352         };
 353
 354         if (this._type === WI.Recording.Type.Canvas2D)
 355             return createCanvasContext("2d");
 356
 357         if (this._type === WI.Recording.Type.BitmapRenderer)
 358             return createCanvasContext("bitmaprenderer");
 359
 360         if (this._type === WI.Recording.Type.CanvasWebGL)
 361             return createCanvasContext("webgl");
 362
 363         console.error("Unknown recording type", this._type);
 364         return null;
 365     }
 366
 367     toJSON()
 368     {
 369         let initialState = {};
 370         if (!isEmptyObject(this._initialState.attributes))
 371             initialState.attributes = this._initialState.attributes;
 372         if (this._initialState.states.length)
 373             initialState.states = this._initialState.states;
 374         if (this._initialState.parameters.length)
 375             initialState.parameters = this._initialState.parameters;
 376         if (this._initialState.content && this._initialState.content.length)
 377             initialState.content = this._initialState.content;
 378
 379         return {
 380             version: this._version,
 381             type: this._type,
 382             initialState,
 383             frames: this._frames.map((frame) => frame.toJSON()),
 384             data: this._data,
 385         };
 386     }
 387
 388     // Private
 389
 390     async _process()
 391     {
 392         if (!this._processContext) {
 393             this._processContext = this.createContext();
 394
 395             if (this._type === WI.Recording.Type.Canvas2D) {
 396                 let initialContent = await WI.ImageUtilities.promisifyLoad(this._initialState.content);
 397                 this._processContext.drawImage(initialContent, 0, 0);
 398
 399                 for (let initialState of this._initialState.states) {
 400                     let state = await WI.RecordingState.swizzleInitialState(this, initialState);
 401                     state.apply(this._type, this._processContext);
 402
 403                     // The last state represents the current state, which should not be saved.
 404                     if (initialState !== this._initialState.states.lastValue) {
 405                         this._processContext.save();
 406                         this._processStates.push(WI.RecordingState.fromContext(this._type, this._processContext));
 407                     }
 408                 }
 409             }
 410         }
 411
 412         // The first action is always a WI.RecordingInitialStateAction, which doesn't need to swizzle().
 413         // Since it is not associated with a WI.RecordingFrame, it has to manually process().
 414         if (!this._actions[0].ready) {
 415             this._actions[0].process(this, this._processContext, this._processStates);
 416             this.dispatchEventToListeners(WI.Recording.Event.ProcessedAction, {action: this._actions[0], index: 0});
 417         }
 418
 419         const workInterval = 10;
 420         let startTime = Date.now();
 421
 422         let cumulativeActionIndex = 0;
 423         let lastAction = this._actions[cumulativeActionIndex];
 424         for (let frameIndex = 0; frameIndex < this._frames.length; ++frameIndex) {
 425             let frame = this._frames[frameIndex];
 426
 427             if (frame.actions.lastValue.ready) {
 428                 cumulativeActionIndex += frame.actions.length;
 429                 lastAction = frame.actions.lastValue;
 430                 continue;
 431             }
 432
 433             for (let actionIndex = 0; actionIndex < frame.actions.length; ++actionIndex) {
 434                 ++cumulativeActionIndex;
 435
 436                 let action = frame.actions[actionIndex];
 437                 if (action.ready) {
 438                     lastAction = action;
 439                     continue;
 440                 }
 441
 442                 await action.swizzle(this);
 443
 444                 action.process(this, this._processContext, this._processStates, {lastAction});
 445
 446                 if (action.isVisual)
 447                     this._visualActionIndexes.push(cumulativeActionIndex);
 448
 449                 if (!actionIndex)
 450                     this.dispatchEventToListeners(WI.Recording.Event.StartProcessingFrame, {frame, index: frameIndex});
 451
 452                 this.dispatchEventToListeners(WI.Recording.Event.ProcessedAction, {action, index: cumulativeActionIndex});
 453
 454                 if (Date.now() - startTime > workInterval) {
 455                     await Promise.delay(); // yield
 456
 457                     startTime = Date.now();
 458                 }
 459
 460                 lastAction = action;
 461
 462                 if (!this._processing)
 463                     return;
 464             }
 465
 466             if (!this._processing)
 467                 return;
 468         }
 469
 470         this._processContext = null;
 471         this._processing = false;
 472     }
 473 };
 474
 475 WI.Recording.Event = {
 476     ProcessedAction: "recording-processed-action",
 477     StartProcessingFrame: "recording-start-processing-frame",
 478 };
 479
 480 WI.Recording._importedRecordingNameSet = new Set;
 481
 482 WI.Recording.CanvasRecordingNamesSymbol = Symbol("canvas-recording-names");
 483
 484 WI.Recording.Type = {
 485     Canvas2D: "canvas-2d",
 486     CanvasBitmapRenderer: "canvas-bitmaprenderer",
 487     CanvasWebGL: "canvas-webgl",
 488 };
 489
 490 // Keep this in sync with WebCore::RecordingSwizzleTypes.
 491 WI.Recording.Swizzle = {
 492     None: 0,
 493     Number: 1,
 494     Boolean: 2,
 495     String: 3,
 496     Array: 4,
 497     TypedArray: 5,
 498     Image: 6,
 499     ImageData: 7,
 500     DOMMatrix: 8,
 501     Path2D: 9,
 502     CanvasGradient: 10,
 503     CanvasPattern: 11,
 504     WebGLBuffer: 12,
 505     WebGLFramebuffer: 13,
 506     WebGLRenderbuffer: 14,
 507     WebGLTexture: 15,
 508     WebGLShader: 16,
 509     WebGLProgram: 17,
 510     WebGLUniformLocation: 18,
 511     ImageBitmap: 19,
 512
 513     // Special frontend-only swizzle types.
 514     CallStack: Symbol("CallStack"),
 515     CallFrame: Symbol("CallFrame"),
 516 };