Web Inspector: Canvas: send a call stack with each action instead of an array of...
[WebKit-https.git] / Source / WebInspectorUI / UserInterface / Models / Recording.js
1 /*
2  * Copyright (C) 2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 WI.Recording = class Recording extends WI.Object
27 {
28     constructor(version, type, initialState, frames, data)
29     {
30         super();
31
32         this._version = version;
33         this._type = type;
34         this._initialState = initialState;
35         this._frames = frames;
36         this._data = data;
37         this._displayName = WI.UIString("Recording");
38
39         this._swizzle = [];
40         this._actions = [new WI.RecordingInitialStateAction].concat(...this._frames.map((frame) => frame.actions));
41         this._visualActionIndexes = [];
42         this._source = null;
43
44         this._processContext = null;
45         this._processStates = [];
46         this._processing = false;
47     }
48
49     static fromPayload(payload, frames)
50     {
51         if (typeof payload !== "object" || payload === null)
52             return null;
53
54         if (isNaN(payload.version) || payload.version <= 0)
55             return null;
56
57         let type = null;
58         switch (payload.type) {
59         case RecordingAgent.Type.Canvas2D:
60             type = WI.Recording.Type.Canvas2D;
61             break;
62         case RecordingAgent.Type.CanvasBitmapRenderer:
63             type = WI.Recording.Type.CanvasBitmapRenderer;
64             break;
65         case RecordingAgent.Type.CanvasWebGL:
66             type = WI.Recording.Type.CanvasWebGL;
67             break;
68         default:
69             type = String(payload.type);
70             break;
71         }
72
73         if (typeof payload.initialState !== "object" || payload.initialState === null)
74             payload.initialState = {};
75         if (typeof payload.initialState.attributes !== "object" || payload.initialState.attributes === null)
76             payload.initialState.attributes = {};
77         if (!Array.isArray(payload.initialState.states) || payload.initialState.states.some((item) => typeof item !== "object" || item === null)) {
78             payload.initialState.states = [];
79
80             // COMPATIBILITY (iOS 12.0): Recording.InitialState.states did not exist yet
81             if (!isEmptyObject(payload.initialState.attributes)) {
82                 let {width, height, ...state} = payload.initialState.attributes;
83                 if (!isEmptyObject(state))
84                     payload.initialState.states.push(state);
85             }
86         }
87         if (!Array.isArray(payload.initialState.parameters))
88             payload.initialState.parameters = [];
89         if (typeof payload.initialState.content !== "string")
90             payload.initialState.content = "";
91
92         if (!Array.isArray(payload.frames))
93             payload.frames = [];
94
95         if (!Array.isArray(payload.data))
96             payload.data = [];
97
98         if (!frames)
99             frames = payload.frames.map(WI.RecordingFrame.fromPayload)
100
101         return new WI.Recording(payload.version, type, payload.initialState, frames, payload.data);
102     }
103
104     static displayNameForSwizzleType(swizzleType)
105     {
106         switch (swizzleType) {
107         case WI.Recording.Swizzle.None:
108             return WI.unlocalizedString("None");
109         case WI.Recording.Swizzle.Number:
110             return WI.unlocalizedString("Number");
111         case WI.Recording.Swizzle.Boolean:
112             return WI.unlocalizedString("Boolean");
113         case WI.Recording.Swizzle.String:
114             return WI.unlocalizedString("String");
115         case WI.Recording.Swizzle.Array:
116             return WI.unlocalizedString("Array");
117         case WI.Recording.Swizzle.TypedArray:
118             return WI.unlocalizedString("TypedArray");
119         case WI.Recording.Swizzle.Image:
120             return WI.unlocalizedString("Image");
121         case WI.Recording.Swizzle.ImageData:
122             return WI.unlocalizedString("ImageData");
123         case WI.Recording.Swizzle.DOMMatrix:
124             return WI.unlocalizedString("DOMMatrix");
125         case WI.Recording.Swizzle.Path2D:
126             return WI.unlocalizedString("Path2D");
127         case WI.Recording.Swizzle.CanvasGradient:
128             return WI.unlocalizedString("CanvasGradient");
129         case WI.Recording.Swizzle.CanvasPattern:
130             return WI.unlocalizedString("CanvasPattern");
131         case WI.Recording.Swizzle.WebGLBuffer:
132             return WI.unlocalizedString("WebGLBuffer");
133         case WI.Recording.Swizzle.WebGLFramebuffer:
134             return WI.unlocalizedString("WebGLFramebuffer");
135         case WI.Recording.Swizzle.WebGLRenderbuffer:
136             return WI.unlocalizedString("WebGLRenderbuffer");
137         case WI.Recording.Swizzle.WebGLTexture:
138             return WI.unlocalizedString("WebGLTexture");
139         case WI.Recording.Swizzle.WebGLShader:
140             return WI.unlocalizedString("WebGLShader");
141         case WI.Recording.Swizzle.WebGLProgram:
142             return WI.unlocalizedString("WebGLProgram");
143         case WI.Recording.Swizzle.WebGLUniformLocation:
144             return WI.unlocalizedString("WebGLUniformLocation");
145         case WI.Recording.Swizzle.ImageBitmap:
146             return WI.unlocalizedString("ImageBitmap");
147         default:
148             console.error("Unknown swizzle type", swizzleType);
149             return null;
150         }
151     }
152
153     static synthesizeError(message)
154     {
155         const target = WI.mainTarget;
156         const source = WI.ConsoleMessage.MessageSource.Other;
157         const level = WI.ConsoleMessage.MessageLevel.Error;
158         let consoleMessage = new WI.ConsoleMessage(target, source, level, WI.UIString("Recording error: %s").format(message));
159         consoleMessage.shouldRevealConsole = true;
160
161         WI.consoleLogViewController.appendConsoleMessage(consoleMessage);
162     }
163
164     // Public
165
166     get displayName() { return this._displayName; }
167     get type() { return this._type; }
168     get initialState() { return this._initialState; }
169     get frames() { return this._frames; }
170     get data() { return this._data; }
171     get actions() { return this._actions; }
172     get visualActionIndexes() { return this._visualActionIndexes; }
173
174     get source() { return this._source; }
175     set source(source) { this._source = source; }
176
177     get processing() { return this._processing; }
178
179     get ready()
180     {
181         return this._actions.lastValue.ready;
182     }
183
184     startProcessing()
185     {
186         console.assert(!this._processing, "Cannot start an already started process().");
187         console.assert(!this.ready, "Cannot start a completed process().");
188         if (this._processing || this.ready)
189             return;
190
191         this._processing = true;
192
193         this._process();
194     }
195
196     stopProcessing()
197     {
198         console.assert(this._processing, "Cannot stop an already stopped process().");
199         console.assert(!this.ready, "Cannot stop a completed process().");
200         if (!this._processing || this.ready)
201             return;
202
203         this._processing = false;
204     }
205
206     createDisplayName(suggestedName)
207     {
208         let recordingNameSet;
209         if (this._source) {
210             recordingNameSet = this._source[WI.Recording.CanvasRecordingNamesSymbol];
211             if (!recordingNameSet)
212                 this._source[WI.Recording.CanvasRecordingNamesSymbol] = recordingNameSet = new Set;
213         } else
214             recordingNameSet = WI.Recording._importedRecordingNameSet;
215
216         let name;
217         if (suggestedName) {
218             name = suggestedName;
219             let duplicateNumber = 2;
220             while (recordingNameSet.has(name))
221                 name = `${suggestedName} (${duplicateNumber++})`;
222         } else {
223             let recordingNumber = 1;
224             do {
225                 name = WI.UIString("Recording %d").format(recordingNumber++);
226             } while (recordingNameSet.has(name));
227         }
228
229         recordingNameSet.add(name);
230         this._displayName = name;
231     }
232
233     async swizzle(index, type)
234     {
235         if (typeof this._swizzle[index] !== "object")
236             this._swizzle[index] = {};
237
238         if (type === WI.Recording.Swizzle.Number)
239             return parseFloat(index);
240
241         if (type === WI.Recording.Swizzle.Boolean)
242             return !!index;
243
244         if (type === WI.Recording.Swizzle.Array)
245             return Array.isArray(index) ? index : [];
246
247         if (type === WI.Recording.Swizzle.DOMMatrix)
248             return new DOMMatrix(index);
249
250         // FIXME: <https://webkit.org/b/176009> Web Inspector: send data for WebGL objects during a recording instead of a placeholder string
251         if (type === WI.Recording.Swizzle.TypedArray
252             || type === WI.Recording.Swizzle.WebGLBuffer
253             || type === WI.Recording.Swizzle.WebGLFramebuffer
254             || type === WI.Recording.Swizzle.WebGLRenderbuffer
255             || type === WI.Recording.Swizzle.WebGLTexture
256             || type === WI.Recording.Swizzle.WebGLShader
257             || type === WI.Recording.Swizzle.WebGLProgram
258             || type === WI.Recording.Swizzle.WebGLUniformLocation) {
259             return index;
260         }
261
262         if (!(type in this._swizzle[index])) {
263             try {
264                 let data = this._data[index];
265                 switch (type) {
266                 case WI.Recording.Swizzle.None:
267                     this._swizzle[index][type] = data;
268                     break;
269
270                 case WI.Recording.Swizzle.String:
271                     this._swizzle[index][type] = String(data);
272                     break;
273
274                 case WI.Recording.Swizzle.Image:
275                     this._swizzle[index][type] = await WI.ImageUtilities.promisifyLoad(data);
276                     break;
277
278                 case WI.Recording.Swizzle.ImageData:
279                     this._swizzle[index][type] = new ImageData(new Uint8ClampedArray(data[0]), parseInt(data[1]), parseInt(data[2]));
280                     break;
281
282                 case WI.Recording.Swizzle.Path2D:
283                     this._swizzle[index][type] = new Path2D(data);
284                     break;
285
286                 case WI.Recording.Swizzle.CanvasGradient:
287                     var gradientType = await this.swizzle(data[0], WI.Recording.Swizzle.String);
288
289                     WI.ImageUtilities.scratchCanvasContext2D((context) => {
290                         this._swizzle[index][type] = gradientType === "radial-gradient" ? context.createRadialGradient(...data[1]) : context.createLinearGradient(...data[1]);
291                     });
292
293                     for (let stop of data[2]) {
294                         let color = await this.swizzle(stop[1], WI.Recording.Swizzle.String);
295                         this._swizzle[index][type].addColorStop(stop[0], color);
296                     }
297                     break;
298
299                 case WI.Recording.Swizzle.CanvasPattern:
300                     var [image, repeat] = await Promise.all([
301                         this.swizzle(data[0], WI.Recording.Swizzle.Image),
302                         this.swizzle(data[1], WI.Recording.Swizzle.String),
303                     ]);
304
305                     WI.ImageUtilities.scratchCanvasContext2D((context) => {
306                         this._swizzle[index][type] = context.createPattern(image, repeat);
307                         this._swizzle[index][type].__image = image;
308                     });
309                     break;
310
311                 case WI.Recording.Swizzle.ImageBitmap:
312                     var image = await this.swizzle(index, WI.Recording.Swizzle.Image);
313                     this._swizzle[index][type] = await createImageBitmap(image);
314                     break;
315
316                 case WI.Recording.Swizzle.CallStack: {
317                     let array = await this.swizzle(data, WI.Recording.Swizzle.Array);
318                     this._swizzle[index][type] = await Promise.all(array.map((item) => this.swizzle(item, WI.Recording.Swizzle.CallFrame)));
319                     break;
320                 }
321
322                 case WI.Recording.Swizzle.CallFrame: {
323                     let array = await this.swizzle(data, WI.Recording.Swizzle.Array);
324                     let [functionName, url] = await Promise.all([
325                         this.swizzle(array[0], WI.Recording.Swizzle.String),
326                         this.swizzle(array[1], WI.Recording.Swizzle.String),
327                     ]);
328                     this._swizzle[index][type] = WI.CallFrame.fromPayload(WI.assumingMainTarget(), {
329                         functionName,
330                         url,
331                         lineNumber: array[2],
332                         columnNumber: array[3],
333                     });
334                     break;
335                 }
336                 }
337             } catch { }
338         }
339
340         return this._swizzle[index][type];
341     }
342
343     createContext()
344     {
345         let createCanvasContext = (type) => {
346             let canvas = document.createElement("canvas");
347             if ("width" in this._initialState.attributes)
348                 canvas.width = this._initialState.attributes.width;
349             if ("height" in this._initialState.attributes)
350                 canvas.height = this._initialState.attributes.height;
351             return canvas.getContext(type, ...this._initialState.parameters);
352         };
353
354         if (this._type === WI.Recording.Type.Canvas2D)
355             return createCanvasContext("2d");
356
357         if (this._type === WI.Recording.Type.BitmapRenderer)
358             return createCanvasContext("bitmaprenderer");
359
360         if (this._type === WI.Recording.Type.CanvasWebGL)
361             return createCanvasContext("webgl");
362
363         console.error("Unknown recording type", this._type);
364         return null;
365     }
366
367     toJSON()
368     {
369         let initialState = {};
370         if (!isEmptyObject(this._initialState.attributes))
371             initialState.attributes = this._initialState.attributes;
372         if (this._initialState.states.length)
373             initialState.states = this._initialState.states;
374         if (this._initialState.parameters.length)
375             initialState.parameters = this._initialState.parameters;
376         if (this._initialState.content && this._initialState.content.length)
377             initialState.content = this._initialState.content;
378
379         return {
380             version: this._version,
381             type: this._type,
382             initialState,
383             frames: this._frames.map((frame) => frame.toJSON()),
384             data: this._data,
385         };
386     }
387
388     // Private
389
390     async _process()
391     {
392         if (!this._processContext) {
393             this._processContext = this.createContext();
394
395             if (this._type === WI.Recording.Type.Canvas2D) {
396                 let initialContent = await WI.ImageUtilities.promisifyLoad(this._initialState.content);
397                 this._processContext.drawImage(initialContent, 0, 0);
398
399                 for (let initialState of this._initialState.states) {
400                     let state = await WI.RecordingState.swizzleInitialState(this, initialState);
401                     state.apply(this._type, this._processContext);
402
403                     // The last state represents the current state, which should not be saved.
404                     if (initialState !== this._initialState.states.lastValue) {
405                         this._processContext.save();
406                         this._processStates.push(WI.RecordingState.fromContext(this._type, this._processContext));
407                     }
408                 }
409             }
410         }
411
412         // The first action is always a WI.RecordingInitialStateAction, which doesn't need to swizzle().
413         // Since it is not associated with a WI.RecordingFrame, it has to manually process().
414         if (!this._actions[0].ready) {
415             this._actions[0].process(this, this._processContext, this._processStates);
416             this.dispatchEventToListeners(WI.Recording.Event.ProcessedAction, {action: this._actions[0], index: 0});
417         }
418
419         const workInterval = 10;
420         let startTime = Date.now();
421
422         let cumulativeActionIndex = 0;
423         let lastAction = this._actions[cumulativeActionIndex];
424         for (let frameIndex = 0; frameIndex < this._frames.length; ++frameIndex) {
425             let frame = this._frames[frameIndex];
426
427             if (frame.actions.lastValue.ready) {
428                 cumulativeActionIndex += frame.actions.length;
429                 lastAction = frame.actions.lastValue;
430                 continue;
431             }
432
433             for (let actionIndex = 0; actionIndex < frame.actions.length; ++actionIndex) {
434                 ++cumulativeActionIndex;
435
436                 let action = frame.actions[actionIndex];
437                 if (action.ready) {
438                     lastAction = action;
439                     continue;
440                 }
441
442                 await action.swizzle(this);
443
444                 action.process(this, this._processContext, this._processStates, {lastAction});
445
446                 if (action.isVisual)
447                     this._visualActionIndexes.push(cumulativeActionIndex);
448
449                 if (!actionIndex)
450                     this.dispatchEventToListeners(WI.Recording.Event.StartProcessingFrame, {frame, index: frameIndex});
451
452                 this.dispatchEventToListeners(WI.Recording.Event.ProcessedAction, {action, index: cumulativeActionIndex});
453
454                 if (Date.now() - startTime > workInterval) {
455                     await Promise.delay(); // yield
456
457                     startTime = Date.now();
458                 }
459
460                 lastAction = action;
461
462                 if (!this._processing)
463                     return;
464             }
465
466             if (!this._processing)
467                 return;
468         }
469
470         this._processContext = null;
471         this._processing = false;
472     }
473 };
474
475 WI.Recording.Event = {
476     ProcessedAction: "recording-processed-action",
477     StartProcessingFrame: "recording-start-processing-frame",
478 };
479
480 WI.Recording._importedRecordingNameSet = new Set;
481
482 WI.Recording.CanvasRecordingNamesSymbol = Symbol("canvas-recording-names");
483
484 WI.Recording.Type = {
485     Canvas2D: "canvas-2d",
486     CanvasBitmapRenderer: "canvas-bitmaprenderer",
487     CanvasWebGL: "canvas-webgl",
488 };
489
490 // Keep this in sync with WebCore::RecordingSwizzleTypes.
491 WI.Recording.Swizzle = {
492     None: 0,
493     Number: 1,
494     Boolean: 2,
495     String: 3,
496     Array: 4,
497     TypedArray: 5,
498     Image: 6,
499     ImageData: 7,
500     DOMMatrix: 8,
501     Path2D: 9,
502     CanvasGradient: 10,
503     CanvasPattern: 11,
504     WebGLBuffer: 12,
505     WebGLFramebuffer: 13,
506     WebGLRenderbuffer: 14,
507     WebGLTexture: 15,
508     WebGLShader: 16,
509     WebGLProgram: 17,
510     WebGLUniformLocation: 18,
511     ImageBitmap: 19,
512
513     // Special frontend-only swizzle types.
514     CallStack: Symbol("CallStack"),
515     CallFrame: Symbol("CallFrame"),
516 };