Some animated image do not animate after reseting their animations
[WebKit-https.git] / Source / WebCore / platform / graphics / ImageFrameCache.cpp
1 /*
2  * Copyright (C) 2016 Apple Inc.  All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ImageFrameCache.h"
28
29 #include "Image.h"
30 #include "ImageObserver.h"
31
32 #if USE(CG)
33 #include "ImageDecoderCG.h"
34 #elif USE(DIRECT2D)
35 #include "ImageDecoderDirect2D.h"
36 #include <WinCodec.h>
37 #else
38 #include "ImageDecoder.h"
39 #endif
40
41 #include <wtf/CheckedArithmetic.h>
42 #include <wtf/MainThread.h>
43 #include <wtf/RunLoop.h>
44
45 namespace WebCore {
46
47 ImageFrameCache::ImageFrameCache(Image* image)
48     : m_image(image)
49 {
50 }
51
52 ImageFrameCache::ImageFrameCache(NativeImagePtr&& nativeImage)
53 {
54     m_frameCount = 1;
55     m_isSizeAvailable = true;
56     growFrames();
57
58     setNativeImage(WTFMove(nativeImage));
59
60     m_decodedSize = m_frames[0].frameBytes();
61
62     // The assumption is the memory image will be displayed with the default
63     // orientation. So set m_sizeRespectingOrientation to be the same as m_size.
64     m_size = m_frames[0].size();
65     m_sizeRespectingOrientation = m_size;
66 }
67
68 ImageFrameCache::~ImageFrameCache()
69 {
70     ASSERT(!hasDecodingQueue());
71 }
72
73 void ImageFrameCache::destroyDecodedData(bool destroyAll, size_t count)
74 {
75     if (destroyAll)
76         count = m_frames.size();
77     
78     unsigned decodedSize = 0;
79     for (size_t i = 0; i <  count; ++i)
80         decodedSize += m_frames[i].clearImage();
81
82     decodedSizeReset(decodedSize);
83 }
84
85 bool ImageFrameCache::destroyDecodedDataIfNecessary(bool destroyAll, size_t count)
86 {
87     unsigned decodedSize = 0;
88     for (auto& frame : m_frames)
89         decodedSize += frame.frameBytes();
90     
91     if (decodedSize < LargeAnimationCutoff)
92         return false;
93     
94     destroyDecodedData(destroyAll, count);
95     return true;
96 }
97
98 void ImageFrameCache::destroyIncompleteDecodedData()
99 {
100     unsigned decodedSize = 0;
101     
102     for (auto& frame : m_frames) {
103         if (!frame.hasMetadata() || frame.isComplete())
104             continue;
105         
106         decodedSize += frame.clear();
107     }
108
109     decodedSizeDecreased(decodedSize);
110 }
111
112 void ImageFrameCache::decodedSizeChanged(long long decodedSize)
113 {
114     if (!decodedSize || !m_image || !m_image->imageObserver())
115         return;
116     
117     m_image->imageObserver()->decodedSizeChanged(m_image, decodedSize);
118 }
119
120 void ImageFrameCache::decodedSizeIncreased(unsigned decodedSize)
121 {
122     if (!decodedSize)
123         return;
124     
125     m_decodedSize += decodedSize;
126     
127     // The fully-decoded frame will subsume the partially decoded data used
128     // to determine image properties.
129     long long changeSize = static_cast<long long>(decodedSize) - m_decodedPropertiesSize;
130     m_decodedPropertiesSize = 0;
131     decodedSizeChanged(changeSize);
132 }
133
134 void ImageFrameCache::decodedSizeDecreased(unsigned decodedSize)
135 {
136     if (!decodedSize)
137         return;
138
139     ASSERT(m_decodedSize >= decodedSize);
140     m_decodedSize -= decodedSize;
141     decodedSizeChanged(-static_cast<long long>(decodedSize));
142 }
143
144 void ImageFrameCache::decodedSizeReset(unsigned decodedSize)
145 {
146     ASSERT(m_decodedSize >= decodedSize);
147     m_decodedSize -= decodedSize;
148
149     // Clearing the ImageSource destroys the extra decoded data used for
150     // determining image properties.
151     decodedSize += m_decodedPropertiesSize;
152     m_decodedPropertiesSize = 0;
153     decodedSizeChanged(-static_cast<long long>(decodedSize));
154 }
155
156 void ImageFrameCache::didDecodeProperties(unsigned decodedPropertiesSize)
157 {
158     if (m_decodedSize)
159         return;
160
161     long long decodedSize = static_cast<long long>(decodedPropertiesSize) - m_decodedPropertiesSize;
162     m_decodedPropertiesSize = decodedPropertiesSize;
163     decodedSizeChanged(decodedSize);
164 }
165
166 void ImageFrameCache::growFrames()
167 {
168     ASSERT(isSizeAvailable());
169     ASSERT(m_frames.size() <= frameCount());
170     m_frames.grow(frameCount());
171 }
172
173 void ImageFrameCache::setNativeImage(NativeImagePtr&& nativeImage)
174 {
175     ASSERT(m_frames.size() == 1);
176     ImageFrame& frame = m_frames[0];
177
178     ASSERT(!isDecoderAvailable());
179
180     frame.m_nativeImage = WTFMove(nativeImage);
181
182     frame.m_decoding = ImageFrame::Decoding::Complete;
183     frame.m_size = nativeImageSize(frame.m_nativeImage);
184     frame.m_hasAlpha = nativeImageHasAlpha(frame.m_nativeImage);
185 }
186
187 void ImageFrameCache::setFrameNativeImageAtIndex(NativeImagePtr&& nativeImage, size_t index, SubsamplingLevel subsamplingLevel)
188 {
189     ASSERT(index < m_frames.size());
190     ImageFrame& frame = m_frames[index];
191
192     ASSERT(isDecoderAvailable());
193
194     frame.m_nativeImage = WTFMove(nativeImage);
195     setFrameMetadataAtIndex(index, subsamplingLevel);
196 }
197
198 void ImageFrameCache::setFrameMetadataAtIndex(size_t index, SubsamplingLevel subsamplingLevel)
199 {
200     ASSERT(index < m_frames.size());
201     ImageFrame& frame = m_frames[index];
202
203     ASSERT(isDecoderAvailable());
204     frame.m_decoding = m_decoder->frameIsCompleteAtIndex(index) ? ImageFrame::Decoding::Complete : ImageFrame::Decoding::Partial;
205     if (frame.hasMetadata())
206         return;
207     
208     frame.m_subsamplingLevel = subsamplingLevel;
209     frame.m_size = m_decoder->frameSizeAtIndex(index, subsamplingLevel);
210     frame.m_orientation = m_decoder->frameOrientationAtIndex(index);
211     frame.m_hasAlpha = m_decoder->frameHasAlphaAtIndex(index);
212     
213     if (repetitionCount())
214         frame.m_duration = m_decoder->frameDurationAtIndex(index);
215 }
216
217 void ImageFrameCache::replaceFrameNativeImageAtIndex(NativeImagePtr&& nativeImage, size_t index, SubsamplingLevel subsamplingLevel)
218 {
219     ASSERT(index < m_frames.size());
220     ImageFrame& frame = m_frames[index];
221
222     if (!frame.hasValidNativeImage(subsamplingLevel)) {
223         // Clear the current image frame and update the observer with this clearance.
224         unsigned decodedSize = frame.clear();
225         decodedSizeDecreased(decodedSize);
226     }
227
228     // Do not cache the NativeImage if adding its frameByes to the MemoryCache will cause numerical overflow.
229     size_t frameBytes = size().unclampedArea() * sizeof(RGBA32);
230     if (!WTF::isInBounds<unsigned>(frameBytes + decodedSize()))
231         return;
232
233     // Copy the new image to the cache.
234     setFrameNativeImageAtIndex(WTFMove(nativeImage), index, subsamplingLevel);
235
236     // Update the observer with the new image frame bytes.
237     decodedSizeIncreased(frame.frameBytes());
238 }
239
240 void ImageFrameCache::cacheFrameNativeImageAtIndex(NativeImagePtr&& nativeImage, size_t index, SubsamplingLevel subsamplingLevel)
241 {
242     if (!isDecoderAvailable())
243         return;
244
245     ASSERT(index < m_frames.size());
246     ASSERT(m_frames[index].isBeingDecoded());
247
248     // Clean the old native image and set a new one
249     replaceFrameNativeImageAtIndex(WTFMove(nativeImage), index, subsamplingLevel);
250
251     // Notify the image with the readiness of the new frame NativeImage.
252     if (m_image)
253         m_image->newFrameNativeImageAvailableAtIndex(index);
254 }
255
256 Ref<WorkQueue> ImageFrameCache::decodingQueue()
257 {
258     if (!m_decodingQueue)
259         m_decodingQueue = WorkQueue::create("org.webkit.ImageDecoder", WorkQueue::Type::Serial, WorkQueue::QOS::UserInteractive);
260     
261     return *m_decodingQueue;
262 }
263
264 void ImageFrameCache::startAsyncDecodingQueue()
265 {
266     if (hasDecodingQueue() || !isDecoderAvailable())
267         return;
268
269     m_frameRequestQueue.open();
270
271     Ref<ImageFrameCache> protectedThis = Ref<ImageFrameCache>(*this);
272     Ref<WorkQueue> protectedQueue = decodingQueue();
273
274     // We need to protect this and m_decodingQueue from being deleted while we are in the decoding loop.
275     decodingQueue()->dispatch([this, protectedThis = WTFMove(protectedThis), protectedQueue = WTFMove(protectedQueue)] {
276         ImageFrameRequest frameRequest;
277
278         while (m_frameRequestQueue.dequeue(frameRequest)) {
279             // Get the frame NativeImage on the decoding thread.
280             NativeImagePtr nativeImage = m_decoder->createFrameImageAtIndex(frameRequest.index, frameRequest.subsamplingLevel, DecodingMode::Immediate);
281
282             // Update the cached frames on the main thread to avoid updating the MemoryCache from a different thread.
283             callOnMainThread([this, protectedQueue = protectedQueue.copyRef(), nativeImage, frameRequest] () mutable {
284                 // The queue may be closed if after we got the frame NativeImage, stopAsyncDecodingQueue() was called
285                 if (protectedQueue.ptr() == m_decodingQueue)
286                     cacheFrameNativeImageAtIndex(WTFMove(nativeImage), frameRequest.index, frameRequest.subsamplingLevel);
287             });
288         }
289     });
290 }
291
292 bool ImageFrameCache::requestFrameAsyncDecodingAtIndex(size_t index, SubsamplingLevel subsamplingLevel)
293 {
294     if (!isDecoderAvailable())
295         return false;
296
297     if (!hasDecodingQueue())
298         startAsyncDecodingQueue();
299
300     ASSERT(index < m_frames.size());
301     ImageFrame& frame = m_frames[index];
302
303     // We need to coalesce multiple requests for decoding the same ImageFrame while it
304     // is still being decoded. This may happen if the image rectangle is repainted
305     // multiple times while the ImageFrame has not finished decoding.
306     if (frame.isBeingDecoded())
307         return true;
308
309     if (subsamplingLevel == SubsamplingLevel::Undefinded)
310         subsamplingLevel = frame.subsamplingLevel();
311
312     if (frame.hasValidNativeImage(subsamplingLevel))
313         return false;
314
315     frame.setDecoding(ImageFrame::Decoding::BeingDecoded);
316     m_frameRequestQueue.enqueue({ index, subsamplingLevel });
317     return true;
318 }
319
320 void ImageFrameCache::stopAsyncDecodingQueue()
321 {
322     if (!hasDecodingQueue())
323         return;
324     
325     m_frameRequestQueue.close();
326     m_decodingQueue = nullptr;
327
328     for (ImageFrame& frame : m_frames) {
329         if (frame.isBeingDecoded())
330             frame.clear();
331     }
332 }
333
334 const ImageFrame& ImageFrameCache::frameAtIndex(size_t index, SubsamplingLevel subsamplingLevel, ImageFrame::Caching caching)
335 {
336     ASSERT(index < m_frames.size());
337     ImageFrame& frame = m_frames[index];
338     if (!isDecoderAvailable() || frame.isBeingDecoded() || caching == ImageFrame::Caching::Empty)
339         return frame;
340     
341     if (subsamplingLevel == SubsamplingLevel::Undefinded)
342         subsamplingLevel = frame.subsamplingLevel();
343
344     if (!frame.isComplete() && caching == ImageFrame::Caching::Metadata)
345         setFrameMetadataAtIndex(index, subsamplingLevel);
346     else if (!frame.hasValidNativeImage(subsamplingLevel) && caching == ImageFrame::Caching::MetadataAndImage)
347         replaceFrameNativeImageAtIndex(m_decoder->createFrameImageAtIndex(index, subsamplingLevel), index, subsamplingLevel);
348
349     return frame;
350 }
351
352 void ImageFrameCache::clearMetadata()
353 {
354     m_frameCount = std::nullopt;
355     m_singlePixelSolidColor = std::nullopt;
356 }
357
358 template<typename T, T (ImageDecoder::*functor)() const>
359 T ImageFrameCache::metadata(const T& defaultValue, std::optional<T>* cachedValue)
360 {
361     if (cachedValue && *cachedValue)
362         return cachedValue->value();
363
364     if (!isDecoderAvailable() || !m_decoder->isSizeAvailable())
365         return defaultValue;
366
367     if (!cachedValue)
368         return (m_decoder->*functor)();
369
370     *cachedValue = (m_decoder->*functor)();
371     didDecodeProperties(m_decoder->bytesDecodedToDetermineProperties());
372     return cachedValue->value();
373 }
374
375 template<typename T, T (ImageFrame::*functor)() const>
376 T ImageFrameCache::frameMetadataAtIndex(size_t index, SubsamplingLevel subsamplingLevel, ImageFrame::Caching caching, std::optional<T>* cachedValue)
377 {
378     if (cachedValue && *cachedValue)
379         return cachedValue->value();
380     
381     const ImageFrame& frame = index < m_frames.size() ? frameAtIndex(index, subsamplingLevel, caching) : ImageFrame::defaultFrame();
382
383     // Don't cache any unavailable frame metadata.
384     if (!frame.hasMetadata() || !cachedValue)
385         return (frame.*functor)();
386     
387     *cachedValue = (frame.*functor)();
388     return cachedValue->value();
389 }
390
391 bool ImageFrameCache::isSizeAvailable()
392 {
393     if (m_isSizeAvailable)
394         return m_isSizeAvailable.value();
395     
396     if (!isDecoderAvailable() || !m_decoder->isSizeAvailable())
397         return false;
398     
399     m_isSizeAvailable = true;
400     didDecodeProperties(m_decoder->bytesDecodedToDetermineProperties());
401     return true;
402 }
403
404 size_t ImageFrameCache::frameCount()
405 {
406     return metadata<size_t, (&ImageDecoder::frameCount)>(m_frames.size(), &m_frameCount);
407 }
408
409 RepetitionCount ImageFrameCache::repetitionCount()
410 {
411     return metadata<RepetitionCount, (&ImageDecoder::repetitionCount)>(RepetitionCountNone, &m_repetitionCount);
412 }
413
414 String ImageFrameCache::filenameExtension()
415 {
416     return metadata<String, (&ImageDecoder::filenameExtension)>(String(), &m_filenameExtension);
417 }
418
419 std::optional<IntPoint> ImageFrameCache::hotSpot()
420 {
421     return metadata<std::optional<IntPoint>, (&ImageDecoder::hotSpot)>(std::nullopt, &m_hotSpot);
422 }
423
424 IntSize ImageFrameCache::size()
425 {
426     return frameMetadataAtIndex<IntSize, (&ImageFrame::size)>(0, SubsamplingLevel::Default, ImageFrame::Caching::Metadata, &m_size);
427 }
428
429 IntSize ImageFrameCache::sizeRespectingOrientation()
430 {
431     return frameMetadataAtIndex<IntSize, (&ImageFrame::sizeRespectingOrientation)>(0, SubsamplingLevel::Default, ImageFrame::Caching::Metadata, &m_sizeRespectingOrientation);
432 }
433
434 Color ImageFrameCache::singlePixelSolidColor()
435 {
436     return frameCount() == 1 ? frameMetadataAtIndex<Color, (&ImageFrame::singlePixelSolidColor)>(0, SubsamplingLevel::Undefinded, ImageFrame::Caching::MetadataAndImage, &m_singlePixelSolidColor) : Color();
437 }
438
439 bool ImageFrameCache::frameIsBeingDecodedAtIndex(size_t index)
440 {
441     return frameMetadataAtIndex<bool, (&ImageFrame::isBeingDecoded)>(index);
442 }
443
444 bool ImageFrameCache::frameIsCompleteAtIndex(size_t index)
445 {
446     return frameMetadataAtIndex<bool, (&ImageFrame::isComplete)>(index);
447 }
448
449 bool ImageFrameCache::frameHasAlphaAtIndex(size_t index)
450 {
451     return frameMetadataAtIndex<bool, (&ImageFrame::hasAlpha)>(index);
452 }
453
454 bool ImageFrameCache::frameHasImageAtIndex(size_t index)
455 {
456     return frameMetadataAtIndex<bool, (&ImageFrame::hasNativeImage)>(index);
457 }
458
459 bool ImageFrameCache::frameHasValidNativeImageAtIndex(size_t index, SubsamplingLevel subsamplingLevel)
460 {
461     return frameHasImageAtIndex(index) && subsamplingLevel >= frameSubsamplingLevelAtIndex(index);
462 }
463
464 SubsamplingLevel ImageFrameCache::frameSubsamplingLevelAtIndex(size_t index)
465 {
466     return frameMetadataAtIndex<SubsamplingLevel, (&ImageFrame::subsamplingLevel)>(index);
467 }
468
469 IntSize ImageFrameCache::frameSizeAtIndex(size_t index, SubsamplingLevel subsamplingLevel)
470 {
471     return frameMetadataAtIndex<IntSize, (&ImageFrame::size)>(index, subsamplingLevel, ImageFrame::Caching::Metadata);
472 }
473
474 unsigned ImageFrameCache::frameBytesAtIndex(size_t index, SubsamplingLevel subsamplingLevel)
475 {
476     return frameMetadataAtIndex<unsigned, (&ImageFrame::frameBytes)>(index, subsamplingLevel, ImageFrame::Caching::Metadata);
477 }
478
479 float ImageFrameCache::frameDurationAtIndex(size_t index)
480 {
481     return frameMetadataAtIndex<float, (&ImageFrame::duration)>(index, SubsamplingLevel::Undefinded, ImageFrame::Caching::Metadata);
482 }
483
484 ImageOrientation ImageFrameCache::frameOrientationAtIndex(size_t index)
485 {
486     return frameMetadataAtIndex<ImageOrientation, (&ImageFrame::orientation)>(index, SubsamplingLevel::Undefinded, ImageFrame::Caching::Metadata);
487 }
488
489 NativeImagePtr ImageFrameCache::frameImageAtIndex(size_t index, SubsamplingLevel subsamplingLevel)
490 {
491     return frameMetadataAtIndex<NativeImagePtr, (&ImageFrame::nativeImage)>(index, subsamplingLevel, ImageFrame::Caching::MetadataAndImage);
492 }
493
494 }