2 * Copyright (C) 2017 Apple Inc. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23 * THE POSSIBILITY OF SUCH DAMAGE.
27 #include "ImageBitmap.h"
29 #include "BitmapImage.h"
31 #include "CachedImage.h"
32 #include "ExceptionOr.h"
33 #include "FileReaderLoader.h"
34 #include "FileReaderLoaderClient.h"
35 #include "GraphicsContext.h"
36 #include "HTMLCanvasElement.h"
37 #include "HTMLImageElement.h"
38 #include "HTMLVideoElement.h"
39 #include "ImageBitmapOptions.h"
40 #include "ImageBuffer.h"
41 #include "ImageData.h"
43 #include "JSImageBitmap.h"
44 #include "LayoutSize.h"
45 #include "RenderElement.h"
46 #include "SharedBuffer.h"
47 #include <wtf/StdLibExtras.h>
51 #if USE(IOSURFACE_CANVAS_BACKING_STORE) || ENABLE(ACCELERATED_2D_CANVAS)
52 static RenderingMode bufferRenderingMode = Accelerated;
54 static RenderingMode bufferRenderingMode = Unaccelerated;
57 Ref<ImageBitmap> ImageBitmap::create(IntSize size)
59 auto imageBitmap = adoptRef(*new ImageBitmap);
60 imageBitmap->m_bitmapData = ImageBuffer::create(FloatSize(size.width(), size.height()), bufferRenderingMode);
64 Ref<ImageBitmap> ImageBitmap::create()
66 return adoptRef(*new ImageBitmap);
69 void ImageBitmap::createPromise(ScriptExecutionContext& scriptExecutionContext, ImageBitmap::Source&& source, ImageBitmapOptions&& options, ImageBitmap::Promise&& promise)
72 [&] (auto& specificSource) {
73 createPromise(scriptExecutionContext, specificSource, WTFMove(options), std::nullopt, WTFMove(promise));
78 void ImageBitmap::createPromise(ScriptExecutionContext& scriptExecutionContext, ImageBitmap::Source&& source, ImageBitmapOptions&& options, int sx, int sy, int sw, int sh, ImageBitmap::Promise&& promise)
80 // 1. If either the sw or sh arguments are specified but zero, return a promise
81 // rejected with an "RangeError" DOMException and abort these steps.
83 promise.reject(RangeError, "Cannot create ImageBitmap with a width or height of 0");
87 if (sw < 0 || sh < 0) {
88 promise.reject(RangeError, "Cannot create ImageBitmap with a negative width or height");
93 [&] (auto& specificSource) {
94 createPromise(scriptExecutionContext, specificSource, WTFMove(options), IntRect { sx, sy, sw, sh }, WTFMove(promise));
99 static bool taintsOrigin(CachedImage& cachedImage)
101 auto* image = cachedImage.image();
105 if (image->sourceURL().protocolIsData())
108 if (!image->hasSingleSecurityOrigin())
111 if (!cachedImage.isCORSSameOrigin())
117 // https://html.spec.whatwg.org/multipage/imagebitmap-and-animations.html#cropped-to-the-source-rectangle-with-formatting
118 static ExceptionOr<IntRect> croppedSourceRectangleWithFormatting(IntSize inputSize, ImageBitmapOptions& options, std::optional<IntRect> rect)
120 // 2. If either or both of resizeWidth and resizeHeight members of options are less
121 // than or equal to 0, then return a promise rejected with "InvalidStateError"
122 // DOMException and abort these steps.
123 if ((options.resizeWidth && options.resizeWidth.value() <= 0) || (options.resizeHeight && options.resizeHeight.value() <= 0))
124 return Exception { InvalidStateError, "Invalid resize dimensions" };
126 // 3. If sx, sy, sw and sh are specified, let sourceRectangle be a rectangle whose
127 // corners are the four points (sx, sy), (sx+sw, sy),(sx+sw, sy+sh), (sx,sy+sh).
128 // Otherwise let sourceRectangle be a rectangle whose corners are the four points
129 // (0,0), (width of input, 0), (width of input, height of input), (0, height of
131 auto sourceRectangle = rect.value_or(IntRect { 0, 0, inputSize.width(), inputSize.height() });
133 // 4. Clip sourceRectangle to the dimensions of input.
135 sourceRectangle.setWidth(std::min(sourceRectangle.width(), inputSize.width()));
136 sourceRectangle.setHeight(std::min(sourceRectangle.height(), inputSize.height()));
138 return { WTFMove(sourceRectangle) };
141 static IntSize outputSizeForSourceRectangle(IntRect sourceRectangle, ImageBitmapOptions& options)
143 // 5. Let outputWidth be determined as follows:
144 auto outputWidth = [&] () -> int {
145 if (options.resizeWidth)
146 return options.resizeWidth.value();
147 if (options.resizeHeight)
148 return ceil(sourceRectangle.width() * static_cast<double>(options.resizeHeight.value()) / sourceRectangle.height());
149 return sourceRectangle.width();
152 // 6. Let outputHeight be determined as follows:
153 auto outputHeight = [&] () -> int {
154 if (options.resizeHeight)
155 return options.resizeHeight.value();
156 if (options.resizeWidth)
157 return ceil(sourceRectangle.height() * static_cast<double>(options.resizeWidth.value()) / sourceRectangle.width());
158 return sourceRectangle.height();
161 return { outputWidth, outputHeight };
164 static InterpolationQuality interpolationQualityForResizeQuality(ImageBitmapOptions::ResizeQuality resizeQuality)
166 switch (resizeQuality) {
167 case ImageBitmapOptions::ResizeQuality::Pixelated:
168 return InterpolationNone;
169 case ImageBitmapOptions::ResizeQuality::Low:
170 return InterpolationDefault; // Low is the default.
171 case ImageBitmapOptions::ResizeQuality::Medium:
172 return InterpolationMedium;
173 case ImageBitmapOptions::ResizeQuality::High:
174 return InterpolationHigh;
176 ASSERT_NOT_REACHED();
177 return InterpolationDefault;
180 // FIXME: More steps from https://html.spec.whatwg.org/multipage/imagebitmap-and-animations.html#cropped-to-the-source-rectangle-with-formatting
182 // 7. Place input on an infinite transparent black grid plane, positioned so that its
183 // top left corner is at the origin of the plane, with the x-coordinate increasing
184 // to the right, and the y-coordinate increasing down, and with each pixel in the
185 // input image data occupying a cell on the plane's grid.
187 // 8. Let output be the rectangle on the plane denoted by sourceRectangle.
189 // 9. Scale output to the size specified by outputWidth and outputHeight. The user
190 // agent should use the value of the resizeQuality option to guide the choice of
191 // scaling algorithm.
193 // 10. If the value of the imageOrientation member of options is "flipY", output must
194 // be flipped vertically, disregarding any image orientation metadata of the source
195 // (such as EXIF metadata), if any.
197 // 11. If image is an img element or a Blob object, let val be the value of the
198 // colorSpaceConversion member of options, and then run these substeps:
200 // 1. If val is "default", the color space conversion behavior is implementation-specific,
201 // and should be chosen according to the color space that the implementation uses for
202 // drawing images onto the canvas.
204 // 2. If val is "none", output must be decoded without performing any color space
205 // conversions. This means that the image decoding algorithm must ignore color profile
206 // metadata embedded in the source data as well as the display device color profile.
208 // 12. Let val be the value of premultiplyAlpha member of options, and then run these substeps:
210 // 1. If val is "default", the alpha premultiplication behavior is implementation-specific,
211 // and should be chosen according to implementation deems optimal for drawing images
214 // 2. If val is "premultiply", the output that is not premultiplied by alpha must have its
215 // color components multiplied by alpha and that is premultiplied by alpha must be left
218 // 3. If val is "none", the output that is not premultiplied by alpha must be left untouched
219 // and that is premultiplied by alpha must have its color components divided by alpha.
221 // 13. Return output.
223 void ImageBitmap::createPromise(ScriptExecutionContext&, RefPtr<HTMLImageElement>& imageElement, ImageBitmapOptions&& options, std::optional<IntRect> rect, ImageBitmap::Promise&& promise)
225 // 2. If image is not completely available, then return a promise rejected with
226 // an "InvalidStateError" DOMException and abort these steps.
228 auto* cachedImage = imageElement->cachedImage();
229 if (!cachedImage || !imageElement->complete()) {
230 promise.reject(InvalidStateError, "Cannot create ImageBitmap that is not completely available");
234 // 3. If image's media data has no intrinsic dimensions (e.g. it's a vector graphic
235 // with no specified content size), and both or either of the resizeWidth and
236 // resizeHeight options are not specified, then return a promise rejected with
237 // an "InvalidStateError" DOMException and abort these steps.
239 auto imageSize = cachedImage->imageSizeForRenderer(imageElement->renderer(), 1.0f);
240 if ((!imageSize.width() || !imageSize.height()) && (!options.resizeWidth || !options.resizeHeight)) {
241 promise.reject(InvalidStateError, "Cannot create ImageBitmap from a source with no intrinsic size without providing resize dimensions");
245 // 4. If image's media data has no intrinsic dimensions (e.g. it's a vector graphics
246 // with no specified content size), it should be rendered to a bitmap of the size
247 // specified by the resizeWidth and the resizeHeight options.
249 if (!imageSize.width() && !imageSize.height()) {
250 imageSize.setWidth(options.resizeWidth.value());
251 imageSize.setHeight(options.resizeHeight.value());
254 // 5. If the sw and sh arguments are not specified and image's media data has both or
255 // either of its intrinsic width and intrinsic height values equal to 0, then return
256 // a promise rejected with an "InvalidStateError" DOMException and abort these steps.
257 // 6. If the sh argument is not specified and image's media data has an intrinsic height
258 // of 0, then return a promise rejected with an "InvalidStateError" DOMException and
259 // abort these steps.
261 // FIXME: It's unclear how these steps can happen, since step 4 required setting a
262 // width and height for the image.
264 if (!rect && (!imageSize.width() || !imageSize.height())) {
265 promise.reject(InvalidStateError, "Cannot create ImageBitmap from a source with no intrinsic size without providing dimensions");
269 // 7. Create a new ImageBitmap object.
271 auto imageBitmap = create();
273 // 8. Let the ImageBitmap object's bitmap data be a copy of image's media data, cropped to
274 // the source rectangle with formatting. If this is an animated image, the ImageBitmap
275 // object's bitmap data must only be taken from the default image of the animation (the
276 // one that the format defines is to be used when animation is not supported or is disabled),
277 // or, if there is no such image, the first frame of the animation.
279 auto sourceRectangle = croppedSourceRectangleWithFormatting(roundedIntSize(imageSize), options, WTFMove(rect));
280 if (sourceRectangle.hasException()) {
281 promise.reject(sourceRectangle.releaseException());
285 auto outputSize = outputSizeForSourceRectangle(sourceRectangle.returnValue(), options);
286 auto bitmapData = ImageBuffer::create(FloatSize(outputSize.width(), outputSize.height()), bufferRenderingMode);
288 auto imageForRender = cachedImage->imageForRenderer(imageElement->renderer());
289 if (!imageForRender) {
290 promise.reject(InvalidStateError, "Cannot create ImageBitmap from image that can't be rendered");
294 FloatRect destRect(FloatPoint(), outputSize);
295 ImagePaintingOptions paintingOptions;
296 paintingOptions.m_interpolationQuality = interpolationQualityForResizeQuality(options.resizeQuality);
298 bitmapData->context().drawImage(*imageForRender, destRect, sourceRectangle.releaseReturnValue(), paintingOptions);
300 imageBitmap->m_bitmapData = WTFMove(bitmapData);
302 // 9. If the origin of image's image is not the same origin as the origin specified by the
303 // entry settings object, then set the origin-clean flag of the ImageBitmap object's
306 imageBitmap->m_originClean = !taintsOrigin(*cachedImage);
308 // 10. Return a new promise, but continue running these steps in parallel.
309 // 11. Resolve the promise with the new ImageBitmap object as the value.
311 promise.resolve(WTFMove(imageBitmap));
314 void ImageBitmap::createPromise(ScriptExecutionContext&, RefPtr<HTMLCanvasElement>& canvasElement, ImageBitmapOptions&& options, std::optional<IntRect> rect, ImageBitmap::Promise&& promise)
316 // 2. If the canvas element's bitmap has either a horizontal dimension or a vertical
317 // dimension equal to zero, then return a promise rejected with an "InvalidStateError"
318 // DOMException and abort these steps.
319 auto size = canvasElement->size();
320 if (!size.width() || !size.height()) {
321 promise.reject(InvalidStateError, "Cannot create ImageBitmap from a canvas that has zero width or height");
325 // 3. Create a new ImageBitmap object.
326 auto imageBitmap = create();
328 // 4. Let the ImageBitmap object's bitmap data be a copy of the canvas element's bitmap
329 // data, cropped to the source rectangle with formatting.
331 auto sourceRectangle = croppedSourceRectangleWithFormatting(size, options, WTFMove(rect));
332 if (sourceRectangle.hasException()) {
333 promise.reject(sourceRectangle.releaseException());
337 auto outputSize = outputSizeForSourceRectangle(sourceRectangle.returnValue(), options);
338 auto bitmapData = ImageBuffer::create(FloatSize(outputSize.width(), outputSize.height()), bufferRenderingMode);
340 auto imageForRender = canvasElement->copiedImage();
341 if (!imageForRender) {
342 promise.reject(InvalidStateError, "Cannot create ImageBitmap from canvas that can't be rendered");
346 FloatRect destRect(FloatPoint(), outputSize);
347 ImagePaintingOptions paintingOptions;
348 paintingOptions.m_interpolationQuality = interpolationQualityForResizeQuality(options.resizeQuality);
350 bitmapData->context().drawImage(*imageForRender, destRect, sourceRectangle.releaseReturnValue(), paintingOptions);
352 imageBitmap->m_bitmapData = WTFMove(bitmapData);
354 // 5. Set the origin-clean flag of the ImageBitmap object's bitmap to the same value as
355 // the origin-clean flag of the canvas element's bitmap.
357 imageBitmap->m_originClean = canvasElement->originClean();
359 // 6. Return a new promise, but continue running these steps in parallel.
360 // 7. Resolve the promise with the new ImageBitmap object as the value.
362 promise.resolve(WTFMove(imageBitmap));
366 void ImageBitmap::createPromise(ScriptExecutionContext&, RefPtr<HTMLVideoElement>& videoElement, ImageBitmapOptions&& options, std::optional<IntRect> rect, ImageBitmap::Promise&& promise)
368 UNUSED_PARAM(videoElement);
369 UNUSED_PARAM(options);
372 // 2. If the video element's networkState attribute is NETWORK_EMPTY, then return
373 // a promise rejected with an "InvalidStateError" DOMException and abort these
376 // 3. If the video element's readyState attribute is either HAVE_NOTHING or
377 // HAVE_METADATA, then return a promise rejected with an "InvalidStateError"
378 // DOMException and abort these steps.
380 // 4. Create a new ImageBitmap object.
382 // 5. Let the ImageBitmap object's bitmap data be a copy of the frame at the current
383 // playback position, at the media resource's intrinsic width and intrinsic height
384 // (i.e. after any aspect-ratio correction has been applied), cropped to the source
385 // rectangle with formatting.
387 // 6. If the origin of the video element is not the same origin as the origin specified
388 // by the entry settings object, then set the origin-clean flag of the ImageBitmap
389 // object's bitmap to false.
391 // 7. Return a new promise, but continue running these steps in parallel.
393 // 8. Resolve the promise with the new ImageBitmap object as the value.
394 promise.reject(TypeError, "createImageBitmap with HTMLVideoElement is not implemented");
398 void ImageBitmap::createPromise(ScriptExecutionContext&, RefPtr<ImageBitmap>& existingImageBitmap, ImageBitmapOptions&& options, std::optional<IntRect> rect, ImageBitmap::Promise&& promise)
400 // 2. If image's [[Detached]] internal slot value is true, return a promise
401 // rejected with an "InvalidStateError" DOMException and abort these steps.
402 if (existingImageBitmap->isDetached() || !existingImageBitmap->buffer()) {
403 promise.reject(InvalidStateError, "Cannot create ImageBitmap from a detached ImageBitmap");
407 // 3. Create a new ImageBitmap object.
408 auto imageBitmap = create();
410 // 4. Let the ImageBitmap object's bitmap data be a copy of the image argument's
411 // bitmap data, cropped to the source rectangle with formatting.
412 auto sourceRectangle = croppedSourceRectangleWithFormatting(existingImageBitmap->buffer()->logicalSize(), options, WTFMove(rect));
413 if (sourceRectangle.hasException()) {
414 promise.reject(sourceRectangle.releaseException());
418 auto outputSize = outputSizeForSourceRectangle(sourceRectangle.returnValue(), options);
419 auto bitmapData = ImageBuffer::create(FloatSize(outputSize.width(), outputSize.height()), bufferRenderingMode);
421 auto imageForRender = existingImageBitmap->buffer()->copyImage();
423 FloatRect destRect(FloatPoint(), outputSize);
424 ImagePaintingOptions paintingOptions;
425 paintingOptions.m_interpolationQuality = interpolationQualityForResizeQuality(options.resizeQuality);
427 bitmapData->context().drawImage(*imageForRender, destRect, sourceRectangle.releaseReturnValue(), paintingOptions);
429 imageBitmap->m_bitmapData = WTFMove(bitmapData);
431 // 5. Set the origin-clean flag of the ImageBitmap object's bitmap to the same
432 // value as the origin-clean flag of the bitmap of the image argument.
433 imageBitmap->m_originClean = existingImageBitmap->originClean();
435 // 6. Return a new promise, but continue running these steps in parallel.
436 // 7. Resolve the promise with the new ImageBitmap object as the value.
437 promise.resolve(WTFMove(imageBitmap));
440 class PendingImageBitmap final : public ActiveDOMObject, public FileReaderLoaderClient {
442 static void fetch(ScriptExecutionContext& scriptExecutionContext, RefPtr<Blob>&& blob, ImageBitmapOptions&& options, std::optional<IntRect> rect, ImageBitmap::Promise&& promise)
444 auto pendingImageBitmap = new PendingImageBitmap(scriptExecutionContext, WTFMove(blob), WTFMove(options), WTFMove(rect), WTFMove(promise));
445 pendingImageBitmap->start(scriptExecutionContext);
449 PendingImageBitmap(ScriptExecutionContext& scriptExecutionContext, RefPtr<Blob>&& blob, ImageBitmapOptions&& options, std::optional<IntRect> rect, ImageBitmap::Promise&& promise)
450 : ActiveDOMObject(&scriptExecutionContext)
451 , m_blobLoader(FileReaderLoader::ReadAsArrayBuffer, this)
452 , m_blob(WTFMove(blob))
453 , m_options(WTFMove(options))
454 , m_rect(WTFMove(rect))
455 , m_promise(WTFMove(promise))
460 void start(ScriptExecutionContext& scriptExecutionContext)
462 m_blobLoader.start(&scriptExecutionContext, *m_blob);
467 const char* activeDOMObjectName() const override
469 return "PendingImageBitmap";
472 bool canSuspendForDocumentSuspension() const override
474 // FIXME: Deal with suspension.
478 // FileReaderLoaderClient
480 void didStartLoading() override
484 void didReceiveData() override
488 void didFinishLoading() override
490 createImageBitmap(m_blobLoader.arrayBufferResult());
494 void didFail(int) override
496 createImageBitmap(nullptr);
500 void createImageBitmap(RefPtr<ArrayBuffer> arrayBuffer)
502 UNUSED_PARAM(arrayBuffer);
504 // 3. Read the Blob object's data. If an error occurs during reading of the object,
505 // then reject the promise with an "InvalidStateError" DOMException, and abort
508 // 4. Apply the image sniffing rules to determine the file format of the image data,
509 // with MIME type of the Blob (as given by the Blob object's type attribute) giving
510 // the official type.
512 // 5. If the image data is not in a supported image file format (e.g. it's not an image
513 // at all), or if the image data is corrupted in some fatal way such that the image
514 // dimensions cannot be obtained (e.g. a vector graphic with no intrinsic size), then
515 // reject the promise with an "InvalidStateError" DOMException, and abort these steps.
517 // 6. Create a new ImageBitmap object.
519 // 7. Let the ImageBitmap object's bitmap data be the image data read from the Blob object,
520 // cropped to the source rectangle with formatting. If this is an animated image, the
521 // ImageBitmap object's bitmap data must only be taken from the default image of the
522 // animation (the one that the format defines is to be used when animation is not supported
523 // or is disabled), or, if there is no such image, the first frame of the animation.
525 // 8. Resolve the promise with the new ImageBitmap object as the value.
526 m_promise.reject(TypeError, "createImageBitmap with ArrayBuffer or Blob is not implemented");
529 FileReaderLoader m_blobLoader;
531 ImageBitmapOptions m_options;
532 std::optional<IntRect> m_rect;
533 ImageBitmap::Promise m_promise;
536 void ImageBitmap::createPromise(ScriptExecutionContext& scriptExecutionContext, RefPtr<Blob>& blob, ImageBitmapOptions&& options, std::optional<IntRect> rect, ImageBitmap::Promise&& promise)
538 // 2. Return a new promise, but continue running these steps in parallel.
539 PendingImageBitmap::fetch(scriptExecutionContext, WTFMove(blob), WTFMove(options), WTFMove(rect), WTFMove(promise));
542 void ImageBitmap::createPromise(ScriptExecutionContext&, RefPtr<ImageData>& imageData, ImageBitmapOptions&& options, std::optional<IntRect> rect, ImageBitmap::Promise&& promise)
544 UNUSED_PARAM(imageData);
545 UNUSED_PARAM(options);
548 // 2. If the image object's data attribute value's [[Detached]] internal slot value
549 // is true, return a promise rejected with an "InvalidStateError" DOMException
550 // and abort these steps.
552 // 3. Create a new ImageBitmap object.
554 // 4. Let the ImageBitmap object's bitmap data be the image data given by the ImageData
555 // object, cropped to the source rectangle with formatting.
557 // 5. Return a new promise, but continue running these steps in parallel.
558 // 6. Resolve the promise with the new ImageBitmap object as the value.
559 promise.reject(TypeError, "createImageBitmap with ImageData is not implemented");
562 ImageBitmap::ImageBitmap() = default;
564 ImageBitmap::~ImageBitmap() = default;
566 unsigned ImageBitmap::width() const
568 if (m_detached || !m_bitmapData)
571 // FIXME: Is this the right width?
572 return m_bitmapData->logicalSize().width();
575 unsigned ImageBitmap::height() const
577 if (m_detached || !m_bitmapData)
580 // FIXME: Is this the right height?
581 return m_bitmapData->logicalSize().height();
584 void ImageBitmap::close()
587 m_bitmapData = nullptr;
590 std::unique_ptr<ImageBuffer> ImageBitmap::transferOwnershipAndClose()
593 return WTFMove(m_bitmapData);