Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
49ed8d4
Use HtmlImageElement for image decoding in skia
harryterkelsen May 24, 2024
6b248d1
start touching up tests
harryterkelsen May 29, 2024
d780fe3
Merge branch 'main' into canvaskit-html-img-element-decode
harryterkelsen May 29, 2024
4e43312
eagerly decode
harryterkelsen May 31, 2024
9b1a448
Merge branch 'main' into canvaskit-html-img-element-decode
harryterkelsen May 31, 2024
e6d4d44
wip
harryterkelsen Jun 4, 2024
8ea4865
Merge branch 'main' into canvaskit-html-img-element-decode
harryterkelsen Jun 4, 2024
88d685d
wip
harryterkelsen Jul 2, 2024
23cb86b
Merge branch 'main' into canvaskit-html-img-element-decode
harryterkelsen Jul 2, 2024
7883559
Merge branch 'main' into canvaskit-html-img-element-decode
harryterkelsen Jul 3, 2024
8a1a7ce
WIP
harryterkelsen Jul 10, 2024
389369d
Merge branch 'main' into canvaskit-html-img-element-decode
harryterkelsen Jul 10, 2024
d900ded
Fix ImageBitmap toByteData case
harryterkelsen Jul 11, 2024
355891b
Fall back to Skia decoding for GIF and WEBP images
harryterkelsen Jul 11, 2024
cfc1c4a
Merge branch 'main' into canvaskit-html-img-element-decode
harryterkelsen Jul 11, 2024
d317302
Undo semantics change
harryterkelsen Jul 11, 2024
c579c8e
Remove unused readImageElementPixelsUnmodified method
harryterkelsen Jul 11, 2024
dd5c8d6
remove outdated comment in test
harryterkelsen Jul 11, 2024
f3997c1
delete old unused tests
harryterkelsen Jul 11, 2024
52b8330
fix analysis errors
harryterkelsen Jul 11, 2024
b207ee7
Merge branch 'main' into canvaskit-html-img-element-decode
harryterkelsen Jul 16, 2024
4f5f863
Merge branch 'main' into pr/harryterkelsen/53201
harryterkelsen Jul 17, 2024
7c6ab13
Respond to review comments
harryterkelsen Jul 17, 2024
0b426fe
Merge branch 'main' into canvaskit-html-img-element-decode
harryterkelsen Jul 18, 2024
c06797c
Merge branch 'canvaskit-html-img-element-decode' of github.com:harryt…
harryterkelsen Jul 18, 2024
e7aff46
Remove unnecessary `await`
harryterkelsen Jul 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
WIP
  • Loading branch information
harryterkelsen committed Jul 10, 2024
commit 8a1a7ce5a8cb842dfeac770ca9629d2a0adf86e5
8 changes: 4 additions & 4 deletions lib/web_ui/lib/semantics.dart
Original file line number Diff line number Diff line change
Expand Up @@ -280,14 +280,14 @@ class SemanticsUpdateBuilder {
required List<StringAttribute> decreasedValueAttributes,
required String hint,
required List<StringAttribute> hintAttributes,
String? tooltip,
TextDirection? textDirection,
required String? tooltip,
required TextDirection? textDirection,
required Float64List transform,
required Int32List childrenInTraversalOrder,
required Int32List childrenInHitTestOrder,
required Int32List additionalActions,
int headingLevel = 0,
String? linkUrl,
required int headingLevel,
required String? linkUrl,
}) {
if (transform.length != 16) {
throw ArgumentError('transform argument must have 16 entries.');
Expand Down
104 changes: 100 additions & 4 deletions lib/web_ui/lib/src/engine/canvaskit/image.dart
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,113 @@ Future<ui.Codec> skiaInstantiateImageCodec(Uint8List list,
final DomBlob blob = createDomBlob(<ByteBuffer>[list.buffer]);
codec = await decodeBlobToCkImage(blob);
}
return ResizingCodec(
return CkResizingCodec(
codec,
targetWidth: targetWidth,
targetHeight: targetHeight,
allowUpscaling: allowUpscaling,
);
}

/// A resizing codec which uses an HTML <canvas> element to scale the image if
/// it is backed by an HTML Image element.
class CkResizingCodec extends ResizingCodec {
CkResizingCodec(
super.delegate, {
super.targetWidth,
super.targetHeight,
super.allowUpscaling,
});

@override
ui.Image scaleImage(
ui.Image image, {
int? targetWidth,
int? targetHeight,
bool allowUpscaling = true,
}) {
final CkImage ckImage = image as CkImage;
if (ckImage.imageElement == null) {
return scaleImageIfNeeded(
image,
targetWidth: targetWidth,
targetHeight: targetHeight,
allowUpscaling: allowUpscaling,
);
} else {
return _scaleImageUsingDomCanvas(
ckImage,
targetWidth: targetWidth,
targetHeight: targetHeight,
allowUpscaling: allowUpscaling,
);
}
}

CkImage _scaleImageUsingDomCanvas(
CkImage image, {
int? targetWidth,
int? targetHeight,
bool allowUpscaling = true,
}) {
assert(image.imageElement != null);
final int width = image.width;
final int height = image.height;
final ui.Size? scaledSize =
scaledImageSize(width, height, targetWidth, targetHeight);
if (scaledSize == null) {
return image;
}
if (!allowUpscaling &&
(scaledSize.width > width || scaledSize.height > height)) {
return image;
}

final int scaledWidth = scaledSize.width.toInt();
final int scaledHeight = scaledSize.height.toInt();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe scaledImageSize should switch to BitmapSize to avoid rounding? There's also the issue that we check for allowUpscaling before calling toInt, which may end up upscaling anyway.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Switched to BitmapSize to avoid ugly .toInt() and .toDouble() calls where possible. Checking for allowUpscaling before calling toInt() should be okay here since we're only working with ints for the width and height.


final DomCanvasElement htmlCanvas = createDomCanvasElement(
width: scaledWidth,
height: scaledHeight,
);
final DomCanvasRenderingContext2D ctx =
htmlCanvas.getContext('2d')! as DomCanvasRenderingContext2D;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm guessing that despite the slowness (due to https://bugs.webkit.org/show_bug.cgi?id=267291) a 2d OffscreenCanvas with transferToImageBitmap would still be faster than an on-screen 2d canvas + getImageData. The latter is guaranteed to always copy, in addition to other copying that happens in canvasKit.MakeImage.

However, if use ImageBitmap as the source, then we'd have to use MakeLazyImageFromTextureSource, which has been a bit unreliable.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unreliable how? I think we should try to avoid explicitly pulling the bytes out of the image. Is there a reason we can't just create an ImageBitmap and use that, instead of grabbing the image data? I also should mention that createImageBitmap actually allows you to scale the image while extracting it from the image source: https://developer.mozilla.org/en-US/docs/Web/API/createImageBitmap

ctx.drawImage(
image.imageElement!,
0,
0,
width,
height,
0,
0,
scaledWidth,
scaledHeight,
);
final DomImageData imageData =
ctx.getImageData(0, 0, scaledWidth, scaledHeight);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Safari can run out 2D canvas memory easily. To prevent that set the size of the htmlCanvas to 0x0. That forces Safari to reclaim the underlying image buffer eagerly. Alternatively, maybe the htmlCanvas could be reused. If the app scaled an image once, it may do it again, but that's speculative.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

final Uint8List pixels = imageData.data.buffer.asUint8List();

final SkImage? skImage = canvasKit.MakeImage(
SkImageInfo(
width: scaledWidth.toDouble(),
height: scaledHeight.toDouble(),
colorType: canvasKit.ColorType.RGBA_8888,
alphaType: canvasKit.AlphaType.Premul,
colorSpace: SkColorSpaceSRGB,
),
pixels,
(4 * scaledWidth).toDouble(),
);

if (skImage == null) {
domWindow.console.warn('Failed to scale image.');
return image;
}

return CkImage(skImage);
}
}

ui.Image createCkImageFromImageElement(
DomHTMLImageElement image,
int naturalWidth,
Expand Down Expand Up @@ -212,9 +311,6 @@ CkImage scaleImage(SkImage image, int? targetWidth, int? targetHeight) {
return ckImage;
}

CkImage scaleImageWithCanvas(
SkImage image, int? targetWidth, int? targetHeight) {}

/// Thrown when the web engine fails to decode an image, either due to a
/// network issue, corrupted image contents, or missing codec.
class ImageCodecException implements Exception {
Expand Down
19 changes: 16 additions & 3 deletions lib/web_ui/lib/src/engine/image_decoder.dart
Original file line number Diff line number Diff line change
Expand Up @@ -381,18 +381,31 @@ class ResizingCodec implements ui.Codec {
final ui.FrameInfo frameInfo = await delegate.getNextFrame();
return AnimatedImageFrameInfo(
frameInfo.duration,
scaleImageIfNeeded(frameInfo.image,
scaleImage(frameInfo.image,
targetWidth: targetWidth,
targetHeight: targetHeight,
allowUpscaling: allowUpscaling),
);
}

ui.Image scaleImage(
ui.Image image, {
int? targetWidth,
int? targetHeight,
bool allowUpscaling = true,
}) =>
scaleImageIfNeeded(
image,
targetWidth: targetWidth,
targetHeight: targetHeight,
allowUpscaling: allowUpscaling,
);

@override
int get repetitionCount => delegate.frameCount;
}

ui.Size? _scaledSize(
ui.Size? scaledImageSize(
int width,
int height,
int? targetWidth,
Expand Down Expand Up @@ -427,7 +440,7 @@ ui.Image scaleImageIfNeeded(
final int width = image.width;
final int height = image.height;
final ui.Size? scaledSize =
_scaledSize(width, height, targetWidth, targetHeight);
scaledImageSize(width, height, targetWidth, targetHeight);
if (scaledSize == null) {
return image;
}
Expand Down
Loading