Skip to content
This repository was archived by the owner on Feb 22, 2023. It is now read-only.
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
37be5c5
Fix padding issue
acoutts Jan 12, 2023
45d2c58
New approach that is more efficient
acoutts Jan 13, 2023
d2492c8
Refactored image reader to enable testing
acoutts Jan 13, 2023
bd55965
Added tests for image stream reader
acoutts Jan 17, 2023
88079a9
Added tests for buffer trimming
acoutts Jan 17, 2023
fee7979
Formatter
acoutts Jan 17, 2023
f0f0640
Bump pubspec version
acoutts Jan 17, 2023
df99b02
Optimize imports
acoutts Jan 17, 2023
90b2993
Cleanup
acoutts Jan 17, 2023
f5c7f8f
Imports cleanup
acoutts Jan 17, 2023
fd858eb
Imports
acoutts Jan 17, 2023
b1f292e
Imports
acoutts Jan 17, 2023
e6e1f58
Update Camera.java
acoutts Jan 17, 2023
800878f
Update CameraRegionUtils.java
acoutts Jan 17, 2023
56995bb
Update ImageStreamReader.java
acoutts Jan 17, 2023
5efd3e6
Format imports on tests
acoutts Jan 17, 2023
6667183
Formatters
acoutts Jan 18, 2023
b7dec7f
Licenses
acoutts Jan 18, 2023
0b8d812
Merge remote-tracking branch 'upstream/main' into fix-handle-padding
acoutts Jan 18, 2023
ce2a3b9
Added support for NV21 streaming on android as a new image format
acoutts Jan 20, 2023
404a954
Remove unused parameter
acoutts Jan 20, 2023
3ec602e
Add NV21 type conversion
acoutts Jan 20, 2023
a010585
Add legacy code path for nv21 to camera package
acoutts Jan 22, 2023
8bd8710
Clean up log messages
acoutts Feb 17, 2023
7b7708b
Formatters
acoutts Feb 17, 2023
d18a997
Dependency overrides as per contributing guidelines
acoutts Feb 17, 2023
37fa30d
Fix tests for image stream reader
acoutts Feb 17, 2023
c4c1719
Test cleanup
acoutts Feb 17, 2023
bf18e06
Added tests for removing padding
acoutts Feb 17, 2023
628a16a
Formatter
acoutts Feb 17, 2023
0ddc5d4
Merge remote-tracking branch 'upstream/main' into fix-handle-padding
acoutts Feb 17, 2023
651d202
Update CHANGELOG.md
acoutts Feb 17, 2023
cbfb447
Changelogs
acoutts Feb 17, 2023
1d1ecbf
Update CHANGELOG.md
acoutts Feb 17, 2023
804ddc7
Update ImageStreamReader.java
acoutts Feb 17, 2023
5a10847
Comments
acoutts Feb 17, 2023
198c55d
comments
acoutts Feb 17, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Formatters
  • Loading branch information
acoutts committed Jan 18, 2023
commit 666718334c0f6f0fa3dfddb41ce4659a82cdb6c8
Original file line number Diff line number Diff line change
Expand Up @@ -301,12 +301,13 @@ public void open(String imageFormatGroup) throws CameraAccessException {
Log.w(TAG, "The selected imageFormatGroup is not supported by Android. Defaulting to yuv420");
imageFormat = ImageFormat.YUV_420_888;
}
imageStreamReader = new ImageStreamReader(ImageReader.newInstance(
resolutionFeature.getPreviewSize().getWidth(),
resolutionFeature.getPreviewSize().getHeight(),
imageFormat,
1)
);
imageStreamReader =
new ImageStreamReader(
ImageReader.newInstance(
resolutionFeature.getPreviewSize().getWidth(),
resolutionFeature.getPreviewSize().getHeight(),
imageFormat,
1));

// Open the camera.
CameraManager cameraManager = CameraUtils.getCameraManager(activity);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,156 +8,158 @@
import android.view.Surface;
import androidx.annotation.NonNull;
import androidx.annotation.VisibleForTesting;
import io.flutter.plugin.common.EventChannel;
import io.flutter.plugins.camera.types.CameraCaptureProperties;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import io.flutter.plugin.common.EventChannel;
import io.flutter.plugins.camera.types.CameraCaptureProperties;

// Wraps an ImageReader to allow for testing of the image handler.
public class ImageStreamReader {
private final ImageReader imageReader;
private final ImageStreamReaderUtils imageStreamReaderUtils;

/**
* Creates a new instance of the {@link ImageStreamReader}.
*
* @param imageReader is the image reader that will receive frames
* @param imageStreamReaderUtils is an instance of {@link ImageStreamReaderUtils}
*/
@VisibleForTesting
public ImageStreamReader(ImageReader imageReader, ImageStreamReaderUtils imageStreamReaderUtils) {
this.imageReader = imageReader;
this.imageStreamReaderUtils = imageStreamReaderUtils;
}

/**
* Creates a new instance of the {@link ImageStreamReader}.
*
* @param imageReader is the image reader that will receive frames
*/
public ImageStreamReader(ImageReader imageReader) {
this.imageReader = imageReader;
this.imageStreamReaderUtils = new ImageStreamReaderUtils();
}

/**
* Processes a new frame (image) from the image reader, remove padding if necessary,
* and send the frame to Dart.
*
* @param image is the image which needs processed as an {@link Image}
* @param imageFormat is the image format from the image reader as an int, a valid {@link ImageFormat}
* @param captureProps is the capture props from the camera class as {@link CameraCaptureProperties}
* @param imageStreamSink is the image stream sink from dart as a dart {@link EventChannel.EventSink}
*/
@VisibleForTesting
public void onImageAvailable(
@NonNull Image image,
int imageFormat,
CameraCaptureProperties captureProps,
EventChannel.EventSink imageStreamSink
) {
List<Map<String, Object>> planes = new ArrayList<>();
for (int i=0; i<image.getPlanes().length; i++) {
// Current plane
Image.Plane plane = image.getPlanes()[i];

// The metadata to be returned to dart
Map<String, Object> planeBuffer = new HashMap<>();
planeBuffer.put("bytesPerPixel", plane.getPixelStride());

// Sometimes YUV420 has additional padding that must be removed. This is only the case if we are
// streaming YUV420, the row stride does not match the image width, and the pixel stride is 1.
if (imageFormat == ImageFormat.YUV_420_888 &&
plane.getRowStride() != image.getWidth() &&
plane.getPixelStride() == 1) {
// The ordering of planes is guaranteed by Android. It always goes Y, U, V.
int planeWidth;
int planeHeight;
if (i == 0) {
// Y is the image size
planeWidth = image.getWidth();
planeHeight = image.getHeight();
} else {
// U and V are guaranteed to be the same size and are half of the image height/width
// in YUV420
planeWidth = image.getWidth() / 2;
planeHeight = image.getHeight() / 2;
}

planeBuffer.put("bytes", imageStreamReaderUtils.removePlaneBufferPadding(plane, planeWidth, planeHeight));

// Make sure the bytesPerRow matches the image width now that we've removed the padding
planeBuffer.put("bytesPerRow", image.getWidth());
} else {
// Just use the data as-is
ByteBuffer buffer = plane.getBuffer();
byte[] bytes = new byte[buffer.remaining()];
buffer.get(bytes, 0, bytes.length);
planeBuffer.put("bytes", bytes);
planeBuffer.put("bytesPerRow", plane.getRowStride());
}
planes.add(planeBuffer);
private final ImageReader imageReader;
private final ImageStreamReaderUtils imageStreamReaderUtils;

/**
* Creates a new instance of the {@link ImageStreamReader}.
*
* @param imageReader is the image reader that will receive frames
* @param imageStreamReaderUtils is an instance of {@link ImageStreamReaderUtils}
*/
@VisibleForTesting
public ImageStreamReader(ImageReader imageReader, ImageStreamReaderUtils imageStreamReaderUtils) {
this.imageReader = imageReader;
this.imageStreamReaderUtils = imageStreamReaderUtils;
}

/**
* Creates a new instance of the {@link ImageStreamReader}.
*
* @param imageReader is the image reader that will receive frames
*/
public ImageStreamReader(ImageReader imageReader) {
this.imageReader = imageReader;
this.imageStreamReaderUtils = new ImageStreamReaderUtils();
}

/**
* Processes a new frame (image) from the image reader, remove padding if necessary, and send the
* frame to Dart.
*
* @param image is the image which needs processed as an {@link Image}
* @param imageFormat is the image format from the image reader as an int, a valid {@link
* ImageFormat}
* @param captureProps is the capture props from the camera class as {@link
* CameraCaptureProperties}
* @param imageStreamSink is the image stream sink from dart as a dart {@link
* EventChannel.EventSink}
*/
@VisibleForTesting
public void onImageAvailable(
@NonNull Image image,
int imageFormat,
CameraCaptureProperties captureProps,
EventChannel.EventSink imageStreamSink) {
List<Map<String, Object>> planes = new ArrayList<>();
for (int i = 0; i < image.getPlanes().length; i++) {
// Current plane
Image.Plane plane = image.getPlanes()[i];

// The metadata to be returned to dart
Map<String, Object> planeBuffer = new HashMap<>();
planeBuffer.put("bytesPerPixel", plane.getPixelStride());

// Sometimes YUV420 has additional padding that must be removed. This is only the case if we are
// streaming YUV420, the row stride does not match the image width, and the pixel stride is 1.
if (imageFormat == ImageFormat.YUV_420_888
&& plane.getRowStride() != image.getWidth()
&& plane.getPixelStride() == 1) {
// The ordering of planes is guaranteed by Android. It always goes Y, U, V.
int planeWidth;
int planeHeight;
if (i == 0) {
// Y is the image size
planeWidth = image.getWidth();
planeHeight = image.getHeight();
} else {
// U and V are guaranteed to be the same size and are half of the image height/width
// in YUV420
planeWidth = image.getWidth() / 2;
planeHeight = image.getHeight() / 2;
}

Map<String, Object> imageBuffer = new HashMap<>();
imageBuffer.put("width", image.getWidth());
imageBuffer.put("height", image.getHeight());
imageBuffer.put("format", image.getFormat());
imageBuffer.put("planes", planes);
imageBuffer.put("lensAperture", captureProps.getLastLensAperture());
imageBuffer.put("sensorExposureTime", captureProps.getLastSensorExposureTime());
Integer sensorSensitivity = captureProps.getLastSensorSensitivity();
imageBuffer.put(
"sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity);

final Handler handler = new Handler(Looper.getMainLooper());
handler.post(() -> imageStreamSink.success(imageBuffer));
image.close();
planeBuffer.put(
"bytes",
imageStreamReaderUtils.removePlaneBufferPadding(plane, planeWidth, planeHeight));

// Make sure the bytesPerRow matches the image width now that we've removed the padding
planeBuffer.put("bytesPerRow", image.getWidth());
} else {
// Just use the data as-is
ByteBuffer buffer = plane.getBuffer();
byte[] bytes = new byte[buffer.remaining()];
buffer.get(bytes, 0, bytes.length);
planeBuffer.put("bytes", bytes);
planeBuffer.put("bytesPerRow", plane.getRowStride());
}
planes.add(planeBuffer);
}

/**
* Returns the image reader surface.
*/
public Surface getSurface() {
return imageReader.getSurface();
}

/**
* Subscribes the image stream reader to handle incoming images using onImageAvailable().
*
* @param captureProps is the capture props from the camera class as {@link CameraCaptureProperties}
* @param imageStreamSink is the image stream sink from dart as {@link EventChannel.EventSink}
* @param handler is generally the background handler of the camera as {@link Handler}
*/
public void subscribeListener(
CameraCaptureProperties captureProps,
EventChannel.EventSink imageStreamSink,
Handler handler
) {
imageReader.setOnImageAvailableListener(reader -> {
Image image = reader.acquireNextImage();
if (image == null) return;

onImageAvailable(image, imageReader.getImageFormat(), captureProps, imageStreamSink);
}, handler);
}

/**
* Removes the listener from the image reader.
*
* @param handler is generally the background handler of the camera
*/
public void removeListener(Handler handler) {
imageReader.setOnImageAvailableListener(null, handler);
}

/**
* Closes the image reader.
*/
public void close() {
imageReader.close();
}
Map<String, Object> imageBuffer = new HashMap<>();
imageBuffer.put("width", image.getWidth());
imageBuffer.put("height", image.getHeight());
imageBuffer.put("format", image.getFormat());
imageBuffer.put("planes", planes);
imageBuffer.put("lensAperture", captureProps.getLastLensAperture());
imageBuffer.put("sensorExposureTime", captureProps.getLastSensorExposureTime());
Integer sensorSensitivity = captureProps.getLastSensorSensitivity();
imageBuffer.put(
"sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity);

final Handler handler = new Handler(Looper.getMainLooper());
handler.post(() -> imageStreamSink.success(imageBuffer));
image.close();
}

/** Returns the image reader surface. */
public Surface getSurface() {
return imageReader.getSurface();
}

/**
* Subscribes the image stream reader to handle incoming images using onImageAvailable().
*
* @param captureProps is the capture props from the camera class as {@link
* CameraCaptureProperties}
* @param imageStreamSink is the image stream sink from dart as {@link EventChannel.EventSink}
* @param handler is generally the background handler of the camera as {@link Handler}
*/
public void subscribeListener(
CameraCaptureProperties captureProps,
EventChannel.EventSink imageStreamSink,
Handler handler) {
imageReader.setOnImageAvailableListener(
reader -> {
Image image = reader.acquireNextImage();
if (image == null) return;

onImageAvailable(image, imageReader.getImageFormat(), captureProps, imageStreamSink);
},
handler);
}

/**
* Removes the listener from the image reader.
*
* @param handler is generally the background handler of the camera
*/
public void removeListener(Handler handler) {
imageReader.setOnImageAvailableListener(null, handler);
}

/** Closes the image reader. */
public void close() {
imageReader.close();
}
}
Loading