Skip to content
This repository was archived by the owner on Feb 22, 2023. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
29beaa7
Fixed crash when streaming in iOS
zuvola Nov 18, 2021
aec0f93
Fixed skip judgment
zuvola Nov 18, 2021
0d9f61a
Merge branch 'flutter:master' into fix_ios_streaming
zuvola Nov 22, 2021
7cff775
Add test
zuvola Nov 22, 2021
5d1fb56
Update CHANGELOG.md
zuvola Nov 22, 2021
7c0fa9e
Fix test
zuvola Nov 22, 2021
3056e17
Add frameStack property to startImageStream
zuvola Dec 10, 2021
1a48ffc
Merge branch 'flutter:master' into fix_ios_streaming
zuvola Dec 10, 2021
4cb5d6e
Merge branch 'fix_ios_streaming' of https://github.com/zuvola/plugins…
zuvola Dec 10, 2021
27957a1
Bump version to 0.9.4+6
zuvola Dec 10, 2021
97d16c7
format
zuvola Dec 10, 2021
1382d62
Fixed tests
zuvola Dec 14, 2021
46d9df0
Merge branch 'main' into fix_ios_streaming
zuvola Jan 27, 2022
84dc53d
Restored the API and set the number of pending frames to 4
zuvola Jan 28, 2022
3ed1e94
Fixed tests
zuvola Feb 7, 2022
9a3e627
Merge commit 'f7138f7222856623796c2ee2e561f866fcf08d38' into fix_ios_…
zuvola Feb 7, 2022
6191e3a
Merge
zuvola Feb 7, 2022
e732d7b
Fixed CHANGELOG
zuvola Feb 7, 2022
b69ea0f
Fixed tests
zuvola Feb 7, 2022
e9beb1f
Merge branch 'main' into fix_ios_streaming
zuvola Mar 8, 2022
74d9d85
Apply PR feedback
zuvola Mar 10, 2022
6b81e1a
Update messages
zuvola Mar 11, 2022
96f8edb
Merge branch 'main' into fix_ios_streaming
zuvola Mar 11, 2022
1167112
Update to use CameraTestUtils
zuvola Mar 11, 2022
24871c6
Update _Test header
zuvola Mar 11, 2022
7635469
Bump version to 0.9.4+17
zuvola Mar 11, 2022
376bbfa
Inject FLTImageStreamHandler object
zuvola Mar 16, 2022
f276bd0
Capital S
zuvola Mar 16, 2022
f69aed6
Use XCTNSPredicateExpectation
zuvola Mar 16, 2022
9653345
Wait 1 second
zuvola Mar 16, 2022
661cf49
Merge branch 'main' into fix_ios_streaming
zuvola Mar 16, 2022
8e334bf
Format
zuvola Mar 16, 2022
7b8d74f
Using KVO to wait
zuvola Mar 16, 2022
ab233fa
Add comments
zuvola Mar 23, 2022
830d5ad
Update comments
zuvola Mar 24, 2022
095faf3
Merge branch 'main' into fix_ios_streaming
stuartmorgan-g Mar 25, 2022
e185988
Version bump
stuartmorgan-g Mar 25, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add frameStack property to startImageStream
  • Loading branch information
zuvola committed Dec 10, 2021
commit 3056e1778cf82be5d7a6fa5606df751a285c6732
Original file line number Diff line number Diff line change
Expand Up @@ -256,14 +256,10 @@ void main() {
);

await controller.initialize();
int _frame = 0;

await controller.startImageStream((CameraImage image) {
_frame++;
});
await controller.startImageStream((CameraImage image) {}, frameStack: 3);

await Future.delayed(Duration(seconds: 5));
expect(_frame > 30, true);

await controller.stopImageStream();
await controller.dispose();
Expand Down
17 changes: 10 additions & 7 deletions packages/camera/camera/ios/Classes/CameraPlugin.m
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,8 @@ @interface FLTCam : NSObject <FlutterTexture,
@property(assign, nonatomic) BOOL audioIsDisconnected;
@property(assign, nonatomic) BOOL isAudioSetup;
@property(assign, nonatomic) BOOL isStreamingImages;
@property(assign, nonatomic) BOOL skipStreamingImages;
@property(assign, nonatomic) int maxStreamingFrameStack;
@property(assign, nonatomic) int streamingFrameStack;
@property(assign, nonatomic) BOOL isPreviewPaused;
@property(assign, nonatomic) ResolutionPreset resolutionPreset;
@property(assign, nonatomic) ExposureMode exposureMode;
Expand Down Expand Up @@ -606,8 +607,8 @@ - (void)captureOutput:(AVCaptureOutput *)output
return;
}
if (_isStreamingImages) {
if (_imageStreamHandler.eventSink && !_skipStreamingImages) {
_skipStreamingImages = YES;
if (_imageStreamHandler.eventSink && (_maxStreamingFrameStack < 1 || _streamingFrameStack < _maxStreamingFrameStack)) {
_streamingFrameStack++;
CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);

Expand Down Expand Up @@ -1112,7 +1113,7 @@ - (void)setExposureOffsetWithResult:(FLTThreadSafeFlutterResult *)result offset:
[result sendSuccessWithData:@(offset)];
}

- (void)startImageStreamWithMessenger:(NSObject<FlutterBinaryMessenger> *)messenger {
- (void)startImageStreamWithMessenger:(NSObject<FlutterBinaryMessenger> *)messenger frameStack:(int)frameStack {
if (!_isStreamingImages) {
FlutterEventChannel *eventChannel =
[FlutterEventChannel eventChannelWithName:@"plugins.flutter.io/camera/imageStream"
Expand All @@ -1122,7 +1123,8 @@ - (void)startImageStreamWithMessenger:(NSObject<FlutterBinaryMessenger> *)messen
[eventChannel setStreamHandler:_imageStreamHandler];

_isStreamingImages = YES;
_skipStreamingImages = NO;
_streamingFrameStack = 0;
_maxStreamingFrameStack = frameStack;
} else {
[_methodChannel invokeMethod:errorMethod
arguments:@"Images from camera are already streaming!"];
Expand Down Expand Up @@ -1427,13 +1429,14 @@ - (void)handleMethodCallAsync:(FlutterMethodCall *)call
}];
}
} else if ([@"startImageStream" isEqualToString:call.method]) {
[_camera startImageStreamWithMessenger:_messenger];
NSNumber *frameStack = call.arguments[@"frameStack"];
[_camera startImageStreamWithMessenger:_messenger frameStack:[frameStack intValue]];
[result sendSuccess];
} else if ([@"stopImageStream" isEqualToString:call.method]) {
[_camera stopImageStream];
[result sendSuccess];
} else if ([@"receivedImageStreamData" isEqualToString:call.method]) {
_camera.skipStreamingImages = NO;
_camera.streamingFrameStack--;
} else {
NSDictionary *argsMap = call.arguments;
NSUInteger cameraId = ((NSNumber *)argsMap[@"cameraId"]).unsignedIntegerValue;
Expand Down
13 changes: 10 additions & 3 deletions packages/camera/camera/lib/src/camera_controller.dart
Original file line number Diff line number Diff line change
Expand Up @@ -403,14 +403,20 @@ class CameraController extends ValueNotifier<CameraValue> {
/// have significant frame rate drops for [CameraPreview] on lower end
/// devices.
///
/// On older iOS devices, frames pending processing can exhaust the memory
/// and cause crashes. In this case, you can use [frameStack] to limit
/// the maximum number of frames pending processing. The default value is 0,
/// which means there is no limit.
///
/// Throws a [CameraException] if image streaming or video recording has
/// already started.
///
/// The `startImageStream` method is only available on Android and iOS (other
/// platforms won't be supported in current setup).
///
// TODO(bmparr): Add settings for resolution and fps.
Future<void> startImageStream(onLatestImageAvailable onAvailable) async {
Future<void> startImageStream(onLatestImageAvailable onAvailable,
{int frameStack = 0}) async {
assert(defaultTargetPlatform == TargetPlatform.android ||
defaultTargetPlatform == TargetPlatform.iOS);
_throwIfNotInitialized("startImageStream");
Expand All @@ -428,7 +434,8 @@ class CameraController extends ValueNotifier<CameraValue> {
}

try {
await _channel.invokeMethod<void>('startImageStream');
await _channel
.invokeMethod<void>('startImageStream', {'frameStack': frameStack});
value = value.copyWith(isStreamingImages: true);
} on PlatformException catch (e) {
throw CameraException(e.code, e.message);
Expand All @@ -438,7 +445,7 @@ class CameraController extends ValueNotifier<CameraValue> {
_imageStreamSubscription =
cameraEventChannel.receiveBroadcastStream().listen(
(dynamic imageData) {
if (defaultTargetPlatform == TargetPlatform.iOS) {
if (defaultTargetPlatform == TargetPlatform.iOS && frameStack > 0) {
try {
_channel.invokeMethod<void>('receivedImageStreamData');
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Given that this is asynchronous in both directions I would expect this could cause significant frame loss even on a reasonably powerful device. How much investigation have you done into specifically what the parameters of the crash are? Is it really necessary to have a maximum of a single frame in flight?

It seems likely that this needs a more nuanced approach (maybe a hard limit still but higher than one, maybe something that's tunable by plugin clients). It's hard to determine the right fix without more details about what the failure condition actually is.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I modified the plugin to allow you to set a limit on the maximum number of frames pending processing.
And I tested it on my devices to see how many frames can be processed in 5 seconds. The results are shown in the table below.
The value is the number of frames processed at max quality and is the average of three times.
The horizontal axis is the maximum number of pending frames (frameStack), where 0 means no limit.
With no limit, iPhone6 and iPadAir crashed.
In this case, if frameStack is set to about 4, frame loss can be reduced to some extent.

  0 1 2 3 4 5 6
iPhoneX 143 76 108 118 133 138 136
iPhone6 13(crash) 40 53 69 70 55 56
iPadAir-1st 14(crash) 44 62 53 63 52 45

I don't have that many devices, but according to flutter/flutter#44436, it seems to be crashing on iPhone 6, 7, 8, etc. These devices still have a large market share that cannot be ignored. Also, for image analysis processes such as OCR, low or medium quality is not always satisfactory.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for this testing. Given this table, I'm fine with starting from setting a fixed value—4 looks like a good place to start—with an explanation in the code of where that number came from, and then if it turns out that causes issues in some cases we can add an API to tune it later.

} on PlatformException catch (e) {
Expand Down
2 changes: 1 addition & 1 deletion packages/camera/camera/test/camera_preview_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ class FakeController extends ValueNotifier<CameraValue>
Future<void> setZoomLevel(double zoom) async {}

@override
Future<void> startImageStream(onAvailable) async {}
Future<void> startImageStream(onAvailable, {int frameStack = 0}) async {}

@override
Future<void> startVideoRecording() async {}
Expand Down