This repository was archived by the owner on Feb 22, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 9.7k
Add byte streaming capability for the camera #965
Merged
Merged
Changes from 1 commit
Commits
Show all changes
39 commits
Select commit
Hold shift + click to select a range
c3f703e
Start of Android side of byte stream passing
bparrishMines f310580
dart side of byte streaming
bparrishMines 87fdb60
Fix android streaming
bparrishMines 673b003
Merge branch 'master' of github.com:flutter/plugins into camera_android
bparrishMines 8d353ad
Add ios byte streaming
bparrishMines 0291a29
Convert buffer to uiimage to pass over
bparrishMines 31f746a
formatting
bparrishMines 5fcfeb2
Stream yuv bytes instead
bparrishMines 040d1ae
Make video format a constant
bparrishMines 16d1d32
Pass back metadata for ios image
bparrishMines 8e88978
Pass back metadata for android image
bparrishMines 67f8304
Dart code now parses camera image buffer
bparrishMines 646283a
YUV image to bgra
bparrishMines 7e99691
Add documentation
bparrishMines 4cbfab9
Only pass available data on Android
bparrishMines aa0e263
Merge branch 'master' of github.com:bparrishMines/plugins into camera…
bparrishMines 9b2ae22
Merge branch 'master' of github.com:flutter/plugins into camera_andro…
bparrishMines 297fe7a
Bump version
bparrishMines bdd9007
Formatting
bparrishMines ccc057b
create imageformat error
bparrishMines 989edf6
Don't return from null
bparrishMines 22ce601
Merge branch 'master' of github.com:bparrishMines/plugins into camera…
bparrishMines b1d7b89
Init buffers in constructor
bparrishMines aa3db0c
Add yuv ios format
bparrishMines 0349ae2
Used presets with defined resolution. Sometimes resolution would come…
bparrishMines 2633c49
Formatting
bparrishMines 6377c64
Move CameraImage classes to separate file
bparrishMines 6660a68
Move camera.dart to src folder
bparrishMines a4d278e
Create camera library
bparrishMines 045dd53
Better name and comments
bparrishMines 01520fe
Change from library camera file
bparrishMines 6155620
bytestream -> imagestream
bparrishMines 5898b4c
Comments and names
bparrishMines 7617bb9
Formatting
bparrishMines f18db98
Added resolution and fps todo
bparrishMines 0bf466a
Unmodify file
bparrishMines a53222b
Empty commit to rerun tests
bparrishMines fe965d1
Remove TODO from documentation
bparrishMines 7c0228f
Merge branch 'master' of github.com:flutter/plugins into camera_andro…
bparrishMines File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Init buffers in constructor
- Loading branch information
commit b1d7b89a1b6399d54339030089ed095069383429
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -23,7 +23,7 @@ @interface FLTSavePhotoDelegate : NSObject <AVCapturePhotoCaptureDelegate> | |
| @end | ||
|
|
||
| @interface FLTByteStreamHandler : NSObject <FlutterStreamHandler> | ||
| @property(readonly, nonatomic) FlutterEventSink eventSink; | ||
| @property FlutterEventSink eventSink; | ||
| @end | ||
|
|
||
| @implementation FLTByteStreamHandler { | ||
bparrishMines marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
|
|
@@ -104,6 +104,8 @@ @interface FLTCam : NSObject <FlutterTexture, | |
| @property(assign, nonatomic) BOOL isRecording; | ||
| @property(assign, nonatomic) BOOL isAudioSetup; | ||
| @property(assign, nonatomic) BOOL isStreamingBytes; | ||
| @property(nonatomic) vImage_Buffer destinationBuffer; | ||
| @property(nonatomic) vImage_Buffer conversionBuffer; | ||
| - (instancetype)initWithCameraName:(NSString *)cameraName | ||
| resolutionPreset:(NSString *)resolutionPreset | ||
| error:(NSError **)error; | ||
|
|
@@ -119,8 +121,6 @@ - (void)captureToFile:(NSString *)filename result:(FlutterResult)result; | |
|
|
||
| @implementation FLTCam | ||
| FourCharCode const videoFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Format that I believe is recommended for iOS 10+, which is minimum requirement for this plugin.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Add this explanation as a comment for future readers to understand the code.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. done |
||
| vImage_Buffer destinationBuffer; | ||
| vImage_Buffer conversionBuffer; | ||
|
|
||
| - (instancetype)initWithCameraName:(NSString *)cameraName | ||
| resolutionPreset:(NSString *)resolutionPreset | ||
|
|
@@ -151,6 +151,9 @@ - (instancetype)initWithCameraName:(NSString *)cameraName | |
| CMVideoFormatDescriptionGetDimensions([[_captureDevice activeFormat] formatDescription]); | ||
| _previewSize = CGSizeMake(dimensions.width, dimensions.height); | ||
|
|
||
| vImageBuffer_Init(&_destinationBuffer, 1280, 720, 32, kvImageNoFlags); | ||
| vImageBuffer_Init(&_conversionBuffer, 1280, 720, 32, kvImageNoFlags); | ||
|
|
||
| _captureVideoOutput = [AVCaptureVideoDataOutput new]; | ||
| _captureVideoOutput.videoSettings = | ||
| @{(NSString *)kCVPixelBufferPixelFormatTypeKey : @(videoFormat)}; | ||
|
|
@@ -213,43 +216,45 @@ - (void)captureOutput:(AVCaptureOutput *)output | |
| }); | ||
| return; | ||
| } | ||
| if (_isStreamingBytes && _byteStreamHandler.eventSink) { | ||
| CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); | ||
| CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); | ||
| if (_isStreamingBytes) { | ||
| if (_byteStreamHandler.eventSink) { | ||
| CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); | ||
| CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); | ||
|
|
||
| size_t imageWidth = CVPixelBufferGetWidth(pixelBuffer); | ||
| size_t imageHeight = CVPixelBufferGetHeight(pixelBuffer); | ||
| size_t imageWidth = CVPixelBufferGetWidth(pixelBuffer); | ||
| size_t imageHeight = CVPixelBufferGetHeight(pixelBuffer); | ||
|
|
||
| NSMutableArray *planes = [NSMutableArray array]; | ||
| NSMutableArray *planes = [NSMutableArray array]; | ||
|
|
||
| size_t planeCount = CVPixelBufferGetPlaneCount(pixelBuffer); | ||
| for (int i = 0; i < planeCount; i++) { | ||
| void *planeAddress = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, i); | ||
| size_t bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, i); | ||
| size_t height = CVPixelBufferGetHeightOfPlane(pixelBuffer, i); | ||
| size_t width = CVPixelBufferGetWidthOfPlane(pixelBuffer, i); | ||
| size_t planeCount = CVPixelBufferGetPlaneCount(pixelBuffer); | ||
| for (int i = 0; i < planeCount; i++) { | ||
| void *planeAddress = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, i); | ||
| size_t bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, i); | ||
| size_t height = CVPixelBufferGetHeightOfPlane(pixelBuffer, i); | ||
| size_t width = CVPixelBufferGetWidthOfPlane(pixelBuffer, i); | ||
|
|
||
| NSNumber *length = @(bytesPerRow * height); | ||
| NSData *bytes = [NSData dataWithBytes:planeAddress length:length.unsignedIntegerValue]; | ||
| NSNumber *length = @(bytesPerRow * height); | ||
| NSData *bytes = [NSData dataWithBytes:planeAddress length:length.unsignedIntegerValue]; | ||
|
|
||
| NSMutableDictionary *planeBuffer = [NSMutableDictionary dictionary]; | ||
| planeBuffer[@"bytesPerRow"] = @(bytesPerRow); | ||
| planeBuffer[@"width"] = @(width); | ||
| planeBuffer[@"height"] = @(height); | ||
| planeBuffer[@"bytes"] = [FlutterStandardTypedData typedDataWithBytes:bytes]; | ||
| NSMutableDictionary *planeBuffer = [NSMutableDictionary dictionary]; | ||
| planeBuffer[@"bytesPerRow"] = @(bytesPerRow); | ||
| planeBuffer[@"width"] = @(width); | ||
| planeBuffer[@"height"] = @(height); | ||
| planeBuffer[@"bytes"] = [FlutterStandardTypedData typedDataWithBytes:bytes]; | ||
|
|
||
| [planes addObject:planeBuffer]; | ||
| } | ||
| [planes addObject:planeBuffer]; | ||
| } | ||
|
|
||
| NSMutableDictionary *imageBuffer = [NSMutableDictionary dictionary]; | ||
| imageBuffer[@"width"] = [NSNumber numberWithUnsignedLong:imageWidth]; | ||
| imageBuffer[@"height"] = [NSNumber numberWithUnsignedLong:imageHeight]; | ||
| imageBuffer[@"format"] = @(videoFormat); | ||
| imageBuffer[@"planes"] = planes; | ||
| NSMutableDictionary *imageBuffer = [NSMutableDictionary dictionary]; | ||
| imageBuffer[@"width"] = [NSNumber numberWithUnsignedLong:imageWidth]; | ||
| imageBuffer[@"height"] = [NSNumber numberWithUnsignedLong:imageHeight]; | ||
| imageBuffer[@"format"] = @(videoFormat); | ||
| imageBuffer[@"planes"] = planes; | ||
|
|
||
| _byteStreamHandler.eventSink(imageBuffer); | ||
| _byteStreamHandler.eventSink(imageBuffer); | ||
|
|
||
| CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); | ||
| CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); | ||
| } | ||
| } | ||
| if (_isRecording) { | ||
| if (_videoWriter.status == AVAssetWriterStatusFailed) { | ||
|
|
@@ -336,10 +341,10 @@ - (CVPixelBufferRef)copyPixelBuffer { | |
| pixelBuffer = _latestPixelBuffer; | ||
| } | ||
|
|
||
| return [self convertYUVImageTOBGRA:pixelBuffer]; | ||
| return [self convertYUVImageToBGRA:pixelBuffer]; | ||
| } | ||
|
|
||
| - (CVPixelBufferRef)convertYUVImageTOBGRA:(CVPixelBufferRef)pixelBuffer { | ||
| - (CVPixelBufferRef)convertYUVImageToBGRA:(CVPixelBufferRef)pixelBuffer { | ||
| CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); | ||
|
|
||
| vImage_YpCbCrToARGB infoYpCbCrToARGB; | ||
|
|
@@ -369,30 +374,20 @@ - (CVPixelBufferRef)convertYUVImageTOBGRA:(CVPixelBufferRef)pixelBuffer { | |
| sourceChromaBuffer.width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 1); | ||
| sourceChromaBuffer.rowBytes = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1); | ||
|
|
||
| if (!destinationBuffer.height) { | ||
| vImageBuffer_Init(&destinationBuffer, sourceLumaBuffer.height, sourceLumaBuffer.width, 32, | ||
| kvImageNoFlags); | ||
| } | ||
|
|
||
| vImageConvert_420Yp8_CbCr8ToARGB8888(&sourceLumaBuffer, &sourceChromaBuffer, &destinationBuffer, | ||
| vImageConvert_420Yp8_CbCr8ToARGB8888(&sourceLumaBuffer, &sourceChromaBuffer, &_destinationBuffer, | ||
| &infoYpCbCrToARGB, NULL, 255, | ||
| kvImagePrintDiagnosticsToConsole); | ||
|
|
||
| CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); | ||
| CVPixelBufferRelease(pixelBuffer); | ||
|
|
||
| if (!conversionBuffer.height) { | ||
| vImageBuffer_Init(&conversionBuffer, sourceLumaBuffer.height, sourceLumaBuffer.width, 32, | ||
| kvImageNoFlags); | ||
| } | ||
|
|
||
| const uint8_t map[4] = {3, 2, 1, 0}; | ||
| vImagePermuteChannels_ARGB8888(&destinationBuffer, &conversionBuffer, map, kvImageNoFlags); | ||
| vImagePermuteChannels_ARGB8888(&_destinationBuffer, &_conversionBuffer, map, kvImageNoFlags); | ||
|
|
||
| CVPixelBufferRef newPixelBuffer = NULL; | ||
| CVPixelBufferCreateWithBytes(NULL, conversionBuffer.width, conversionBuffer.height, | ||
| kCVPixelFormatType_32BGRA, conversionBuffer.data, | ||
| conversionBuffer.rowBytes, NULL, NULL, NULL, &newPixelBuffer); | ||
| CVPixelBufferCreateWithBytes(NULL, _conversionBuffer.width, _conversionBuffer.height, | ||
| kCVPixelFormatType_32BGRA, _conversionBuffer.data, | ||
| _conversionBuffer.rowBytes, NULL, NULL, NULL, &newPixelBuffer); | ||
|
|
||
| return newPixelBuffer; | ||
| } | ||
|
|
||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.