[camera_android_camerax] Fix camera preview rotation when capture orientation is locked

packages/camera/camera_android_camerax/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.6.15
+
+* Fixes incorrect camera preview rotation when capture orientation is locked.
+
+## 0.6.14
+
+* Fixes incorrect camera preview rotation when capture orientation is unlocked.
+
 ## 0.6.13
 
 * Adds API support query for image streaming.

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/GeneratedCameraXLibrary.java

@@ -1441,9 +1441,6 @@ void requestCameraPermissions(
     @NonNull
     String getTempFilePath(@NonNull String prefix, @NonNull String suffix);
 
-    @NonNull
-    Boolean isPreviewPreTransformed();
-
     /** The codec used by SystemServicesHostApi. */
     static @NonNull MessageCodec<Object> getCodec() {
       return SystemServicesHostApiCodec.INSTANCE;
@@ -1511,29 +1508,6 @@ public void error(Throwable error) {
           channel.setMessageHandler(null);
         }
       }
-      {
-        BasicMessageChannel<Object> channel =
-            new BasicMessageChannel<>(
-                binaryMessenger,
-                "dev.flutter.pigeon.SystemServicesHostApi.isPreviewPreTransformed",
-                getCodec());
-        if (api != null) {
-          channel.setMessageHandler(
-              (message, reply) -> {
-                ArrayList<Object> wrapped = new ArrayList<Object>();
-                try {
-                  Boolean output = api.isPreviewPreTransformed();
-                  wrapped.add(0, output);
-                } catch (Throwable exception) {
-                  ArrayList<Object> wrappedError = wrapError(exception);
-                  wrapped = wrappedError;
-                }
-                reply.reply(wrapped);
-              });
-        } else {
-          channel.setMessageHandler(null);
-        }
-      }
     }
   }
   /** Generated class from Pigeon that represents Flutter messages that can be called from Java. */
@@ -1761,6 +1735,9 @@ void create(
 
     void setTargetRotation(@NonNull Long identifier, @NonNull Long rotation);
 
+    @NonNull
+    Boolean surfaceProducerHandlesCropAndRotation();
+
     /** The codec used by PreviewHostApi. */
     static @NonNull MessageCodec<Object> getCodec() {
       return PreviewHostApiCodec.INSTANCE;
@@ -1898,6 +1875,29 @@ static void setup(@NonNull BinaryMessenger binaryMessenger, @Nullable PreviewHos
           channel.setMessageHandler(null);
         }
       }
+      {
+        BasicMessageChannel<Object> channel =
+            new BasicMessageChannel<>(
+                binaryMessenger,
+                "dev.flutter.pigeon.PreviewHostApi.surfaceProducerHandlesCropAndRotation",
+                getCodec());
+        if (api != null) {
+          channel.setMessageHandler(
+              (message, reply) -> {
+                ArrayList<Object> wrapped = new ArrayList<Object>();
+                try {
+                  Boolean output = api.surfaceProducerHandlesCropAndRotation();
+                  wrapped.add(0, output);
+                } catch (Throwable exception) {
+                  ArrayList<Object> wrappedError = wrapError(exception);
+                  wrapped = wrappedError;
+                }
+                reply.reply(wrapped);
+              });
+        } else {
+          channel.setMessageHandler(null);
+        }
+      }
     }
   }
   /** Generated interface from Pigeon that represents a handler of messages from Flutter. */

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/PreviewHostApiImpl.java

@@ -155,7 +155,10 @@ String getProvideSurfaceErrorDescription(int resultCode) {
   public void releaseFlutterSurfaceTexture() {
     if (flutterSurfaceProducer != null) {
       flutterSurfaceProducer.release();
+      return;
     }
+    throw new IllegalStateException(
+        "releaseFlutterSurfaceTexture() cannot be called if the flutterSurfaceProducer for the camera preview has not yet been initialized.");
   }
 
   /** Returns the resolution information for the specified {@link Preview}. */
@@ -179,6 +182,16 @@ public void setTargetRotation(@NonNull Long identifier, @NonNull Long rotation)
     preview.setTargetRotation(rotation.intValue());
   }
 
+  @NonNull
+  @Override
+  public Boolean surfaceProducerHandlesCropAndRotation() {
+    if (flutterSurfaceProducer != null) {
+      return flutterSurfaceProducer.handlesCropAndRotation();
+    }
+    throw new IllegalStateException(
+        "surfaceProducerHandlesCropAndRotation() cannot be called if the flutterSurfaceProducer for the camera preview has not yet been initialized.");
+  }
+
   /** Retrieves the {@link Preview} instance associated with the specified {@code identifier}. */
   private Preview getPreviewInstance(@NonNull Long identifier) {
     return Objects.requireNonNull(instanceManager.getInstance(identifier));

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/SystemServicesHostApiImpl.java

@@ -6,7 +6,6 @@
 
 import android.app.Activity;
 import android.content.Context;
-import android.os.Build;
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 import androidx.annotation.VisibleForTesting;
@@ -104,18 +103,4 @@ public String getTempFilePath(@NonNull String prefix, @NonNull String suffix) {
           null);
     }
   }
-
-  /**
-   * Returns whether or not Impeller uses an {@code ImageReader} backend to provide a {@code
-   * Surface} to CameraX to build the preview. If it is backed by an {@code ImageReader}, then
-   * CameraX will not automatically apply the transformation needed to correct the preview.
-   *
-   * <p>This is determined by the engine, which approximately uses {@code SurfaceTexture}s on
-   * Android SDKs below 29.
-   */
-  @Override
-  @NonNull
-  public Boolean isPreviewPreTransformed() {
-    return Build.VERSION.SDK_INT < 29;
-  }
 }

packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/PreviewTest.java

@@ -263,6 +263,20 @@ public void setTargetRotation_makesCallToSetTargetRotation() {
     verify(mockPreview).setTargetRotation(targetRotation);
   }
 
+  @Test
+  public void
+      surfaceProducerHandlesCropAndRotation_returnsIfSurfaceProducerHandlesCropAndRotation() {
+    final PreviewHostApiImpl hostApi =
+        new PreviewHostApiImpl(mockBinaryMessenger, testInstanceManager, mockTextureRegistry);
+    final TextureRegistry.SurfaceProducer mockSurfaceProducer =
+        mock(TextureRegistry.SurfaceProducer.class);
+
+    hostApi.flutterSurfaceProducer = mockSurfaceProducer;
+    when(mockSurfaceProducer.handlesCropAndRotation()).thenReturn(true);
+
+    assertEquals(hostApi.surfaceProducerHandlesCropAndRotation(), true);
+  }
+
   // TODO(bparrishMines): Replace with inline calls to onSurfaceCleanup once available on stable;
   // see https://github.com/flutter/flutter/issues/16125. This separate method only exists to scope
   // the suppression.

packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/SystemServicesTest.java

@@ -5,9 +5,7 @@
 package io.flutter.plugins.camerax;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThrows;
-import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.mockStatic;
@@ -33,7 +31,6 @@
 import org.mockito.junit.MockitoJUnit;
 import org.mockito.junit.MockitoRule;
 import org.robolectric.RobolectricTestRunner;
-import org.robolectric.annotation.Config;
 
 @RunWith(RobolectricTestRunner.class)
 public class SystemServicesTest {
@@ -136,28 +133,4 @@ public void getTempFilePath_throwsRuntimeExceptionOnIOException() {
 
     mockedStaticFile.close();
   }
-
-  @Test
-  @Config(sdk = 28)
-  public void isPreviewPreTransformed_returnsTrueWhenRunningBelowSdk29() {
-    final SystemServicesHostApiImpl systemServicesHostApi =
-        new SystemServicesHostApiImpl(mockBinaryMessenger, mockInstanceManager, mockContext);
-    assertTrue(systemServicesHostApi.isPreviewPreTransformed());
-  }
-
-  @Test
-  @Config(sdk = 28)
-  public void isPreviewPreTransformed_returnsTrueWhenRunningSdk28() {
-    final SystemServicesHostApiImpl systemServicesHostApi =
-        new SystemServicesHostApiImpl(mockBinaryMessenger, mockInstanceManager, mockContext);
-    assertTrue(systemServicesHostApi.isPreviewPreTransformed());
-  }
-
-  @Test
-  @Config(sdk = 29)
-  public void isPreviewPreTransformed_returnsFalseWhenRunningAboveSdk28() {
-    final SystemServicesHostApiImpl systemServicesHostApi =
-        new SystemServicesHostApiImpl(mockBinaryMessenger, mockInstanceManager, mockContext);
-    assertFalse(systemServicesHostApi.isPreviewPreTransformed());
-  }
 }

packages/camera/camera_android_camerax/lib/src/android_camera_camerax.dart

@@ -47,6 +47,7 @@ import 'recording.dart';
 import 'resolution_filter.dart';
 import 'resolution_selector.dart';
 import 'resolution_strategy.dart';
+import 'rotated_preview.dart';
 import 'surface.dart';
 import 'system_services.dart';
 import 'use_case.dart';
@@ -183,12 +184,11 @@ class AndroidCameraCameraX extends CameraPlatform {
   /// set for the camera in use.
   static const String zoomStateNotSetErrorCode = 'zoomStateNotSet';
 
-  /// Whether or not the capture orientation is locked.
+  /// The current locked capture orientaiton.
   ///
-  /// Indicates a new target rotation should not be set as it has been locked by
-  /// [lockCaptureOrientation].
+  /// Null if the capture orientation is not locked.
   @visibleForTesting
-  bool captureOrientationLocked = false;
+  DeviceOrientation? lockedCaptureOrientation;
 
   /// Whether or not the default rotation for [UseCase]s needs to be set
   /// manually because the capture orientation was previously locked.
@@ -238,12 +238,21 @@ class AndroidCameraCameraX extends CameraPlatform {
   late bool cameraIsFrontFacing;
 
   /// The camera sensor orientation.
+  ///
+  /// This can change if the camera being used changes. Also, it is independent
+  /// of the device orientation or user interface orientation.
   @visibleForTesting
-  late int sensorOrientation;
+  late double sensorOrientationDegrees;
+
+  /// Whether or not the Android surface producer automatically handles
+  /// correcting the rotation of camera previews for the device this plugin runs on.
+  late bool _handlesCropAndRotation;
 
-  /// Subscription for listening to changes in device orientation.
-  StreamSubscription<DeviceOrientationChangedEvent>?
-      _subscriptionForDeviceOrientationChanges;
+  /// The initial orientation of the device when the camera is created.
+  late DeviceOrientation _initialDeviceOrientation;
+
+  /// Correctly rotated preview [Widget], if used.
+  RotatedPreview? _rotatedPreview;
 
   /// Returns list of all available cameras and their descriptions.
   @override
@@ -380,10 +389,10 @@ class AndroidCameraCameraX extends CameraPlatform {
 
     // Retrieve info required for correcting the rotation of the camera preview
     // if necessary.
-
-    final Camera2CameraInfo camera2CameraInfo =
-        await proxy.getCamera2CameraInfo(cameraInfo!);
-    sensorOrientation = await proxy.getSensorOrientation(camera2CameraInfo);
+    sensorOrientationDegrees = cameraDescription.sensorOrientation.toDouble();
+    _handlesCropAndRotation =
+        await proxy.previewSurfaceProducerHandlesCropAndRotation(preview!);
+    _initialDeviceOrientation = await proxy.getUiOrientation();
 
     return flutterSurfaceTextureId;
   }
@@ -444,7 +453,6 @@ class AndroidCameraCameraX extends CameraPlatform {
     await liveCameraState?.removeObservers();
     processCameraProvider?.unbindAll();
     await imageAnalysis?.clearAnalyzer();
-    await _subscriptionForDeviceOrientationChanges?.cancel();
   }
 
   /// The camera has been initialized.
@@ -496,7 +504,7 @@ class AndroidCameraCameraX extends CameraPlatform {
     // if orientation is ever unlocked and (2) the capture orientation is locked
     // and should not be changed until unlocked.
     shouldSetDefaultRotation = true;
-    captureOrientationLocked = true;
+    lockedCaptureOrientation = orientation;
 
     // Get target rotation based on locked orientation.
     final int targetLockedRotation =
@@ -506,13 +514,19 @@ class AndroidCameraCameraX extends CameraPlatform {
     await imageCapture!.setTargetRotation(targetLockedRotation);
     await imageAnalysis!.setTargetRotation(targetLockedRotation);
     await videoCapture!.setTargetRotation(targetLockedRotation);
+
+    // Update preview to maintain correct rotation, if needed.
+    _rotatedPreview?.setLockedCaptureOrientation(lockedCaptureOrientation);
   }
 
   /// Unlocks the capture orientation.
   @override
   Future<void> unlockCaptureOrientation(int cameraId) async {
     // Flag that default rotation should be set for UseCases as needed.
-    captureOrientationLocked = false;
+    lockedCaptureOrientation = null;
+
+    // Update preview to maintain correct rotation, if needed.
+    _rotatedPreview?.setLockedCaptureOrientation(null);
   }
 
   /// Sets the exposure point for automatically determining the exposure values.
@@ -836,7 +850,34 @@ class AndroidCameraCameraX extends CameraPlatform {
       );
     }
 
-    return Texture(textureId: cameraId);
+    final Widget preview = Texture(textureId: cameraId);
+
+    if (_handlesCropAndRotation) {
+      return preview;
+    }
+
+    final Stream<DeviceOrientation> deviceOrientationStream =
+        onDeviceOrientationChanged()
+            .map((DeviceOrientationChangedEvent e) => e.orientation);
+    if (cameraIsFrontFacing) {
+      _rotatedPreview = RotatedPreview.frontFacingCamera(
+        _initialDeviceOrientation,
+        deviceOrientationStream,
+        sensorOrientationDegrees: sensorOrientationDegrees,
+        initialLockedCaptureOrientation: lockedCaptureOrientation,
+        child: preview,
+      );
+    } else {
+      _rotatedPreview = RotatedPreview.backFacingCamera(
+        _initialDeviceOrientation,
+        deviceOrientationStream,
+        sensorOrientationDegrees: sensorOrientationDegrees,
+        initialLockedCaptureOrientation: lockedCaptureOrientation,
+        child: preview,
+      );
+    }
+
+    return _rotatedPreview!;
   }
 
   /// Captures an image and returns the file where it was saved.
@@ -856,7 +897,7 @@ class AndroidCameraCameraX extends CameraPlatform {
 
     // Set target rotation to default CameraX rotation only if capture
     // orientation not locked.
-    if (!captureOrientationLocked && shouldSetDefaultRotation) {
+    if (lockedCaptureOrientation == null && shouldSetDefaultRotation) {
       await imageCapture!
           .setTargetRotation(await proxy.getDefaultDisplayRotation());
     }
@@ -985,7 +1026,7 @@ class AndroidCameraCameraX extends CameraPlatform {
 
     // Set target rotation to default CameraX rotation only if capture
     // orientation not locked.
-    if (!captureOrientationLocked && shouldSetDefaultRotation) {
+    if (lockedCaptureOrientation == null && shouldSetDefaultRotation) {
       await videoCapture!
           .setTargetRotation(await proxy.getDefaultDisplayRotation());
     }
@@ -1116,7 +1157,7 @@ class AndroidCameraCameraX extends CameraPlatform {
 
     // Set target rotation to default CameraX rotation only if capture
     // orientation not locked.
-    if (!captureOrientationLocked && shouldSetDefaultRotation) {
+    if (lockedCaptureOrientation == null && shouldSetDefaultRotation) {
       await imageAnalysis!
           .setTargetRotation(await proxy.getDefaultDisplayRotation());
     }