Switch MLKitAnalyzer to use the new sensorToBuffer API
Instead of relying on the deprecated CoordinateTransform, we pass the sensorToBuffer matrix to MLKitAnalyzer
Bug: 285043389
Test: manual test and ./gradlew bOS
Change-Id: I419e6016da251ec6d2c42068ee14dbc5d2418b1c
diff --git a/camera/camera-core/src/main/java/androidx/camera/core/impl/utils/TransformUtils.java b/camera/camera-core/src/main/java/androidx/camera/core/impl/utils/TransformUtils.java
index 4454f71..2e41dc8 100644
--- a/camera/camera-core/src/main/java/androidx/camera/core/impl/utils/TransformUtils.java
+++ b/camera/camera-core/src/main/java/androidx/camera/core/impl/utils/TransformUtils.java
@@ -141,6 +141,22 @@
}
/**
+ * Rotates {@link SizeF} according to the rotation degrees.
+ *
+ * <p> A 640, 480 rect rotated 90 degrees clockwise will become a 480, 640 rect.
+ */
+ @NonNull
+ public static RectF rotateRect(@NonNull RectF rect, int rotationDegrees) {
+ Preconditions.checkArgument(rotationDegrees % 90 == 0,
+ "Invalid rotation degrees: " + rotationDegrees);
+ if (is90or270(within360(rotationDegrees))) {
+ return new RectF(0, 0, /*right=*/rect.height(), /*bottom=*/rect.width());
+ } else {
+ return rect;
+ }
+ }
+
+ /**
* Gets the size after cropping and rotating.
*
* @return rotated size
diff --git a/camera/camera-mlkit-vision/src/main/java/androidx/camera/mlkit/vision/MlKitAnalyzer.java b/camera/camera-mlkit-vision/src/main/java/androidx/camera/mlkit/vision/MlKitAnalyzer.java
index 55b0cfd..605c78a 100644
--- a/camera/camera-mlkit-vision/src/main/java/androidx/camera/mlkit/vision/MlKitAnalyzer.java
+++ b/camera/camera-mlkit-vision/src/main/java/androidx/camera/mlkit/vision/MlKitAnalyzer.java
@@ -16,6 +16,8 @@
package androidx.camera.mlkit.vision;
import static androidx.camera.core.ImageAnalysis.COORDINATE_SYSTEM_ORIGINAL;
+import static androidx.camera.core.impl.utils.TransformUtils.getRectToRect;
+import static androidx.camera.core.impl.utils.TransformUtils.rotateRect;
import static com.google.android.gms.common.internal.Preconditions.checkArgument;
import static com.google.mlkit.vision.interfaces.Detector.TYPE_BARCODE_SCANNING;
@@ -23,6 +25,7 @@
import static com.google.mlkit.vision.interfaces.Detector.TYPE_TEXT_RECOGNITION;
import android.graphics.Matrix;
+import android.graphics.RectF;
import android.media.Image;
import android.util.Size;
@@ -35,9 +38,7 @@
import androidx.camera.core.ImageProxy;
import androidx.camera.core.Logger;
import androidx.camera.view.TransformExperimental;
-import androidx.camera.view.transform.CoordinateTransform;
import androidx.camera.view.transform.ImageProxyTransformFactory;
-import androidx.camera.view.transform.OutputTransform;
import androidx.core.util.Consumer;
import com.google.android.gms.tasks.Task;
@@ -61,8 +62,8 @@
* <p> This class handles the coordinate transformation between ML Kit output and the target
* coordinate system. Using the {@code targetCoordinateSystem} set in the constructor, it
* calculates the {@link Matrix} with the value provided by CameraX via
- * {@link ImageAnalysis.Analyzer#updateTransform} and forwards it to the ML Kit {@code Detector}. The
- * coordinates returned by MLKit will be in the specified coordinate system.
+ * {@link ImageAnalysis.Analyzer#updateTransform} and forwards it to the ML Kit {@code Detector}.
+ * The coordinates returned by MLKit will be in the specified coordinate system.
*
* <p> This class is designed to work seamlessly with the {@code CameraController} class in
* camera-view. When used with {@link ImageAnalysis} in camera-core, the following scenarios may
@@ -155,7 +156,7 @@
@OptIn(markerClass = TransformExperimental.class)
public final void analyze(@NonNull ImageProxy imageProxy) {
// By default, the matrix is identity for COORDINATE_SYSTEM_ORIGINAL.
- Matrix transform = new Matrix();
+ Matrix analysisToTarget = new Matrix();
if (mTargetCoordinateSystem != COORDINATE_SYSTEM_ORIGINAL) {
// Calculate the transform if not COORDINATE_SYSTEM_ORIGINAL.
Matrix sensorToTarget = mSensorToTarget;
@@ -166,16 +167,24 @@
imageProxy.close();
return;
}
- OutputTransform analysisTransform =
- mImageAnalysisTransformFactory.getOutputTransform(imageProxy);
- Size cropRectSize = new Size(imageProxy.getCropRect().width(),
- imageProxy.getCropRect().height());
- CoordinateTransform coordinateTransform = new CoordinateTransform(analysisTransform,
- new OutputTransform(sensorToTarget, cropRectSize));
- coordinateTransform.transform(transform);
+ Matrix sensorToAnalysis =
+ new Matrix(imageProxy.getImageInfo().getSensorToBufferTransformMatrix());
+ // Calculate the rotation added by ML Kit.
+ RectF sourceRect = new RectF(0, 0, imageProxy.getWidth(),
+ imageProxy.getHeight());
+ RectF bufferRect = rotateRect(sourceRect,
+ imageProxy.getImageInfo().getRotationDegrees());
+ Matrix analysisToMlKitRotation = getRectToRect(sourceRect, bufferRect,
+ imageProxy.getImageInfo().getRotationDegrees());
+ // Concat the MLKit transformation with sensor to Analysis.
+ sensorToAnalysis.postConcat(analysisToMlKitRotation);
+ // Invert to get analysis to sensor.
+ sensorToAnalysis.invert(analysisToTarget);
+ // Concat sensor to target to get analysisToTarget.
+ analysisToTarget.postConcat(sensorToTarget);
}
// Detect the image recursively, starting from index 0.
- detectRecursively(imageProxy, 0, transform, new HashMap<>(), new HashMap<>());
+ detectRecursively(imageProxy, 0, analysisToTarget, new HashMap<>(), new HashMap<>());
}
/**
diff --git a/camera/camera-mlkit-vision/src/test/java/androidx/camera/mlkit/vision/MlKitAnalyzerTest.kt b/camera/camera-mlkit-vision/src/test/java/androidx/camera/mlkit/vision/MlKitAnalyzerTest.kt
index c4dcc0f..73919a9 100644
--- a/camera/camera-mlkit-vision/src/test/java/androidx/camera/mlkit/vision/MlKitAnalyzerTest.kt
+++ b/camera/camera-mlkit-vision/src/test/java/androidx/camera/mlkit/vision/MlKitAnalyzerTest.kt
@@ -16,13 +16,14 @@
package androidx.camera.mlkit.vision
-import android.graphics.Matrix
import android.graphics.Rect
+import android.graphics.RectF
import android.media.Image
import android.os.Build
import android.util.Size
import androidx.camera.core.ImageAnalysis
import androidx.camera.core.ImageProxy
+import androidx.camera.core.impl.utils.TransformUtils.getRectToRect
import androidx.camera.core.impl.utils.executor.CameraXExecutors.directExecutor
import androidx.camera.testing.fakes.FakeImageInfo
import androidx.camera.testing.fakes.FakeImageProxy
@@ -51,7 +52,14 @@
private const val RETURN_VALUE = "return value"
private const val TIMESTAMP = 100L
private const val ROTATION_DEGREES = 180
- private val CROP_RECT = Rect(0, 0, 640, 480)
+ private val SENSOR_RECT = Rect(0, 0, 4000, 3000)
+ private val VIEW_RECT = Rect(0, 0, 1024, 768)
+ private val IMAGE_ANALYSIS_RECT = Rect(0, 0, 640, 480)
+ private val SENSOR_TO_BUFFER = getRectToRect(
+ RectF(SENSOR_RECT),
+ RectF(IMAGE_ANALYSIS_RECT),
+ 0
+ )
}
@Test
@@ -150,7 +158,7 @@
@Test
fun transformationAndRotationIsCorrect() {
// Arrange.
- val additionalTransform = Matrix()
+ val additionalTransform = getRectToRect(RectF(SENSOR_RECT), RectF(VIEW_RECT), 0)
additionalTransform.setScale(2F, 2F)
val detector = FakeDetector(RETURN_VALUE, TYPE_BARCODE_SCANNING)
val analyzer = MlKitAnalyzer(
@@ -165,25 +173,23 @@
analyzer.analyze(createFakeImageProxy())
// Assert that the matrix is passed to the MLKit detector.
- val expected = floatArrayOf(-0.00625F, 0F, 2.0F, 0F, -0.0083333F, 2.0F, 0.0F, 0.0F, 1.0F)
+ val expected = floatArrayOf(-12.5F, 0F, 8000F, 0F, -12.5F, 6000F, 0F, 0F, 1F)
val actual = FloatArray(9)
detector.latestMatrix!!.getValues(actual)
- actual.forEachIndexed { i, element ->
- // Assert allowing float rounding error.
- assertThat(expected[i]).isWithin(1E-4F).of(element)
- }
+ assertThat(actual).usingTolerance(1E-3).containsExactly(expected).inOrder()
}
private fun createFakeImageProxy(): ImageProxy {
val imageInfo = FakeImageInfo()
imageInfo.timestamp = TIMESTAMP
imageInfo.rotationDegrees = ROTATION_DEGREES
+ imageInfo.sensorToBufferTransformMatrix = SENSOR_TO_BUFFER
val imageProxy = FakeImageProxy(imageInfo)
imageProxy.image = mock(Image::class.java)
- imageProxy.width = CROP_RECT.width()
- imageProxy.height = CROP_RECT.height()
- imageProxy.setCropRect(CROP_RECT)
+ imageProxy.width = IMAGE_ANALYSIS_RECT.width()
+ imageProxy.height = IMAGE_ANALYSIS_RECT.height()
+ imageProxy.setCropRect(IMAGE_ANALYSIS_RECT)
return imageProxy
}
diff --git a/camera/camera-view/src/main/java/androidx/camera/view/CameraController.java b/camera/camera-view/src/main/java/androidx/camera/view/CameraController.java
index 9c2f968..d935dd4 100644
--- a/camera/camera-view/src/main/java/androidx/camera/view/CameraController.java
+++ b/camera/camera-view/src/main/java/androidx/camera/view/CameraController.java
@@ -80,7 +80,6 @@
import androidx.camera.video.Recording;
import androidx.camera.video.VideoCapture;
import androidx.camera.video.VideoRecordEvent;
-import androidx.camera.view.transform.OutputTransform;
import androidx.camera.view.video.AudioConfig;
import androidx.core.content.PermissionChecker;
import androidx.core.util.Consumer;
@@ -1098,16 +1097,16 @@
@OptIn(markerClass = {TransformExperimental.class})
@MainThread
- void updatePreviewViewTransform(@Nullable OutputTransform outputTransform) {
+ void updatePreviewViewTransform(@Nullable Matrix matrix) {
checkMainThread();
if (mAnalysisAnalyzer == null) {
return;
}
- if (outputTransform == null) {
+ if (matrix == null) {
mAnalysisAnalyzer.updateTransform(null);
} else if (mAnalysisAnalyzer.getTargetCoordinateSystem()
== COORDINATE_SYSTEM_VIEW_REFERENCED) {
- mAnalysisAnalyzer.updateTransform(outputTransform.getMatrix());
+ mAnalysisAnalyzer.updateTransform(matrix);
}
}
diff --git a/camera/camera-view/src/main/java/androidx/camera/view/PreviewView.java b/camera/camera-view/src/main/java/androidx/camera/view/PreviewView.java
index f2a95b5..cb853a1 100644
--- a/camera/camera-view/src/main/java/androidx/camera/view/PreviewView.java
+++ b/camera/camera-view/src/main/java/androidx/camera/view/PreviewView.java
@@ -660,7 +660,7 @@
mPreviewViewMeteringPointFactory.recalculate(new Size(getWidth(), getHeight()),
getLayoutDirection());
if (mCameraController != null) {
- mCameraController.updatePreviewViewTransform(getOutputTransform());
+ mCameraController.updatePreviewViewTransform(getSensorToViewTransform());
}
}
diff --git a/camera/camera-view/src/test/java/androidx/camera/view/CameraControllerTest.kt b/camera/camera-view/src/test/java/androidx/camera/view/CameraControllerTest.kt
index 22a51d3..5ac8b85 100644
--- a/camera/camera-view/src/test/java/androidx/camera/view/CameraControllerTest.kt
+++ b/camera/camera-view/src/test/java/androidx/camera/view/CameraControllerTest.kt
@@ -44,7 +44,6 @@
import androidx.camera.video.Quality
import androidx.camera.video.QualitySelector
import androidx.camera.view.CameraController.COORDINATE_SYSTEM_VIEW_REFERENCED
-import androidx.camera.view.transform.OutputTransform
import androidx.concurrent.futures.CallbackToFutureAdapter
import androidx.test.annotation.UiThreadTest
import androidx.test.core.app.ApplicationProvider
@@ -277,10 +276,7 @@
}
}
controller.setImageAnalysisAnalyzer(mainThreadExecutor(), analyzer)
- val outputTransform = previewTransform?.let {
- OutputTransform(it, Size(1, 1))
- }
- controller.updatePreviewViewTransform(outputTransform)
+ controller.updatePreviewViewTransform(previewTransform)
return matrix
}
