Add Camera Extensions Realtime Latency Estimate API
- adds new realtime latency estimate api to image capture
- the realtime latency can be obtained if the attached session processor supports realtime latency estimates; for example the image capture is associated with a camera extensions session and the current extension supports realtime latency estimates
Bug: 289261257
Test: ./gradlew :camera:camera-extensions:connectedDebugAndroidTest, ./gradlew :camera:camera-core:testDebugUnitTest
Change-Id: I8194ba914e0d4cb408515f10e1d8db589ad6a959
diff --git a/camera/camera-core/src/main/java/androidx/camera/core/ImageCapture.java b/camera/camera-core/src/main/java/androidx/camera/core/ImageCapture.java
index d5f554e..dd99ca8 100644
--- a/camera/camera-core/src/main/java/androidx/camera/core/ImageCapture.java
+++ b/camera/camera-core/src/main/java/androidx/camera/core/ImageCapture.java
@@ -105,6 +105,7 @@
import androidx.camera.core.impl.MutableOptionsBundle;
import androidx.camera.core.impl.OptionsBundle;
import androidx.camera.core.impl.SessionConfig;
+import androidx.camera.core.impl.SessionProcessor;
import androidx.camera.core.impl.StreamSpec;
import androidx.camera.core.impl.UseCaseConfig;
import androidx.camera.core.impl.UseCaseConfigFactory;
@@ -1154,6 +1155,37 @@
}
/**
+ * Returns an estimate of the capture and processing sequence duration based on the current
+ * camera configuration and scene conditions. The value will vary as the scene and/or camera
+ * configuration change.
+ *
+ * <p>The processing estimate can vary based on device processing load.
+ *
+ * <p>If the image capture latency estimate is not supported then
+ * {@link ImageCaptureLatencyEstimate#UNDEFINED_IMAGE_CAPTURE_LATENCY} is returned. If the
+ * capture latency is not supported then the capture latency component will be
+ * {@link ImageCaptureLatencyEstimate#UNDEFINED_CAPTURE_LATENCY}. If the processing
+ * latency is not supported then the processing latency component will be
+ * {@link ImageCaptureLatencyEstimate#UNDEFINED_PROCESSING_LATENCY}.
+ **/
+ @RestrictTo(Scope.LIBRARY_GROUP)
+ @NonNull
+ public ImageCaptureLatencyEstimate getRealtimeCaptureLatencyEstimate() {
+ final CameraInternal camera = getCamera();
+ if (camera == null) {
+ return ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY;
+ }
+
+ final CameraConfig config = camera.getExtendedConfig();
+ final SessionProcessor sessionProcessor = config.getSessionProcessor();
+ final Pair<Long, Long> latencyEstimate = sessionProcessor.getRealtimeCaptureLatency();
+ if (latencyEstimate == null) {
+ return ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY;
+ }
+ return new ImageCaptureLatencyEstimate(latencyEstimate.first, latencyEstimate.second);
+ }
+
+ /**
* Describes the error that occurred during an image capture operation (such as {@link
* ImageCapture#takePicture(Executor, OnImageCapturedCallback)}).
*
@@ -2367,6 +2399,7 @@
* Sets the {@link DynamicRange}.
*
* <p>This is currently only exposed to internally set the dynamic range to SDR.
+ *
* @return The current Builder.
* @see DynamicRange
*/
diff --git a/camera/camera-core/src/main/java/androidx/camera/core/ImageCaptureLatencyEstimate.java b/camera/camera-core/src/main/java/androidx/camera/core/ImageCaptureLatencyEstimate.java
new file mode 100644
index 0000000..4c5c02d
--- /dev/null
+++ b/camera/camera-core/src/main/java/androidx/camera/core/ImageCaptureLatencyEstimate.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.camera.core;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.RequiresApi;
+import androidx.annotation.RestrictTo;
+
+import java.util.Objects;
+
+/**
+ * Defines the estimated duration an image capture will take capturing and processing for the
+ * current scene condition and/or camera configuration.
+ *
+ * <p>The estimate comprises of two components: {@link #captureLatencyMillis},
+ * {@link #processingLatencyMillis}
+ */
+@RestrictTo(RestrictTo.Scope.LIBRARY_GROUP)
+@RequiresApi(21) // TODO(b/200306659): Remove and replace with annotation on package-info.java
+public class ImageCaptureLatencyEstimate {
+ /** The capture latency is unsupported or undefined */
+ public static final long UNDEFINED_CAPTURE_LATENCY = -1;
+
+ /** The processing latency is unsupported or undefined */
+ public static final long UNDEFINED_PROCESSING_LATENCY = -1;
+
+ /** The image capture latency estimate is unsupported or undefined */
+ @NonNull
+ public static final ImageCaptureLatencyEstimate UNDEFINED_IMAGE_CAPTURE_LATENCY =
+ new ImageCaptureLatencyEstimate(UNDEFINED_CAPTURE_LATENCY,
+ UNDEFINED_PROCESSING_LATENCY);
+
+ /**
+ * The estimated duration in milliseconds from when the camera begins capturing frames to the
+ * moment the camera has completed capturing frames. If this estimate is not supported or not
+ * available then it will be {@link #UNDEFINED_CAPTURE_LATENCY}.
+ */
+ public final long captureLatencyMillis;
+
+ /**
+ * The estimated duration in milliseconds from when the processing begins until the processing
+ * has completed and the final processed capture is available. If this estimate is not supported
+ * or not available then it will be {@link #UNDEFINED_PROCESSING_LATENCY}.
+ */
+ public final long processingLatencyMillis;
+
+ ImageCaptureLatencyEstimate(long captureLatencyMillis, long processingLatencyMillis) {
+ this.captureLatencyMillis = captureLatencyMillis;
+ this.processingLatencyMillis = processingLatencyMillis;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof ImageCaptureLatencyEstimate)) return false;
+ ImageCaptureLatencyEstimate that = (ImageCaptureLatencyEstimate) o;
+ return captureLatencyMillis == that.captureLatencyMillis
+ && processingLatencyMillis == that.processingLatencyMillis;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(captureLatencyMillis, processingLatencyMillis);
+ }
+
+ @NonNull
+ @Override
+ public String toString() {
+ return "captureLatencyMillis=" + captureLatencyMillis
+ + ", processingLatencyMillis=" + processingLatencyMillis;
+ }
+}
diff --git a/camera/camera-core/src/main/java/androidx/camera/core/impl/SessionProcessor.java b/camera/camera-core/src/main/java/androidx/camera/core/impl/SessionProcessor.java
index 70b5c2b..f644291 100644
--- a/camera/camera-core/src/main/java/androidx/camera/core/impl/SessionProcessor.java
+++ b/camera/camera-core/src/main/java/androidx/camera/core/impl/SessionProcessor.java
@@ -18,6 +18,7 @@
import android.hardware.camera2.CaptureResult;
import android.media.ImageReader;
+import android.util.Pair;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
@@ -137,6 +138,30 @@
}
/**
+ * Returns the dynamically calculated capture latency pair in milliseconds.
+ *
+ * The measurement is expected to take in to account dynamic parameters such as the current
+ * scene, the state of 3A algorithms, the state of internal HW modules and return a more
+ * accurate assessment of the capture and/or processing latency.</p>
+ *
+ * @return pair that includes the estimated input frame/frames camera capture latency as the
+ * first field. This is the time between {@link CaptureCallback#onCaptureStarted} and
+ * {@link CaptureCallback#onCaptureProcessStarted}. The second field value includes the
+ * estimated post-processing latency. This is the time between
+ * {@link CaptureCallback#onCaptureProcessStarted} until the processed frame returns back to the
+ * client registered surface.
+ * Both first and second values will be in milliseconds. The total still capture latency will be
+ * the sum of both the first and second values of the pair.
+ * The pair is expected to be null if the dynamic latency estimation is not supported.
+ * If clients have not configured a still capture output, then this method can also return a
+ * null pair.
+ */
+ @Nullable
+ default Pair<Long, Long> getRealtimeCaptureLatency() {
+ return null;
+ }
+
+ /**
* Callback for {@link #startRepeating} and {@link #startCapture}.
*/
interface CaptureCallback {
diff --git a/camera/camera-extensions/src/androidTest/java/androidx/camera/extensions/internal/AdvancedSessionProcessorTest.kt b/camera/camera-extensions/src/androidTest/java/androidx/camera/extensions/internal/AdvancedSessionProcessorTest.kt
index eb342e6..0570e39 100644
--- a/camera/camera-extensions/src/androidTest/java/androidx/camera/extensions/internal/AdvancedSessionProcessorTest.kt
+++ b/camera/camera-extensions/src/androidTest/java/androidx/camera/extensions/internal/AdvancedSessionProcessorTest.kt
@@ -44,6 +44,7 @@
import android.media.ImageReader
import android.media.ImageWriter
import android.os.Build
+import android.util.Pair
import android.util.Size
import android.view.Surface
import androidx.annotation.RequiresApi
@@ -225,6 +226,22 @@
fakeSessionProcessImpl.assertStartTriggerIsCalledWithParameters(parametersMap)
}
+ @Test
+ fun getRealtimeLatencyEstimate_advancedSessionProcessorInvokesSessionProcessorImpl() =
+ runBlocking {
+ val fakeSessionProcessImpl = object : SessionProcessorImpl by FakeSessionProcessImpl() {
+ override fun getRealtimeCaptureLatency(): Pair<Long, Long> = Pair(1000L, 10L)
+ }
+ val advancedSessionProcessor = AdvancedSessionProcessor(
+ fakeSessionProcessImpl, emptyList(), context
+ )
+
+ val realtimeCaptureLatencyEstimate = advancedSessionProcessor.realtimeCaptureLatency
+
+ assertThat(realtimeCaptureLatencyEstimate?.first).isEqualTo(1000L)
+ assertThat(realtimeCaptureLatencyEstimate?.second).isEqualTo(10L)
+ }
+
private suspend fun assumeAllowsSharedSurface() = withContext(Dispatchers.Main) {
val imageReader = ImageReader.newInstance(640, 480, ImageFormat.YUV_420_888, 2)
val maxSharedSurfaceCount =
diff --git a/camera/camera-extensions/src/androidTest/java/androidx/camera/extensions/internal/sessionprocessor/BasicExtenderSessionProcessorTest.kt b/camera/camera-extensions/src/androidTest/java/androidx/camera/extensions/internal/sessionprocessor/BasicExtenderSessionProcessorTest.kt
index 1634996..b1b7d73 100644
--- a/camera/camera-extensions/src/androidTest/java/androidx/camera/extensions/internal/sessionprocessor/BasicExtenderSessionProcessorTest.kt
+++ b/camera/camera-extensions/src/androidTest/java/androidx/camera/extensions/internal/sessionprocessor/BasicExtenderSessionProcessorTest.kt
@@ -337,6 +337,20 @@
)
}
+ @Test
+ fun getRealtimeCaptureLatencyEstimate_invokesCaptureExtenderImpl(): Unit = runBlocking {
+ assumeTrue(hasCaptureProcessor)
+ fakeCaptureExtenderImpl = object : FakeImageCaptureExtenderImpl(hasCaptureProcessor) {
+ override fun getRealtimeCaptureLatency(): Pair<Long, Long> = Pair(1000L, 10L)
+ }
+
+ basicExtenderSessionProcessor = BasicExtenderSessionProcessor(
+ fakePreviewExtenderImpl, fakeCaptureExtenderImpl, emptyList(), emptyList(), context
+ )
+
+ assertThat(basicExtenderSessionProcessor.realtimeCaptureLatency).isEqualTo(Pair(1000L, 10L))
+ }
+
class ResultMonitor {
private var latch: CountDownLatch? = null
private var keyToCheck: CaptureRequest.Key<*>? = null
@@ -833,7 +847,7 @@
}
}
- private class FakeImageCaptureExtenderImpl(
+ private open class FakeImageCaptureExtenderImpl(
private val hasCaptureProcessor: Boolean = false,
private val throwErrorOnProcess: Boolean = false
) : ImageCaptureExtenderImpl, FakeExtenderStateListener() {
diff --git a/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/Version.java b/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/Version.java
index c740199..f34287a 100644
--- a/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/Version.java
+++ b/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/Version.java
@@ -39,6 +39,7 @@
public static final Version VERSION_1_1 = Version.create(1, 1, 0, "");
public static final Version VERSION_1_2 = Version.create(1, 2, 0, "");
public static final Version VERSION_1_3 = Version.create(1, 3, 0, "");
+ public static final Version VERSION_1_4 = Version.create(1, 4, 0, "");
private static final Pattern VERSION_STRING_PATTERN =
Pattern.compile("(\\d+)(?:\\.(\\d+))(?:\\.(\\d+))(?:\\-(.+))?");
diff --git a/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/AdvancedSessionProcessor.java b/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/AdvancedSessionProcessor.java
index a9c4432..85c344d 100644
--- a/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/AdvancedSessionProcessor.java
+++ b/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/AdvancedSessionProcessor.java
@@ -23,6 +23,7 @@
import android.hardware.camera2.CaptureResult;
import android.hardware.camera2.TotalCaptureResult;
import android.media.Image;
+import android.util.Pair;
import android.util.Size;
import android.view.Surface;
@@ -187,6 +188,16 @@
mImpl.abortCapture(captureSequenceId);
}
+ @Nullable
+ @Override
+ public Pair<Long, Long> getRealtimeCaptureLatency() {
+ if (ClientVersion.isMinimumCompatibleVersion(Version.VERSION_1_4)
+ && ExtensionVersion.isMinimumCompatibleVersion(Version.VERSION_1_4)) {
+ return mImpl.getRealtimeCaptureLatency();
+ }
+ return null;
+ }
+
/**
* Adapter to transform a {@link OutputSurface} to a {@link OutputSurfaceImpl}.
*/
diff --git a/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/BasicExtenderSessionProcessor.java b/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/BasicExtenderSessionProcessor.java
index 649640c..51fe764 100644
--- a/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/BasicExtenderSessionProcessor.java
+++ b/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/BasicExtenderSessionProcessor.java
@@ -647,4 +647,14 @@
return captureSequenceId;
}
+
+ @Nullable
+ @Override
+ public Pair<Long, Long> getRealtimeCaptureLatency() {
+ if (ClientVersion.isMinimumCompatibleVersion(Version.VERSION_1_4)
+ && ExtensionVersion.isMinimumCompatibleVersion(Version.VERSION_1_4)) {
+ return mImageCaptureExtenderImpl.getRealtimeCaptureLatency();
+ }
+ return null;
+ }
}
diff --git a/camera/camera-testlib-extensions/src/main/java/androidx/camera/extensions/impl/ImageCaptureExtenderImpl.java b/camera/camera-testlib-extensions/src/main/java/androidx/camera/extensions/impl/ImageCaptureExtenderImpl.java
index 6ed8f4d..f235fbe 100644
--- a/camera/camera-testlib-extensions/src/main/java/androidx/camera/extensions/impl/ImageCaptureExtenderImpl.java
+++ b/camera/camera-testlib-extensions/src/main/java/androidx/camera/extensions/impl/ImageCaptureExtenderImpl.java
@@ -170,4 +170,27 @@
*/
@NonNull
List<CaptureResult.Key> getAvailableCaptureResultKeys();
+
+ /**
+ * Returns the dynamically calculated capture latency pair in milliseconds.
+ *
+ * <p>In contrast to {@link #getEstimatedCaptureLatencyRange} this method is guaranteed to be
+ * called after the camera capture session is initialized and camera preview is enabled.
+ * The measurement is expected to take in to account dynamic parameters such as the current
+ * scene, the state of 3A algorithms, the state of internal HW modules and return a more
+ * accurate assessment of the still capture latency.</p>
+ *
+ * @return pair that includes the estimated input frame/frames camera capture latency as the
+ * first field and the estimated post-processing latency {@link CaptureProcessorImpl#process}
+ * as the second pair field. Both first and second fields will be in milliseconds. The total
+ * still capture latency will be the sum of both the first and second values.
+ * The pair is expected to be null if the dynamic latency estimation is not supported.
+ * If clients have not configured a still capture output, then this method can also return a
+ * null pair.
+ * @since 1.4
+ */
+ @Nullable
+ default Pair<Long, Long> getRealtimeCaptureLatency() {
+ return null;
+ };
}
diff --git a/camera/camera-testlib-extensions/src/main/java/androidx/camera/extensions/impl/advanced/SessionProcessorImpl.java b/camera/camera-testlib-extensions/src/main/java/androidx/camera/extensions/impl/advanced/SessionProcessorImpl.java
index fabfc2b..e3ace72 100644
--- a/camera/camera-testlib-extensions/src/main/java/androidx/camera/extensions/impl/advanced/SessionProcessorImpl.java
+++ b/camera/camera-testlib-extensions/src/main/java/androidx/camera/extensions/impl/advanced/SessionProcessorImpl.java
@@ -21,8 +21,11 @@
import android.hardware.camera2.CameraCharacteristics;
import android.hardware.camera2.CaptureRequest;
import android.hardware.camera2.CaptureResult;
+import android.util.Pair;
import android.view.Surface;
+import androidx.annotation.Nullable;
+
import java.util.Map;
/**
@@ -193,6 +196,32 @@
void abortCapture(int captureSequenceId);
/**
+ * Returns the dynamically calculated capture latency pair in milliseconds.
+ *
+ * <p>In contrast to {@link AdvancedExtenderImpl#getEstimatedCaptureLatencyRange} this method is
+ * guaranteed to be called after {@link #onCaptureSessionStart}.
+ * The measurement is expected to take in to account dynamic parameters such as the current
+ * scene, the state of 3A algorithms, the state of internal HW modules and return a more
+ * accurate assessment of the still capture latency.</p>
+ *
+ * @return pair that includes the estimated input frame/frames camera capture latency as the
+ * first field. This is the time between {@link #onCaptureStarted} and
+ * {@link #onCaptureProcessStarted}. The second field value includes the estimated
+ * post-processing latency. This is the time between {@link #onCaptureProcessStarted} until
+ * the processed frame returns back to the client registered surface.
+ * Both first and second values will be in milliseconds. The total still capture latency will be
+ * the sum of both the first and second values of the pair.
+ * The pair is expected to be null if the dynamic latency estimation is not supported.
+ * If clients have not configured a still capture output, then this method can also return a
+ * null pair.
+ * @since 1.4
+ */
+ @Nullable
+ default Pair<Long, Long> getRealtimeCaptureLatency() {
+ return null;
+ };
+
+ /**
* Callback for notifying the status of {@link #startCapture(CaptureCallback)} and
* {@link #startRepeating(CaptureCallback)}.
*/
diff --git a/camera/integration-tests/coretestapp/src/androidTest/java/androidx/camera/integration/core/ImageCaptureTest.kt b/camera/integration-tests/coretestapp/src/androidTest/java/androidx/camera/integration/core/ImageCaptureTest.kt
index f93db30..066282a 100644
--- a/camera/integration-tests/coretestapp/src/androidTest/java/androidx/camera/integration/core/ImageCaptureTest.kt
+++ b/camera/integration-tests/coretestapp/src/androidTest/java/androidx/camera/integration/core/ImageCaptureTest.kt
@@ -30,6 +30,7 @@
import android.os.Build
import android.os.Environment
import android.provider.MediaStore
+import android.util.Pair
import android.util.Rational
import android.util.Size
import android.view.Surface
@@ -1634,6 +1635,40 @@
capturedImage_withHighResolutionEnabled(preview, imageAnalysis)
}
+ @Test
+ @SdkSuppress(minSdkVersion = 28)
+ fun getRealtimeCaptureLatencyEstimate_whenSessionProcessorSupportsRealtimeLatencyEstimate() =
+ runBlocking {
+ val expectedCaptureLatencyMillis = 1000L
+ val expectedProcessingLatencyMillis = 100L
+ val sessionProcessor = object : SessionProcessor by FakeSessionProcessor(
+ inputFormatPreview = null, // null means using the same output surface
+ inputFormatCapture = null
+ ) {
+ override fun getRealtimeCaptureLatency(): Pair<Long, Long> =
+ Pair(expectedCaptureLatencyMillis, expectedProcessingLatencyMillis)
+ }
+
+ val imageCapture = ImageCapture.Builder().build()
+ val preview = Preview.Builder().build()
+
+ withContext(Dispatchers.Main) {
+ preview.setSurfaceProvider(SurfaceTextureProvider.createSurfaceTextureProvider())
+ val cameraSelector =
+ getCameraSelectorWithSessionProcessor(BACK_SELECTOR, sessionProcessor)
+ cameraProvider.bindToLifecycle(
+ fakeLifecycleOwner, cameraSelector, imageCapture, preview
+ )
+ }
+
+ val latencyEstimate = imageCapture.realtimeCaptureLatencyEstimate
+ // Check the realtime latency estimate is correct.
+ assertThat(latencyEstimate.captureLatencyMillis).isEqualTo(expectedCaptureLatencyMillis)
+ assertThat(latencyEstimate.processingLatencyMillis).isEqualTo(
+ expectedProcessingLatencyMillis
+ )
+ }
+
private fun capturedImage_withHighResolutionEnabled(
preview: Preview? = null,
imageAnalysis: ImageAnalysis? = null
