[BasicExtenderRefactor] Implement BasicExtenderSessionProcessor
Implement BasciExtenderSessionProcessor which is a SessionProcessor
based on OEM's basic extender implementation.
Bug: 256749443
Test: BasicExtenderSessionProcessorTest
Change-Id: I76864b2a0742126a9db105960665b132c3f1333b
diff --git a/camera/camera-core/src/main/java/androidx/camera/core/ImageCapture.java b/camera/camera-core/src/main/java/androidx/camera/core/ImageCapture.java
index df23742..4898c8b 100644
--- a/camera/camera-core/src/main/java/androidx/camera/core/ImageCapture.java
+++ b/camera/camera-core/src/main/java/androidx/camera/core/ImageCapture.java
@@ -108,7 +108,6 @@
import androidx.camera.core.impl.ImmediateSurface;
import androidx.camera.core.impl.MutableConfig;
import androidx.camera.core.impl.MutableOptionsBundle;
-import androidx.camera.core.impl.MutableTagBundle;
import androidx.camera.core.impl.OptionsBundle;
import androidx.camera.core.impl.SessionConfig;
import androidx.camera.core.impl.UseCaseConfig;
@@ -420,49 +419,17 @@
};
} else if (isSessionProcessorEnabledInCurrentCamera()) {
ImageReaderProxy imageReader;
+ // SessionProcessor only outputs JPEG format.
if (getImageFormat() == ImageFormat.JPEG) {
- imageReader =
- new AndroidImageReaderProxy(ImageReader.newInstance(resolution.getWidth(),
- resolution.getHeight(), getImageFormat(), MAX_IMAGES));
- } else if (getImageFormat() == ImageFormat.YUV_420_888) { // convert it into Jpeg
- if (Build.VERSION.SDK_INT >= 26) {
- // Jpeg rotation / quality will be set to softwareJpegProcessor later in
- // ImageCaptureRequestProcessor.
- softwareJpegProcessor =
- new YuvToJpegProcessor(getJpegQualityInternal(), MAX_IMAGES);
-
- ModifiableImageReaderProxy inputReader =
- new ModifiableImageReaderProxy(
- ImageReader.newInstance(resolution.getWidth(),
- resolution.getHeight(),
- ImageFormat.YUV_420_888,
- MAX_IMAGES));
-
- CaptureBundle captureBundle = CaptureBundles.singleDefaultCaptureBundle();
- ProcessingImageReader processingImageReader = new ProcessingImageReader.Builder(
- inputReader,
- captureBundle,
- softwareJpegProcessor
- ).setPostProcessExecutor(mExecutor).setOutputFormat(ImageFormat.JPEG).build();
-
- // Ensure the ImageProxy contains the same capture stage id expected from the
- // ProcessingImageReader.
- MutableTagBundle tagBundle = MutableTagBundle.create();
- // Implicit non-null type use for getCaptureStages().
- //noinspection ConstantConditions
- tagBundle.putTag(processingImageReader.getTagBundleKey(),
- captureBundle.getCaptureStages().get(0).getId());
- inputReader.setImageTagBundle(tagBundle);
-
- imageReader = processingImageReader;
- } else {
- throw new UnsupportedOperationException("Does not support API level < 26");
- }
+ // SessionProcessor can't guarantee that image and capture result have the same
+ // time stamp. Thus we can't use MetadataImageReader
+ imageReader = ImageReaderProxys.createIsolatedReader(resolution.getWidth(),
+ resolution.getHeight(), ImageFormat.JPEG, MAX_IMAGES);
+ mMetadataMatchingCaptureCallback = new CameraCaptureCallback() {
+ };
} else {
throw new IllegalArgumentException("Unsupported image format:" + getImageFormat());
}
- mMetadataMatchingCaptureCallback = new CameraCaptureCallback() {
- };
mImageReader = new SafeCloseImageReaderProxy(imageReader);
} else if (mCaptureProcessor != null || mUseSoftwareJpeg) {
// Capture processor set from configuration takes precedence over software JPEG.
diff --git a/camera/camera-core/src/main/java/androidx/camera/core/impl/RequestProcessor.java b/camera/camera-core/src/main/java/androidx/camera/core/impl/RequestProcessor.java
index ef6c237..e47c72a 100644
--- a/camera/camera-core/src/main/java/androidx/camera/core/impl/RequestProcessor.java
+++ b/camera/camera-core/src/main/java/androidx/camera/core/impl/RequestProcessor.java
@@ -97,30 +97,30 @@
* Callback to be invoked during the capture.
*/
interface Callback {
- void onCaptureStarted(
+ default void onCaptureStarted(
@NonNull Request request,
long frameNumber,
- long timestamp);
+ long timestamp) {}
- void onCaptureProgressed(
+ default void onCaptureProgressed(
@NonNull Request request,
- @NonNull CameraCaptureResult captureResult);
+ @NonNull CameraCaptureResult captureResult) {}
- void onCaptureCompleted(
+ default void onCaptureCompleted(
@NonNull Request request,
- @NonNull CameraCaptureResult captureResult);
+ @NonNull CameraCaptureResult captureResult) {}
- void onCaptureFailed(
+ default void onCaptureFailed(
@NonNull Request request,
- @NonNull CameraCaptureFailure captureFailure);
+ @NonNull CameraCaptureFailure captureFailure) {}
- void onCaptureBufferLost(
+ default void onCaptureBufferLost(
@NonNull Request request,
long frameNumber,
- int outputConfigId);
+ int outputConfigId) {}
- void onCaptureSequenceCompleted(int sequenceId, long frameNumber);
+ default void onCaptureSequenceCompleted(int sequenceId, long frameNumber) {}
- void onCaptureSequenceAborted(int sequenceId);
+ default void onCaptureSequenceAborted(int sequenceId) {}
}
}
diff --git a/camera/camera-extensions/src/androidTest/java/androidx/camera/extensions/internal/AdvancedSessionProcessorTest.kt b/camera/camera-extensions/src/androidTest/java/androidx/camera/extensions/internal/AdvancedSessionProcessorTest.kt
index acd5027..e640f36 100644
--- a/camera/camera-extensions/src/androidTest/java/androidx/camera/extensions/internal/AdvancedSessionProcessorTest.kt
+++ b/camera/camera-extensions/src/androidTest/java/androidx/camera/extensions/internal/AdvancedSessionProcessorTest.kt
@@ -201,53 +201,6 @@
verifyUseCasesOutput(fakeSessionProcessImpl, preview, imageCapture, imageAnalysis)
}
- @SdkSuppress(minSdkVersion = 29) // YUV to JPEG requires api level >= 29
- @Test
- fun useCasesCanWork_captureOutputFormatIsYUV() = runBlocking {
- var captureOutputSurface: Surface? = null
- var intermediaConfigId = -1
- var captureOutputSurfaceFormat = 0
- val fakeSessionProcessImpl = FakeSessionProcessImpl(
- // Directly use output surface
- previewConfigBlock = { outputSurfaceImpl ->
- Camera2OutputConfigImplBuilder
- .newSurfaceConfig(outputSurfaceImpl.surface)
- .build()
- },
- // Has intermediate image reader to process YUV.
- captureConfigBlock = { outputSurfaceImpl ->
- captureOutputSurfaceFormat = outputSurfaceImpl.imageFormat
- captureOutputSurface = outputSurfaceImpl.surface
- Camera2OutputConfigImplBuilder
- .newImageReaderConfig(outputSurfaceImpl.size, ImageFormat.YUV_420_888, 2)
- .build()
- .also {
- intermediaConfigId = it.id
- }
- },
- onCaptureSessionStarted = {
- // Emulates the processing, write an image to the output surface.
- val imageWriter = ImageWriter.newInstance(captureOutputSurface!!, 2)
- it.setImageProcessor(intermediaConfigId) { _, _, image, _ ->
- val inputImage = imageWriter.dequeueInputImage()
- imageWriter.queueInputImage(inputImage)
- image.decrement()
- }
- }
- )
-
- val preview = Preview.Builder().build()
- val imageCapture = ImageCapture.Builder()
- .setSupportedResolutions(
- listOf(
- android.util.Pair(ImageFormat.YUV_420_888, arrayOf(Size(640, 480)))
- )
- )
- .build()
- verifyUseCasesOutput(fakeSessionProcessImpl, preview, imageCapture)
- assertThat(captureOutputSurfaceFormat).isEqualTo(ImageFormat.YUV_420_888)
- }
-
private suspend fun assumeAllowsSharedSurface() = withContext(Dispatchers.Main) {
val imageReader = ImageReader.newInstance(640, 480, ImageFormat.YUV_420_888, 2)
val maxSharedSurfaceCount =
@@ -318,7 +271,7 @@
}
// Test if physicalCameraId is set and returned in the image received in the image processor.
- @SdkSuppress(minSdkVersion = 29) // YUV to JPEG requires api level >= 29
+ @SdkSuppress(minSdkVersion = 28) // physical camera id is supported in API28+
@Test
fun useCasesCanWork_setPhysicalCameraId() = runBlocking {
assumeAllowsSharedSurface()
@@ -326,9 +279,9 @@
assumeTrue(physicalCameraIdList.isNotEmpty())
val physicalCameraId = physicalCameraIdList[0]
- var captureOutputSurface: Surface? = null
- var intermediaConfigId = -1
+ var analysisOutputSurface: Surface? = null
var sharedConfigId = -1
+ var intermediaConfigId = -1
val deferredImagePhysicalCameraId = CompletableDeferred<String?>()
val deferredSharedImagePhysicalCameraId = CompletableDeferred<String?>()
@@ -340,25 +293,29 @@
.setPhysicalCameraId(physicalCameraId)
.build()
},
+ // Directly use output surface
+ captureConfigBlock = {
+ Camera2OutputConfigImplBuilder
+ .newSurfaceConfig(it.surface)
+ .build()
+ },
// Has intermediate image reader to process YUV
- captureConfigBlock = { outputSurfaceImpl ->
- captureOutputSurface = outputSurfaceImpl.surface
+ analysisConfigBlock = { outputSurfaceImpl ->
+ analysisOutputSurface = outputSurfaceImpl.surface
val sharedConfig = Camera2OutputConfigImplBuilder.newImageReaderConfig(
outputSurfaceImpl.size, outputSurfaceImpl.imageFormat, 2
- ).setPhysicalCameraId(physicalCameraId).build()
- sharedConfigId = sharedConfig.id
+ ).setPhysicalCameraId(physicalCameraId)
+ .build().also { sharedConfigId = it.id }
Camera2OutputConfigImplBuilder
.newImageReaderConfig(outputSurfaceImpl.size, ImageFormat.YUV_420_888, 2)
.setPhysicalCameraId(physicalCameraId)
.addSurfaceSharingOutputConfig(sharedConfig)
.build()
- .also {
- intermediaConfigId = it.id
- }
+ .also { intermediaConfigId = it.id }
},
onCaptureSessionStarted = { requestProcessor ->
- val imageWriter = ImageWriter.newInstance(captureOutputSurface!!, 2)
+ val imageWriter = ImageWriter.newInstance(analysisOutputSurface!!, 2)
requestProcessor.setImageProcessor(intermediaConfigId) {
_, _, image, physicalCameraIdOfImage ->
deferredImagePhysicalCameraId.complete(physicalCameraIdOfImage)
@@ -375,12 +332,9 @@
)
val preview = Preview.Builder().build()
- val imageCapture = ImageCapture.Builder()
- .setSupportedResolutions(
- listOf(android.util.Pair(ImageFormat.YUV_420_888, arrayOf(Size(640, 480))))
- )
- .build()
- verifyUseCasesOutput(fakeSessionProcessImpl, preview, imageCapture)
+ val imageCapture = ImageCapture.Builder().build()
+ val imageAnalysis = ImageAnalysis.Builder().build()
+ verifyUseCasesOutput(fakeSessionProcessImpl, preview, imageCapture, imageAnalysis)
assertThat(deferredImagePhysicalCameraId.awaitWithTimeout(2000))
.isEqualTo(physicalCameraId)
assertThat(deferredSharedImagePhysicalCameraId.awaitWithTimeout(2000))
diff --git a/camera/camera-extensions/src/androidTest/java/androidx/camera/extensions/internal/sessionprocessor/BasicExtenderSessionProcessorTest.kt b/camera/camera-extensions/src/androidTest/java/androidx/camera/extensions/internal/sessionprocessor/BasicExtenderSessionProcessorTest.kt
new file mode 100644
index 0000000..ec04282
--- /dev/null
+++ b/camera/camera-extensions/src/androidTest/java/androidx/camera/extensions/internal/sessionprocessor/BasicExtenderSessionProcessorTest.kt
@@ -0,0 +1,542 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.camera.extensions.internal.sessionprocessor
+
+import android.content.Context
+import android.graphics.ImageFormat
+import android.hardware.camera2.CameraCharacteristics
+import android.hardware.camera2.CaptureRequest
+import android.hardware.camera2.CaptureResult
+import android.hardware.camera2.TotalCaptureResult
+import android.media.Image
+import android.media.ImageWriter
+import android.util.Pair
+import android.util.Size
+import android.view.Surface
+import androidx.camera.camera2.Camera2Config
+import androidx.camera.core.CameraFilter
+import androidx.camera.core.CameraInfo
+import androidx.camera.core.CameraSelector
+import androidx.camera.core.ImageAnalysis
+import androidx.camera.core.ImageCapture
+import androidx.camera.core.ImageCaptureException
+import androidx.camera.core.ImageProxy
+import androidx.camera.core.Preview
+import androidx.camera.core.UseCaseGroup
+import androidx.camera.core.impl.CameraConfig
+import androidx.camera.core.impl.Config
+import androidx.camera.core.impl.ExtendedCameraConfigProviderStore
+import androidx.camera.core.impl.Identifier
+import androidx.camera.core.impl.MutableOptionsBundle
+import androidx.camera.core.impl.SessionProcessor
+import androidx.camera.core.impl.utils.Exif
+import androidx.camera.core.impl.utils.executor.CameraXExecutors
+import androidx.camera.extensions.impl.CaptureProcessorImpl
+import androidx.camera.extensions.impl.CaptureStageImpl
+import androidx.camera.extensions.impl.ExtenderStateListener
+import androidx.camera.extensions.impl.ImageCaptureExtenderImpl
+import androidx.camera.extensions.impl.PreviewExtenderImpl
+import androidx.camera.extensions.impl.PreviewExtenderImpl.ProcessorType
+import androidx.camera.extensions.impl.PreviewExtenderImpl.ProcessorType.PROCESSOR_TYPE_IMAGE_PROCESSOR
+import androidx.camera.extensions.impl.PreviewExtenderImpl.ProcessorType.PROCESSOR_TYPE_NONE
+import androidx.camera.extensions.impl.PreviewExtenderImpl.ProcessorType.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY
+import androidx.camera.extensions.impl.PreviewImageProcessorImpl
+import androidx.camera.extensions.impl.ProcessResultImpl
+import androidx.camera.extensions.impl.RequestUpdateProcessorImpl
+import androidx.camera.lifecycle.ProcessCameraProvider
+import androidx.camera.testing.CameraUtil
+import androidx.camera.testing.SurfaceTextureProvider
+import androidx.camera.testing.fakes.FakeLifecycleOwner
+import androidx.test.core.app.ApplicationProvider
+import androidx.test.filters.LargeTest
+import androidx.test.filters.SdkSuppress
+import androidx.testutils.assertThrows
+import com.google.common.truth.Truth.assertThat
+import java.util.concurrent.Executor
+import java.util.concurrent.Semaphore
+import java.util.concurrent.TimeUnit
+import kotlinx.coroutines.CompletableDeferred
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withContext
+import kotlinx.coroutines.withTimeout
+import org.junit.After
+import org.junit.Assume.assumeTrue
+import org.junit.Before
+import org.junit.Rule
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+
+@LargeTest
+@SdkSuppress(minSdkVersion = 29) // Extensions supported on API 29+
+@RunWith(Parameterized::class)
+class BasicExtenderSessionProcessorTest(
+ private val hasCaptureProcessor: Boolean,
+ private val previewProcessorType: ProcessorType
+) {
+ companion object {
+ @Parameterized.Parameters(name = "hasCaptureProcessor = {0}, previewProcessorType = {1}")
+ @JvmStatic
+ fun parameters() = listOf(
+ arrayOf(false /* No CaptureProcessor */, PROCESSOR_TYPE_NONE),
+ arrayOf(true /* Has CaptureProcessor */, PROCESSOR_TYPE_NONE),
+ arrayOf(false /* No CaptureProcessor */, PROCESSOR_TYPE_REQUEST_UPDATE_ONLY),
+ arrayOf(true /* Has CaptureProcessor */, PROCESSOR_TYPE_REQUEST_UPDATE_ONLY),
+ arrayOf(false /* No CaptureProcessor */, PROCESSOR_TYPE_IMAGE_PROCESSOR),
+ arrayOf(true /* Has CaptureProcessor */, PROCESSOR_TYPE_IMAGE_PROCESSOR)
+ )
+
+ private fun createCaptureStage(
+ id: Int = 0,
+ parameters: List<Pair<CaptureRequest.Key<Any>, Any>> = mutableListOf()
+ ): CaptureStageImpl {
+ return object : CaptureStageImpl {
+ override fun getId() = id
+ override fun getParameters() = parameters
+ }
+ }
+ }
+
+ @get:Rule
+ val useCamera = CameraUtil.grantCameraPermissionAndPreTest(
+ CameraUtil.PreTestCameraIdList(Camera2Config.defaultConfig())
+ )
+ private val context = ApplicationProvider.getApplicationContext<Context>()
+ private lateinit var cameraProvider: ProcessCameraProvider
+ private lateinit var fakeLifecycleOwner: FakeLifecycleOwner
+ private val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA
+ private lateinit var fakePreviewExtenderImpl: FakePreviewExtenderImpl
+ private lateinit var fakeCaptureExtenderImpl: FakeImageCaptureExtenderImpl
+ private lateinit var basicExtenderSessionProcessor: BasicExtenderSessionProcessor
+
+ @Before
+ fun setUp() = runBlocking {
+ cameraProvider = ProcessCameraProvider.getInstance(context)[10, TimeUnit.SECONDS]
+ withContext(Dispatchers.Main) {
+ fakeLifecycleOwner = FakeLifecycleOwner()
+ fakeLifecycleOwner.startAndResume()
+ }
+
+ fakePreviewExtenderImpl = FakePreviewExtenderImpl(previewProcessorType)
+ fakeCaptureExtenderImpl = FakeImageCaptureExtenderImpl(hasCaptureProcessor)
+ basicExtenderSessionProcessor = BasicExtenderSessionProcessor(
+ fakePreviewExtenderImpl, fakeCaptureExtenderImpl, context
+ )
+ }
+
+ @After
+ fun tearDown() = runBlocking {
+ if (::cameraProvider.isInitialized) {
+ withContext(Dispatchers.Main) {
+ cameraProvider.unbindAll()
+ cameraProvider.shutdown()[10, TimeUnit.SECONDS]
+ }
+ }
+ }
+
+ @Test
+ fun canOutputCorrectly(): Unit = runBlocking {
+ val preview = Preview.Builder().build()
+ val imageCapture = ImageCapture.Builder().build()
+ val imageAnalysis = ImageAnalysis.Builder().build()
+ val previewSemaphore = Semaphore(0)
+ val analysisSemaphore = Semaphore(0)
+ verifyUseCasesOutput(
+ preview,
+ imageCapture,
+ imageAnalysis,
+ previewSemaphore,
+ analysisSemaphore
+ )
+ }
+
+ @Test
+ fun imageCaptureError(): Unit = runBlocking {
+ assumeTrue(hasCaptureProcessor)
+ fakeCaptureExtenderImpl = FakeImageCaptureExtenderImpl(
+ hasCaptureProcessor, throwErrorOnProcess = true
+ )
+ basicExtenderSessionProcessor = BasicExtenderSessionProcessor(
+ fakePreviewExtenderImpl, fakeCaptureExtenderImpl, context
+ )
+ val preview = Preview.Builder().build()
+ val imageCapture = ImageCapture.Builder().build()
+ val imageAnalysis = ImageAnalysis.Builder().build()
+ assertThrows<ImageCaptureException> {
+ verifyUseCasesOutput(preview, imageCapture, imageAnalysis)
+ }
+ }
+
+ @Test
+ fun canOutputCorrectly_withoutAnalysis(): Unit = runBlocking {
+ val preview = Preview.Builder().build()
+ val imageCapture = ImageCapture.Builder().build()
+ val previewSemaphore = Semaphore(0)
+ verifyUseCasesOutput(
+ preview = preview,
+ imageCapture = imageCapture,
+ previewFrameSemaphore = previewSemaphore
+ )
+ }
+
+ suspend fun getSensorRotationDegrees(rotation: Int): Int {
+ return withContext(Dispatchers.Main) {
+ val camera = cameraProvider.bindToLifecycle(fakeLifecycleOwner, cameraSelector)
+ camera.cameraInfo.getSensorRotationDegrees(rotation)
+ }
+ }
+
+ @Test
+ fun canOutputCorrectly_setTargetRotation(): Unit = runBlocking {
+ assumeTrue(hasCaptureProcessor)
+ val preview = Preview.Builder().build()
+ val imageCapture = ImageCapture.Builder()
+ .setTargetRotation(Surface.ROTATION_0)
+ .build()
+ val previewSemaphore = Semaphore(0)
+ verifyUseCasesOutput(
+ preview = preview,
+ imageCapture = imageCapture,
+ previewFrameSemaphore = previewSemaphore,
+ expectedExifRotation = getSensorRotationDegrees(Surface.ROTATION_0)
+ )
+ }
+
+ @Test
+ fun canOutputCorrectlyAfterStopStart(): Unit = runBlocking {
+ val preview = Preview.Builder().build()
+ val imageCapture = ImageCapture.Builder().build()
+ val imageAnalysis = ImageAnalysis.Builder().build()
+ val previewSemaphore = Semaphore(0)
+ val analysisSemaphore = Semaphore(0)
+
+ verifyUseCasesOutput(
+ preview,
+ imageCapture,
+ imageAnalysis,
+ previewSemaphore,
+ analysisSemaphore
+ )
+
+ fakeLifecycleOwner.pauseAndStop()
+
+ delay(1000)
+ previewSemaphore.drainPermits()
+ analysisSemaphore.drainPermits()
+ fakeLifecycleOwner.startAndResume()
+
+ assertThat(previewSemaphore.tryAcquire(3, TimeUnit.SECONDS)).isTrue()
+
+ imageAnalysis.let {
+ assertThat(analysisSemaphore.tryAcquire(3, TimeUnit.SECONDS)).isTrue()
+ }
+
+ verifyStillCapture(imageCapture)
+ }
+
+ /**
+ * Verify if the given use cases have expected output.
+ * 1) Preview frame is received
+ * 2) imageCapture gets a captured JPEG image
+ * 3) imageAnalysis gets a Image in Analyzer.
+ */
+ private suspend fun verifyUseCasesOutput(
+ preview: Preview,
+ imageCapture: ImageCapture,
+ imageAnalysis: ImageAnalysis? = null,
+ previewFrameSemaphore: Semaphore? = null,
+ analysisSemaphore: Semaphore? = null,
+ expectedExifRotation: Int = 0,
+ ) {
+ withContext(Dispatchers.Main) {
+ preview.setSurfaceProvider(
+ SurfaceTextureProvider.createAutoDrainingSurfaceTextureProvider {
+ if (previewFrameSemaphore?.availablePermits() == 0) {
+ previewFrameSemaphore.release()
+ }
+ }
+ )
+ imageAnalysis?.setAnalyzer(CameraXExecutors.mainThreadExecutor()) {
+ it.close()
+ if (analysisSemaphore?.availablePermits() == 0) {
+ analysisSemaphore.release()
+ }
+ }
+ val cameraSelector =
+ getCameraSelectorWithSessionProcessor(
+ cameraSelector,
+ basicExtenderSessionProcessor
+ )
+
+ val useCaseGroupBuilder = UseCaseGroup.Builder()
+ useCaseGroupBuilder.addUseCase(preview)
+ useCaseGroupBuilder.addUseCase(imageCapture)
+ imageAnalysis?.let { useCaseGroupBuilder.addUseCase(it) }
+
+ cameraProvider.bindToLifecycle(
+ fakeLifecycleOwner,
+ cameraSelector,
+ useCaseGroupBuilder.build()
+ )
+ }
+
+ previewFrameSemaphore?.let {
+ assertThat(it.tryAcquire(3, TimeUnit.SECONDS)).isTrue()
+ }
+
+ analysisSemaphore?.let {
+ assertThat(analysisSemaphore.tryAcquire(3, TimeUnit.SECONDS)).isTrue()
+ }
+
+ verifyStillCapture(imageCapture, expectedExifRotation)
+ }
+
+ private suspend fun verifyStillCapture(
+ imageCapture: ImageCapture,
+ expectExifRotation: Int = 0
+ ) {
+ val deferCapturedImage = CompletableDeferred<ImageProxy>()
+ imageCapture.takePicture(
+ CameraXExecutors.mainThreadExecutor(),
+ object : ImageCapture.OnImageCapturedCallback() {
+ override fun onCaptureSuccess(image: ImageProxy) {
+ deferCapturedImage.complete(image)
+ }
+
+ override fun onError(exception: ImageCaptureException) {
+ deferCapturedImage.completeExceptionally(exception)
+ }
+ })
+ withTimeout(6000) {
+ deferCapturedImage.await().use {
+ assertThat(it.format).isEqualTo(ImageFormat.JPEG)
+ if (expectExifRotation != 0) {
+ val exif = Exif.createFromImageProxy(it)
+ assertThat(exif.rotation).isEqualTo(expectExifRotation)
+ }
+ }
+ }
+ }
+
+ private fun getCameraSelectorWithSessionProcessor(
+ cameraSelector: CameraSelector,
+ sessionProcessor: SessionProcessor
+ ): CameraSelector {
+ val identifier = Identifier.create("idStr")
+ ExtendedCameraConfigProviderStore.addConfig(identifier) { _, _ ->
+ object : CameraConfig {
+ override fun getConfig(): Config {
+ return MutableOptionsBundle.create()
+ }
+
+ override fun getCompatibilityId(): Identifier {
+ return Identifier.create(0)
+ }
+
+ override fun getSessionProcessor(
+ valueIfMissing: SessionProcessor?
+ ): SessionProcessor {
+ return sessionProcessor
+ }
+
+ override fun getSessionProcessor(): SessionProcessor {
+ return sessionProcessor
+ }
+ }
+ }
+ val builder = CameraSelector.Builder.fromSelector(cameraSelector)
+ builder.addCameraFilter(object : CameraFilter {
+ override fun filter(cameraInfos: MutableList<CameraInfo>): MutableList<CameraInfo> {
+ val newCameraInfos = mutableListOf<CameraInfo>()
+ newCameraInfos.addAll(cameraInfos)
+ return newCameraInfos
+ }
+
+ override fun getIdentifier(): Identifier {
+ return identifier
+ }
+ })
+ return builder.build()
+ }
+
+ open class FakeExtenderStateListener : ExtenderStateListener {
+ override fun onInit(
+ cameraId: String,
+ cameraCharacteristics: CameraCharacteristics,
+ context: Context
+ ) {
+ }
+
+ override fun onDeInit() {}
+ override fun onPresetSession() = null
+ override fun onEnableSession() = null
+ override fun onDisableSession() = null
+ }
+
+ private class FakePreviewExtenderImpl(
+ private var processorType: ProcessorType = PROCESSOR_TYPE_NONE
+ ) : PreviewExtenderImpl, FakeExtenderStateListener() {
+ var fakePreviewImageProcessorImpl: FakePreviewImageProcessorImpl? = null
+ override fun isExtensionAvailable(
+ cameraId: String,
+ cameraCharacteristics: CameraCharacteristics
+ ): Boolean {
+ return true
+ }
+
+ override fun init(cameraId: String, cameraCharacteristics: CameraCharacteristics) {}
+ override fun getCaptureStage() = createCaptureStage()
+ override fun getProcessorType() = processorType
+ override fun getProcessor() =
+ when (processorType) {
+ PROCESSOR_TYPE_NONE -> null
+ PROCESSOR_TYPE_REQUEST_UPDATE_ONLY -> FakeRequestUpdateProcessor()
+ PROCESSOR_TYPE_IMAGE_PROCESSOR -> {
+ fakePreviewImageProcessorImpl = FakePreviewImageProcessorImpl()
+ fakePreviewImageProcessorImpl
+ }
+ }
+
+ override fun getSupportedResolutions() = null
+ override fun onDeInit() {
+ fakePreviewImageProcessorImpl?.close()
+ }
+ }
+
+ private class FakeImageCaptureExtenderImpl(
+ private val hasCaptureProcessor: Boolean = false,
+ private val throwErrorOnProcess: Boolean = false
+ ) : ImageCaptureExtenderImpl, FakeExtenderStateListener() {
+ val fakeCaptureProcessorImpl: FakeCaptureProcessorImpl? by lazy {
+ if (hasCaptureProcessor) {
+ FakeCaptureProcessorImpl(throwErrorOnProcess)
+ } else {
+ null
+ }
+ }
+
+ override fun isExtensionAvailable(
+ cameraId: String,
+ cameraCharacteristics: CameraCharacteristics
+ ): Boolean {
+ return true
+ }
+
+ override fun init(cameraId: String, cameraCharacteristics: CameraCharacteristics) {}
+ override fun getCaptureProcessor() = fakeCaptureProcessorImpl
+
+ override fun getCaptureStages() = listOf(createCaptureStage())
+
+ override fun getMaxCaptureStage(): Int {
+ return 2
+ }
+
+ override fun getSupportedResolutions() = null
+ override fun getEstimatedCaptureLatencyRange(size: Size?) = null
+ override fun getAvailableCaptureRequestKeys(): MutableList<CaptureRequest.Key<Any>> {
+ return mutableListOf()
+ }
+
+ override fun getAvailableCaptureResultKeys(): MutableList<CaptureResult.Key<Any>> {
+ return mutableListOf()
+ }
+
+ override fun onDeInit() {
+ fakeCaptureProcessorImpl?.close()
+ }
+ }
+
+ private class FakeCaptureProcessorImpl(
+ val throwErrorOnProcess: Boolean = false
+ ) : CaptureProcessorImpl {
+ private var imageWriter: ImageWriter? = null
+ override fun process(results: MutableMap<Int, Pair<Image, TotalCaptureResult>>?) {
+ if (throwErrorOnProcess) {
+ throw RuntimeException("Process failed")
+ }
+ val image = imageWriter!!.dequeueInputImage()
+ imageWriter!!.queueInputImage(image)
+ }
+
+ override fun process(
+ results: MutableMap<Int, Pair<Image, TotalCaptureResult>>?,
+ resultCallback: ProcessResultImpl?,
+ executor: Executor?
+ ) {
+ process(results)
+ }
+
+ override fun onOutputSurface(surface: Surface, imageFormat: Int) {
+ imageWriter = ImageWriter.newInstance(surface, 2)
+ }
+
+ override fun onResolutionUpdate(size: Size) {}
+ override fun onImageFormatUpdate(imageFormat: Int) {}
+ fun close() {
+ imageWriter?.close()
+ imageWriter = null
+ }
+ }
+
+ private class FakePreviewImageProcessorImpl : PreviewImageProcessorImpl {
+ private var imageWriter: ImageWriter? = null
+ override fun process(image: Image?, result: TotalCaptureResult?) {
+ val emptyImage = imageWriter!!.dequeueInputImage()
+ imageWriter!!.queueInputImage(emptyImage)
+ }
+
+ override fun process(
+ image: Image?,
+ result: TotalCaptureResult?,
+ resultCallback: ProcessResultImpl?,
+ executor: Executor?
+ ) {
+ process(image, result)
+ }
+
+ override fun onOutputSurface(surface: Surface, imageFormat: Int) {
+ imageWriter = ImageWriter.newInstance(surface, 2)
+ }
+
+ override fun onResolutionUpdate(size: Size) {}
+ override fun onImageFormatUpdate(imageFormat: Int) {}
+ fun close() {
+ imageWriter?.close()
+ imageWriter = null
+ }
+ }
+
+ private class FakeRequestUpdateProcessor : RequestUpdateProcessorImpl {
+ override fun onOutputSurface(surface: Surface, imageFormat: Int) {
+ throw RuntimeException("Should not invoke this")
+ }
+
+ override fun onResolutionUpdate(size: Size) {
+ throw RuntimeException("Should not invoke this")
+ }
+
+ override fun onImageFormatUpdate(imageFormat: Int) {
+ throw RuntimeException("Should not invoke this")
+ }
+
+ override fun process(result: TotalCaptureResult?): CaptureStageImpl {
+ return createCaptureStage()
+ }
+ }
+}
\ No newline at end of file
diff --git a/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/BasicExtenderSessionProcessor.java b/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/BasicExtenderSessionProcessor.java
new file mode 100644
index 0000000..39594e8
--- /dev/null
+++ b/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/BasicExtenderSessionProcessor.java
@@ -0,0 +1,541 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.camera.extensions.internal.sessionprocessor;
+
+import static androidx.camera.extensions.impl.PreviewExtenderImpl.ProcessorType.PROCESSOR_TYPE_IMAGE_PROCESSOR;
+import static androidx.camera.extensions.impl.PreviewExtenderImpl.ProcessorType.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY;
+
+import android.content.Context;
+import android.graphics.ImageFormat;
+import android.hardware.camera2.CameraCharacteristics;
+import android.hardware.camera2.CameraDevice;
+import android.hardware.camera2.CaptureRequest;
+import android.hardware.camera2.CaptureResult;
+import android.hardware.camera2.TotalCaptureResult;
+import android.util.Pair;
+
+import androidx.annotation.GuardedBy;
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+import androidx.annotation.OptIn;
+import androidx.annotation.RequiresApi;
+import androidx.camera.camera2.impl.Camera2CameraCaptureResultConverter;
+import androidx.camera.camera2.interop.CaptureRequestOptions;
+import androidx.camera.camera2.interop.ExperimentalCamera2Interop;
+import androidx.camera.core.Logger;
+import androidx.camera.core.impl.CameraCaptureFailure;
+import androidx.camera.core.impl.CameraCaptureResult;
+import androidx.camera.core.impl.Config;
+import androidx.camera.core.impl.OutputSurface;
+import androidx.camera.core.impl.RequestProcessor;
+import androidx.camera.core.impl.SessionProcessor;
+import androidx.camera.extensions.impl.CaptureProcessorImpl;
+import androidx.camera.extensions.impl.CaptureStageImpl;
+import androidx.camera.extensions.impl.ImageCaptureExtenderImpl;
+import androidx.camera.extensions.impl.PreviewExtenderImpl;
+import androidx.camera.extensions.impl.PreviewImageProcessorImpl;
+import androidx.camera.extensions.impl.RequestUpdateProcessorImpl;
+import androidx.core.util.Preconditions;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * A {@link SessionProcessor} based on OEMs' basic extender implementation.
+ */
+@OptIn(markerClass = ExperimentalCamera2Interop.class)
+@RequiresApi(26) // TODO(b/200306659): Remove and replace with annotation on package-info.java
+public class BasicExtenderSessionProcessor extends SessionProcessorBase {
+ private static final String TAG = "BasicSessionProcessor";
+
+ private static final int PREVIEW_PROCESS_MAX_IMAGES = 2;
+ @NonNull
+ private final Context mContext;
+ @NonNull
+ private final PreviewExtenderImpl mPreviewExtenderImpl;
+ @NonNull
+ private final ImageCaptureExtenderImpl mImageCaptureExtenderImpl;
+
+ final Object mLock = new Object();
+ volatile StillCaptureProcessor mStillCaptureProcessor = null;
+ volatile PreviewProcessor mPreviewProcessor = null;
+ volatile RequestUpdateProcessorImpl mRequestUpdateProcessor = null;
+ private volatile Camera2OutputConfig mPreviewOutputConfig;
+ private volatile Camera2OutputConfig mCaptureOutputConfig;
+ @Nullable
+ private volatile Camera2OutputConfig mAnalysisOutputConfig = null;
+ private volatile OutputSurface mPreviewOutputSurfaceConfig;
+ private volatile OutputSurface mCaptureOutputSurfaceConfig;
+ private volatile RequestProcessor mRequestProcessor;
+ volatile boolean mIsCapturing = false;
+ private final AtomicInteger mNextCaptureSequenceId = new AtomicInteger(0);
+ static AtomicInteger sLastOutputConfigId = new AtomicInteger(0);
+ @GuardedBy("mLock")
+ private final Map<CaptureRequest.Key<?>, Object> mParameters = new LinkedHashMap<>();
+
+ public BasicExtenderSessionProcessor(@NonNull PreviewExtenderImpl previewExtenderImpl,
+ @NonNull ImageCaptureExtenderImpl imageCaptureExtenderImpl,
+ @NonNull Context context) {
+ mPreviewExtenderImpl = previewExtenderImpl;
+ mImageCaptureExtenderImpl = imageCaptureExtenderImpl;
+ mContext = context;
+ }
+
+ @NonNull
+ @Override
+ protected Camera2SessionConfig initSessionInternal(@NonNull String cameraId,
+ @NonNull Map<String, CameraCharacteristics> cameraCharacteristicsMap,
+ @NonNull OutputSurface previewSurfaceConfig,
+ @NonNull OutputSurface imageCaptureSurfaceConfig,
+ @Nullable OutputSurface imageAnalysisSurfaceConfig) {
+ Logger.d(TAG, "PreviewExtenderImpl.onInit");
+ mPreviewExtenderImpl.onInit(cameraId, cameraCharacteristicsMap.get(cameraId),
+ mContext);
+ Logger.d(TAG, "ImageCaptureExtenderImpl.onInit");
+ mImageCaptureExtenderImpl.onInit(cameraId, cameraCharacteristicsMap.get(cameraId),
+ mContext);
+
+ mPreviewOutputSurfaceConfig = previewSurfaceConfig;
+ mCaptureOutputSurfaceConfig = imageCaptureSurfaceConfig;
+
+ // Preview
+ PreviewExtenderImpl.ProcessorType processorType =
+ mPreviewExtenderImpl.getProcessorType();
+ Logger.d(TAG, "preview processorType=" + processorType);
+ if (processorType == PROCESSOR_TYPE_IMAGE_PROCESSOR) {
+ mPreviewOutputConfig = ImageReaderOutputConfig.create(
+ sLastOutputConfigId.getAndIncrement(),
+ previewSurfaceConfig.getSize(),
+ ImageFormat.YUV_420_888,
+ PREVIEW_PROCESS_MAX_IMAGES);
+ PreviewImageProcessorImpl previewImageProcessor =
+ (PreviewImageProcessorImpl) mPreviewExtenderImpl.getProcessor();
+ mPreviewProcessor = new PreviewProcessor(
+ previewImageProcessor, mPreviewOutputSurfaceConfig.getSurface(),
+ mPreviewOutputSurfaceConfig.getSize());
+ } else if (processorType == PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) {
+ mPreviewOutputConfig = SurfaceOutputConfig.create(
+ sLastOutputConfigId.getAndIncrement(),
+ previewSurfaceConfig.getSurface());
+ mRequestUpdateProcessor =
+ (RequestUpdateProcessorImpl) mPreviewExtenderImpl.getProcessor();
+ } else {
+ mPreviewOutputConfig = SurfaceOutputConfig.create(
+ sLastOutputConfigId.getAndIncrement(),
+ previewSurfaceConfig.getSurface());
+ }
+
+ // Image Capture
+ CaptureProcessorImpl captureProcessor = mImageCaptureExtenderImpl.getCaptureProcessor();
+ Logger.d(TAG, "CaptureProcessor=" + captureProcessor);
+
+ if (captureProcessor != null) {
+ mCaptureOutputConfig = ImageReaderOutputConfig.create(
+ sLastOutputConfigId.getAndIncrement(),
+ imageCaptureSurfaceConfig.getSize(),
+ ImageFormat.YUV_420_888,
+ mImageCaptureExtenderImpl.getMaxCaptureStage());
+ mStillCaptureProcessor = new StillCaptureProcessor(
+ captureProcessor, mCaptureOutputSurfaceConfig.getSurface(),
+ mCaptureOutputSurfaceConfig.getSize());
+ } else {
+ mCaptureOutputConfig = SurfaceOutputConfig.create(
+ sLastOutputConfigId.getAndIncrement(),
+ imageCaptureSurfaceConfig.getSurface());
+ }
+
+ // Image Analysis
+ if (imageAnalysisSurfaceConfig != null) {
+ mAnalysisOutputConfig = SurfaceOutputConfig.create(
+ sLastOutputConfigId.getAndIncrement(),
+ imageAnalysisSurfaceConfig.getSurface());
+ }
+
+ Camera2SessionConfigBuilder builder =
+ new Camera2SessionConfigBuilder()
+ .addOutputConfig(mPreviewOutputConfig)
+ .addOutputConfig(mCaptureOutputConfig)
+ .setSessionTemplateId(CameraDevice.TEMPLATE_PREVIEW);
+
+ if (mAnalysisOutputConfig != null) {
+ builder.addOutputConfig(mAnalysisOutputConfig);
+ }
+
+ CaptureStageImpl captureStagePreview = mPreviewExtenderImpl.onPresetSession();
+ Logger.d(TAG, "preview onPresetSession:" + captureStagePreview);
+
+ CaptureStageImpl captureStageCapture = mImageCaptureExtenderImpl.onPresetSession();
+ Logger.d(TAG, "capture onPresetSession:" + captureStageCapture);
+
+ if (captureStagePreview != null && captureStagePreview.getParameters() != null) {
+ for (Pair<CaptureRequest.Key, Object> parameter :
+ captureStagePreview.getParameters()) {
+ builder.addSessionParameter(parameter.first, parameter.second);
+ }
+ }
+
+ if (captureStageCapture != null && captureStageCapture.getParameters() != null) {
+ for (Pair<CaptureRequest.Key, Object> parameter :
+ captureStageCapture.getParameters()) {
+ builder.addSessionParameter(parameter.first, parameter.second);
+ }
+ }
+ return builder.build();
+ }
+
+ @Override
+ protected void deInitSessionInternal() {
+ Logger.d(TAG, "preview onDeInit");
+ mPreviewExtenderImpl.onDeInit();
+ Logger.d(TAG, "capture onDeInit");
+ mImageCaptureExtenderImpl.onDeInit();
+
+ if (mPreviewProcessor != null) {
+ mPreviewProcessor.close();
+ mPreviewProcessor = null;
+ }
+ if (mStillCaptureProcessor != null) {
+ mStillCaptureProcessor.close();
+ mStillCaptureProcessor = null;
+ }
+ }
+
+ @Override
+ public void setParameters(@NonNull Config config) {
+ synchronized (mLock) {
+ HashMap<CaptureRequest.Key<?>, Object> map = new HashMap<>();
+
+ CaptureRequestOptions options =
+ CaptureRequestOptions.Builder.from(config).build();
+
+ for (Config.Option<?> option : options.listOptions()) {
+ @SuppressWarnings("unchecked")
+ CaptureRequest.Key<Object> key = (CaptureRequest.Key<Object>) option.getToken();
+ map.put(key, options.retrieveOption(option));
+ }
+ mParameters.putAll(map);
+ applyRotationAndJpegQualityToProcessor();
+ }
+ }
+
+ @Override
+ public void onCaptureSessionStart(@NonNull RequestProcessor requestProcessor) {
+ mRequestProcessor = requestProcessor;
+
+ CaptureStageImpl captureStage1 = mPreviewExtenderImpl.onEnableSession();
+ Logger.d(TAG, "preview onEnableSession: " + captureStage1);
+
+ CaptureStageImpl captureStage2 = mImageCaptureExtenderImpl.onEnableSession();
+ Logger.d(TAG, "capture onEnableSession:" + captureStage2);
+
+ if (captureStage1 != null) {
+ submitRequestByCaptureStage(requestProcessor, captureStage1);
+ }
+
+ if (captureStage2 != null) {
+ submitRequestByCaptureStage(requestProcessor, captureStage2);
+ }
+
+ if (mPreviewProcessor != null) {
+ setImageProcessor(mPreviewOutputConfig.getId(),
+ new ImageProcessor() {
+ @Override
+ public void onNextImageAvailable(int outputStreamId, long timestampNs,
+ @NonNull ImageReference imageReference,
+ @Nullable String physicalCameraId) {
+ if (mPreviewProcessor != null) {
+ mPreviewProcessor.notifyImage(imageReference);
+ }
+ }
+ });
+ mPreviewProcessor.start();
+ }
+ }
+
+ private void applyParameters(RequestBuilder builder) {
+ synchronized (mLock) {
+ for (CaptureRequest.Key<?> key : mParameters.keySet()) {
+ Object value = mParameters.get(key);
+ if (value != null) {
+ builder.setParameters(key, value);
+ }
+ }
+ }
+ }
+
+ private void applyRotationAndJpegQualityToProcessor() {
+ synchronized (mLock) {
+ if (mStillCaptureProcessor == null) {
+ return;
+ }
+ Integer orientationObj = (Integer) mParameters.get(CaptureRequest.JPEG_ORIENTATION);
+ if (orientationObj != null) {
+ mStillCaptureProcessor.setRotationDegrees(orientationObj);
+ }
+
+ Byte qualityObj = (Byte) mParameters.get(CaptureRequest.JPEG_QUALITY);
+ if (qualityObj != null) {
+ mStillCaptureProcessor.setJpegQuality((int) qualityObj);
+ }
+ }
+ }
+
+
+ private void submitRequestByCaptureStage(RequestProcessor requestProcessor,
+ CaptureStageImpl captureStage) {
+ RequestBuilder builder = new RequestBuilder();
+ builder.addTargetOutputConfigIds(mPreviewOutputConfig.getId());
+ if (mAnalysisOutputConfig != null) {
+ builder.addTargetOutputConfigIds(mAnalysisOutputConfig.getId());
+ }
+ for (Pair<CaptureRequest.Key, Object> keyObjectPair : captureStage.getParameters()) {
+ builder.setParameters(keyObjectPair.first, keyObjectPair.second);
+ }
+
+ builder.setTemplateId(CameraDevice.TEMPLATE_PREVIEW);
+ requestProcessor.submit(builder.build(), new RequestProcessor.Callback() {
+ });
+ }
+
+ @Override
+ public void onCaptureSessionEnd() {
+ CaptureStageImpl captureStage1 = mPreviewExtenderImpl.onDisableSession();
+ Logger.d(TAG, "preview onDisableSession: " + captureStage1);
+
+ CaptureStageImpl captureStage2 = mImageCaptureExtenderImpl.onDisableSession();
+ Logger.d(TAG, "capture onDisableSession:" + captureStage2);
+ if (captureStage1 != null) {
+ submitRequestByCaptureStage(mRequestProcessor, captureStage1);
+ }
+
+ if (captureStage2 != null) {
+ submitRequestByCaptureStage(mRequestProcessor, captureStage2);
+ }
+ mRequestProcessor = null;
+ mIsCapturing = false;
+ }
+
+ @Override
+ public int startRepeating(@NonNull CaptureCallback captureCallback) {
+ int repeatingCaptureSequenceId = mNextCaptureSequenceId.getAndIncrement();
+ if (mRequestProcessor == null) {
+ captureCallback.onCaptureFailed(repeatingCaptureSequenceId);
+ captureCallback.onCaptureSequenceAborted(repeatingCaptureSequenceId);
+ } else {
+ updateRepeating(repeatingCaptureSequenceId, captureCallback);
+ }
+
+ return repeatingCaptureSequenceId;
+ }
+
+ void updateRepeating(int repeatingCaptureSequenceId, @NonNull CaptureCallback captureCallback) {
+ if (mRequestProcessor == null) {
+ Logger.d(TAG, "mRequestProcessor is null, ignore repeating request");
+ return;
+ }
+ RequestBuilder builder = new RequestBuilder();
+ builder.addTargetOutputConfigIds(mPreviewOutputConfig.getId());
+ if (mAnalysisOutputConfig != null) {
+ builder.addTargetOutputConfigIds(mAnalysisOutputConfig.getId());
+ }
+ builder.setTemplateId(CameraDevice.TEMPLATE_PREVIEW);
+ applyParameters(builder);
+ CaptureStageImpl captureStage = mPreviewExtenderImpl.getCaptureStage();
+ if (captureStage != null) {
+ for (Pair<CaptureRequest.Key, Object> keyObjectPair :
+ captureStage.getParameters()) {
+ builder.setParameters(keyObjectPair.first, keyObjectPair.second);
+ }
+ }
+
+ RequestProcessor.Callback callback = new RequestProcessor.Callback() {
+ @Override
+ public void onCaptureCompleted(@NonNull RequestProcessor.Request request,
+ @NonNull CameraCaptureResult cameraCaptureResult) {
+ CaptureResult captureResult =
+ Camera2CameraCaptureResultConverter.getCaptureResult(
+ cameraCaptureResult);
+ Preconditions.checkArgument(captureResult instanceof TotalCaptureResult,
+ "Cannot get TotalCaptureResult from the cameraCaptureResult ");
+ TotalCaptureResult totalCaptureResult = (TotalCaptureResult) captureResult;
+
+ if (mPreviewProcessor != null) {
+ mPreviewProcessor.notifyCaptureResult(totalCaptureResult);
+ }
+
+ if (mRequestUpdateProcessor != null) {
+ CaptureStageImpl captureStage =
+ mRequestUpdateProcessor.process(totalCaptureResult);
+
+ if (captureStage != null) {
+ updateRepeating(repeatingCaptureSequenceId, captureCallback);
+ }
+ }
+
+ captureCallback.onCaptureSequenceCompleted(repeatingCaptureSequenceId);
+ }
+ };
+
+ Logger.d(TAG, "requestProcessor setRepeating");
+ mRequestProcessor.setRepeating(builder.build(), callback);
+ }
+
+ @Override
+ public void stopRepeating() {
+ mRequestProcessor.stopRepeating();
+ }
+
+ @Override
+ public int startCapture(@NonNull CaptureCallback captureCallback) {
+ int captureSequenceId = mNextCaptureSequenceId.getAndIncrement();
+
+ if (mRequestProcessor == null || mIsCapturing) {
+ Logger.d(TAG, "startCapture failed");
+ captureCallback.onCaptureFailed(captureSequenceId);
+ captureCallback.onCaptureSequenceAborted(captureSequenceId);
+ return captureSequenceId;
+ }
+ mIsCapturing = true;
+
+ List<RequestProcessor.Request> requestList = new ArrayList<>();
+ List<CaptureStageImpl> captureStages = mImageCaptureExtenderImpl.getCaptureStages();
+ List<Integer> captureIdList = new ArrayList<>();
+
+ for (CaptureStageImpl captureStage : captureStages) {
+ RequestBuilder builder = new RequestBuilder();
+ builder.addTargetOutputConfigIds(mCaptureOutputConfig.getId());
+ builder.setTemplateId(CameraDevice.TEMPLATE_STILL_CAPTURE);
+ builder.setCaptureStageId(captureStage.getId());
+
+ captureIdList.add(captureStage.getId());
+
+ applyParameters(builder);
+
+ for (Pair<CaptureRequest.Key, Object> keyObjectPair :
+ captureStage.getParameters()) {
+ builder.setParameters(keyObjectPair.first, keyObjectPair.second);
+ }
+ requestList.add(builder.build());
+ }
+
+ Logger.d(TAG, "Wait for capture stage id: " + captureIdList);
+
+ RequestProcessor.Callback callback = new RequestProcessor.Callback() {
+ boolean mIsCaptureFailed = false;
+ boolean mIsCaptureStarted = false;
+
+ @Override
+ public void onCaptureStarted(@NonNull RequestProcessor.Request request,
+ long frameNumber, long timestamp) {
+ if (!mIsCaptureStarted) {
+ mIsCaptureStarted = true;
+ captureCallback.onCaptureStarted(captureSequenceId, timestamp);
+ }
+ }
+
+ @Override
+ public void onCaptureCompleted(@NonNull RequestProcessor.Request request,
+ @NonNull CameraCaptureResult cameraCaptureResult) {
+ CaptureResult captureResult =
+ Camera2CameraCaptureResultConverter.getCaptureResult(
+ cameraCaptureResult);
+ Preconditions.checkArgument(captureResult instanceof TotalCaptureResult,
+ "Cannot get capture TotalCaptureResult from the cameraCaptureResult ");
+ TotalCaptureResult totalCaptureResult = (TotalCaptureResult) captureResult;
+
+ RequestBuilder.RequestProcessorRequest requestProcessorRequest =
+ (RequestBuilder.RequestProcessorRequest) request;
+
+ if (mStillCaptureProcessor != null) {
+ mStillCaptureProcessor.notifyCaptureResult(
+ totalCaptureResult,
+ requestProcessorRequest.getCaptureStageId());
+ } else {
+ captureCallback.onCaptureProcessStarted(captureSequenceId);
+ captureCallback.onCaptureSequenceCompleted(captureSequenceId);
+ mIsCapturing = false;
+ }
+ }
+
+ @Override
+ public void onCaptureFailed(@NonNull RequestProcessor.Request request,
+ @NonNull CameraCaptureFailure captureFailure) {
+ if (!mIsCaptureFailed) {
+ mIsCaptureFailed = true;
+ captureCallback.onCaptureFailed(captureSequenceId);
+ captureCallback.onCaptureSequenceAborted(captureSequenceId);
+ mIsCapturing = false;
+ }
+ }
+
+ @Override
+ public void onCaptureSequenceAborted(int sequenceId) {
+ captureCallback.onCaptureSequenceAborted(captureSequenceId);
+ mIsCapturing = false;
+ }
+ };
+
+ Logger.d(TAG, "startCapture");
+ if (mStillCaptureProcessor != null) {
+ mStillCaptureProcessor.startCapture(captureIdList,
+ new StillCaptureProcessor.OnCaptureResultCallback() {
+ @Override
+ public void onCompleted() {
+ captureCallback.onCaptureSequenceCompleted(captureSequenceId);
+ mIsCapturing = false;
+ }
+
+ @Override
+ public void onError(@NonNull Exception e) {
+ captureCallback.onCaptureFailed(captureSequenceId);
+ mIsCapturing = false;
+ }
+ });
+ }
+ setImageProcessor(mCaptureOutputConfig.getId(),
+ new ImageProcessor() {
+ boolean mIsFirstFrame = true;
+
+ @Override
+ public void onNextImageAvailable(int outputStreamId, long timestampNs,
+ @NonNull ImageReference imageReference,
+ @Nullable String physicalCameraId) {
+ Logger.d(TAG,
+ "onNextImageAvailable outputStreamId=" + outputStreamId);
+ if (mStillCaptureProcessor != null) {
+ mStillCaptureProcessor.notifyImage(imageReference);
+ }
+
+ if (mIsFirstFrame) {
+ captureCallback.onCaptureProcessStarted(captureSequenceId);
+ mIsFirstFrame = false;
+ }
+ }
+ });
+ mRequestProcessor.submit(requestList, callback);
+ return captureSequenceId;
+ }
+
+ @Override
+ public void abortCapture(int captureSequenceId) {
+ mRequestProcessor.abortCaptures();
+ }
+}
diff --git a/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/Camera2SessionConfigBuilder.java b/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/Camera2SessionConfigBuilder.java
index 60e13b7..57baff7 100644
--- a/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/Camera2SessionConfigBuilder.java
+++ b/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/Camera2SessionConfigBuilder.java
@@ -55,8 +55,8 @@
* Sets session parameters.
*/
@NonNull
- <T> Camera2SessionConfigBuilder addSessionParameter(
- @NonNull CaptureRequest.Key<T> key, @Nullable T value) {
+ Camera2SessionConfigBuilder addSessionParameter(
+ @NonNull CaptureRequest.Key key, @Nullable Object value) {
mSessionParameters.put(key, value);
return this;
}
diff --git a/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/ImageReaderOutputConfig.java b/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/ImageReaderOutputConfig.java
index 454e4a0..1ae12a1 100644
--- a/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/ImageReaderOutputConfig.java
+++ b/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/ImageReaderOutputConfig.java
@@ -24,6 +24,7 @@
import com.google.auto.value.AutoValue;
+import java.util.Collections;
import java.util.List;
/**
@@ -42,6 +43,12 @@
return new AutoValue_ImageReaderOutputConfig(id, surfaceGroupId, physicalCameraId,
sharedOutputConfigs, size, imageFormat, maxImages);
}
+
+ static ImageReaderOutputConfig create(
+ int id, @NonNull Size size, int imageFormat, int maxImages) {
+ return new AutoValue_ImageReaderOutputConfig(id, -1, null,
+ Collections.emptyList(), size, imageFormat, maxImages);
+ }
/**
* Returns the size of the surface.
*/
diff --git a/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/RequestBuilder.java b/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/RequestBuilder.java
new file mode 100644
index 0000000..32d17b9
--- /dev/null
+++ b/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/RequestBuilder.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.camera.extensions.internal.sessionprocessor;
+
+import android.hardware.camera2.CameraDevice;
+import android.hardware.camera2.CaptureRequest;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.OptIn;
+import androidx.annotation.RequiresApi;
+import androidx.camera.camera2.impl.Camera2ImplConfig;
+import androidx.camera.camera2.interop.ExperimentalCamera2Interop;
+import androidx.camera.core.impl.Config;
+import androidx.camera.core.impl.RequestProcessor;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A builder for building {@link androidx.camera.core.impl.RequestProcessor.Request}.
+ */
+@OptIn(markerClass = ExperimentalCamera2Interop.class)
+@RequiresApi(21) // TODO(b/200306659): Remove and replace with annotation on package-info.java
+class RequestBuilder {
+ private List<Integer> mTargetOutputConfigIds = new ArrayList<>();
+ private Map<CaptureRequest.Key<?>, Object> mParameters = new HashMap<>();
+ private int mTemplateId = CameraDevice.TEMPLATE_PREVIEW;
+ int mCaptureStageId;
+
+ RequestBuilder() {
+ }
+
+ @NonNull
+ RequestBuilder addTargetOutputConfigIds(int targetOutputConfigId) {
+ mTargetOutputConfigIds.add(targetOutputConfigId);
+ return this;
+ }
+
+ @NonNull
+ RequestBuilder setParameters(@NonNull CaptureRequest.Key<?> key,
+ @NonNull Object value) {
+ mParameters.put(key, value);
+ return this;
+ }
+
+ @NonNull
+ RequestBuilder setTemplateId(int templateId) {
+ mTemplateId = templateId;
+ return this;
+ }
+
+ @NonNull
+ public RequestBuilder setCaptureStageId(int captureStageId) {
+ mCaptureStageId = captureStageId;
+ return this;
+ }
+
+ @NonNull
+ RequestProcessor.Request build() {
+ return new RequestProcessorRequest(
+ mTargetOutputConfigIds, mParameters, mTemplateId, mCaptureStageId);
+ }
+
+ static class RequestProcessorRequest implements RequestProcessor.Request {
+ final List<Integer> mTargetOutputConfigIds;
+ final Config mParameterConfig;
+ final int mTemplateId;
+ final int mCaptureStageId;
+
+ RequestProcessorRequest(List<Integer> targetOutputConfigIds,
+ Map<CaptureRequest.Key<?>, Object> parameters,
+ int templateId,
+ int captureStageId) {
+ mTargetOutputConfigIds = targetOutputConfigIds;
+ mTemplateId = templateId;
+ mCaptureStageId = captureStageId;
+
+ Camera2ImplConfig.Builder camera2ConfigBuilder = new Camera2ImplConfig.Builder();
+ for (CaptureRequest.Key<?> key : parameters.keySet()) {
+ @SuppressWarnings("unchecked")
+ CaptureRequest.Key<Object> objKey = (CaptureRequest.Key<Object>) key;
+ camera2ConfigBuilder.setCaptureRequestOption(objKey,
+ parameters.get(objKey));
+ }
+ mParameterConfig = camera2ConfigBuilder.build();
+ }
+
+ @Override
+ @NonNull
+ public List<Integer> getTargetOutputConfigIds() {
+ return mTargetOutputConfigIds;
+ }
+
+ @Override
+ @NonNull
+ public Config getParameters() {
+ return mParameterConfig;
+ }
+
+ @Override
+ public int getTemplateId() {
+ return mTemplateId;
+ }
+
+ public int getCaptureStageId() {
+ return mCaptureStageId;
+ }
+ }
+}
diff --git a/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/SurfaceOutputConfig.java b/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/SurfaceOutputConfig.java
index 31a7144..4c56295 100644
--- a/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/SurfaceOutputConfig.java
+++ b/camera/camera-extensions/src/main/java/androidx/camera/extensions/internal/sessionprocessor/SurfaceOutputConfig.java
@@ -24,6 +24,7 @@
import com.google.auto.value.AutoValue;
+import java.util.Collections;
import java.util.List;
/**
@@ -44,6 +45,10 @@
sharedOutputConfigs, surface);
}
+ static SurfaceOutputConfig create(int id, @NonNull Surface surface) {
+ return create(id, -1, null, Collections.emptyList(), surface);
+ }
+
/**
* Get the {@link Surface}. It'll return a valid surface only when type is TYPE_SURFACE.
*/
diff --git a/camera/camera-testing/src/main/java/androidx/camera/testing/SurfaceTextureProvider.java b/camera/camera-testing/src/main/java/androidx/camera/testing/SurfaceTextureProvider.java
index 8aa0f5c..8387b97 100644
--- a/camera/camera-testing/src/main/java/androidx/camera/testing/SurfaceTextureProvider.java
+++ b/camera/camera-testing/src/main/java/androidx/camera/testing/SurfaceTextureProvider.java
@@ -30,6 +30,7 @@
import android.view.TextureView;
import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.camera.core.Logger;
import androidx.camera.core.Preview;
@@ -132,6 +133,21 @@
*/
@NonNull
public static Preview.SurfaceProvider createAutoDrainingSurfaceTextureProvider() {
+ return createAutoDrainingSurfaceTextureProvider(null);
+ }
+
+ /**
+ * Creates a {@link Preview.SurfaceProvider} that is backed by a {@link SurfaceTexture}.
+ *
+ * <p>This method also creates a backing OpenGL thread that will automatically drain frames
+ * from the SurfaceTexture as they become available.
+ *
+ * @param frameAvailableListener listener to be invoked when frame is updated.
+ */
+ @NonNull
+ public static Preview.SurfaceProvider createAutoDrainingSurfaceTextureProvider(
+ @Nullable SurfaceTexture.OnFrameAvailableListener frameAvailableListener
+ ) {
return (surfaceRequest) -> {
HandlerThread handlerThread = new HandlerThread(String.format("CameraX"
+ "-AutoDrainThread-%x", surfaceRequest.hashCode()));
@@ -147,8 +163,12 @@
SurfaceTexture surfaceTexture = new SurfaceTexture(textureIds[0]);
surfaceTexture.setDefaultBufferSize(surfaceRequest.getResolution().getWidth(),
surfaceRequest.getResolution().getHeight());
- surfaceTexture.setOnFrameAvailableListener(
- SurfaceTexture::updateTexImage, handler);
+ surfaceTexture.setOnFrameAvailableListener((st) -> {
+ st.updateTexImage();
+ if (frameAvailableListener != null) {
+ frameAvailableListener.onFrameAvailable(st);
+ }
+ }, handler);
Surface surface = new Surface(surfaceTexture);
surfaceRequest.provideSurface(surface,
