Skip to content

Commit

Permalink
Refine "join" mode in video renderer for surface changes.
Browse files Browse the repository at this point in the history 
The join mode is used for two cases: surface switching and mid-playback
enabling of video.

In both cases, we want to pretend to be ready despite not having rendered
a new "first frame". So far, we also avoided force-rendering the first
frame immediately because it causes a stuttering effect for the
mid-playback enable case. The surface switch case doesn't have this
stuttering issue as the same codec is used without interruption. Not
force-rendering the frame immediately causes the first-frame rendered
callback to arrive too early though, which may lead to cases where
apps hide shutter views too quickly.

This problem can be solved by only avoiding the force-render for the
mid-playback enabling case, but not for the surface switching case.

PiperOrigin-RevId: 622105916
  • Loading branch information
@tonihei
tonihei authored and Copybara-Service committed Apr 5, 2024
1 parent 8867642 commit e0fa697
Show file tree
Hide file tree
Showing 4 changed files with 113 additions and 23 deletions.
2 changes: 2 additions & 0 deletions RELEASENOTES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@
* Fix issue where HDR color info handling causes codec mishavior and
prevents adaptive format switches for SDR video tracks
([#1158](https://github.com/androidx/media/issues/1158)).
* Fix issue where `Listener.onRenderedFirstFrame()` arrives too early when
switching surfaces mid-playback.
* Text:
* WebVTT: Prevent directly consecutive cues from creating spurious
additional `CuesWithTiming` instances from `WebvttParser.parse`
Expand Down
11 changes: 9 additions & 2 deletions ...ries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -660,7 +660,10 @@ protected void onPositionReset(long positionUs, boolean joining) throws ExoPlayb
}
videoFrameReleaseControl.reset();
if (joining) {
videoFrameReleaseControl.join();
// Don't render next frame immediately to let the codec catch up with the playback position
// first. This prevents a stuttering effect caused by showing the first frame and then
// dropping many of the subsequent frames during the catch up phase.
videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false);
}
maybeSetupTunnelingForFirstFrame();
consecutiveDroppedFrameCount = 0;
Expand Down Expand Up @@ -835,7 +838,11 @@ private void setOutput(@Nullable Object output) throws ExoPlaybackException {
// If we know the video size, report it again immediately.
maybeRenotifyVideoSizeChanged();
if (state == STATE_STARTED) {
videoFrameReleaseControl.join();
// We want to "join" playback to prevent an intermediate buffering state in the player
// before we rendered the new first frame. Since there is no reason to believe the next
// frame is delayed and the renderer needs to catch up, we still request to render the
// next frame as soon as possible.
videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true);
}
// When effects previewing is enabled, set display surface and an unknown size.
if (videoSinkProvider.isInitialized()) {
Expand Down
23 changes: 17 additions & 6 deletions ...ies/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameReleaseControl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@ boolean shouldIgnoreFrame(
private long lastReleaseRealtimeUs;
private long lastPresentationTimeUs;
private long joiningDeadlineMs;
private boolean joiningRenderNextFrameImmediately;
private float playbackSpeed;
private Clock clock;

Expand Down Expand Up @@ -298,8 +299,17 @@ public boolean isReady(boolean rendererReady) {
}
}

/** Joins the release control to a new stream. */
public void join() {
/**
* Joins the release control to a new stream.
*
* <p>The release control will pretend to be {@linkplain #isReady ready} for short time even if
* the first frame hasn't been rendered yet to avoid interrupting an ongoing playback.
*
* @param renderNextFrameImmediately Whether the next frame should be released as soon as possible
* or only at its preferred scheduled release time.
*/
public void join(boolean renderNextFrameImmediately) {
joiningRenderNextFrameImmediately = renderNextFrameImmediately;
joiningDeadlineMs =
allowedJoiningTimeMs > 0 ? (clock.elapsedRealtime() + allowedJoiningTimeMs) : C.TIME_UNSET;
}
Expand Down Expand Up @@ -353,8 +363,9 @@ public void join() {
frameReleaseInfo.releaseTimeNs =
frameReleaseHelper.adjustReleaseTime(systemTimeNs + (frameReleaseInfo.earlyUs * 1_000));
frameReleaseInfo.earlyUs = (frameReleaseInfo.releaseTimeNs - systemTimeNs) / 1_000;
// While joining, late frames are skipped.
boolean treatDropAsSkip = joiningDeadlineMs != C.TIME_UNSET;
// While joining, late frames are skipped while we catch up with the playback position.
boolean treatDropAsSkip =
joiningDeadlineMs != C.TIME_UNSET && !joiningRenderNextFrameImmediately;
if (frameTimingEvaluator.shouldIgnoreFrame(
frameReleaseInfo.earlyUs, positionUs, elapsedRealtimeUs, isLastFrame, treatDropAsSkip)) {
return FRAME_RELEASE_IGNORE;
Expand Down Expand Up @@ -425,8 +436,8 @@ private long calculateEarlyTimeUs(
/** Returns whether a frame should be force released. */
private boolean shouldForceRelease(
long positionUs, long earlyUs, long outputStreamStartPositionUs) {
if (joiningDeadlineMs != C.TIME_UNSET) {
// No force releasing during joining.
if (joiningDeadlineMs != C.TIME_UNSET && !joiningRenderNextFrameImmediately) {
// No force releasing of the initial or late frames during joining unless requested.
return false;
}
switch (firstFrameState) {
Expand Down
100 changes: 85 additions & 15 deletions ...exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameReleaseControlTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,25 +44,52 @@ public void isReady_afterReleasingFrame_returnsTrue() {
}

@Test
public void isReady_withinJoiningDeadline_returnsTrue() {
public void isReady_withinJoiningDeadlineWhenRenderingNextFrameImmediately_returnsTrue() {
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl =
createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100);
videoFrameReleaseControl.setClock(clock);

videoFrameReleaseControl.join();
videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true);

assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue();
}

@Test
public void isReady_joiningDeadlineExceeded_returnsFalse() {
public void isReady_withinJoiningDeadlineWhenNotRenderingNextFrameImmediately_returnsTrue() {
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl =
createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100);
videoFrameReleaseControl.setClock(clock);

videoFrameReleaseControl.join();
videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false);

assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue();
}

@Test
public void isReady_joiningDeadlineExceededWhenRenderingNextFrameImmediately_returnsFalse() {
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl =
createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100);
videoFrameReleaseControl.setClock(clock);

videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true);
assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue();

clock.advanceTime(/* timeDiffMs= */ 101);

assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isFalse();
}

@Test
public void isReady_joiningDeadlineExceededWhenNotRenderingNextFrameImmediately_returnsFalse() {
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl =
createVideoFrameReleaseControl(/* allowedJoiningTimeMs= */ 100);
videoFrameReleaseControl.setClock(clock);

videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false);
assertThat(videoFrameReleaseControl.isReady(/* rendererReady= */ false)).isTrue();

clock.advanceTime(/* timeDiffMs= */ 101);
Expand Down Expand Up @@ -323,7 +350,9 @@ public void getFrameReleaseAction_frameLate_returnsDrop() throws ExoPlaybackExce
}

@Test
public void getFrameReleaseAction_dropWhileJoining_returnsSkip() throws ExoPlaybackException {
public void
getFrameReleaseAction_lateFrameWhileJoiningWhenNotRenderingFirstFrameImmediately_returnsSkip()
throws ExoPlaybackException {
VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
new VideoFrameReleaseControl.FrameReleaseInfo();
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
Expand All @@ -337,26 +366,56 @@ public void getFrameReleaseAction_dropWhileJoining_returnsSkip() throws ExoPlayb
/* allowedJoiningTimeMs= */ 1234);
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);

videoFrameReleaseControl.onStarted();

// First frame released.
// Start joining.
videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false);

// First output is TRY_AGAIN_LATER because the time hasn't moved yet
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 0,
/* positionUs= */ 0,
/* presentationTimeUs= */ 5_000,
/* positionUs= */ 10_000,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
videoFrameReleaseControl.onFrameReleasedIsFirstFrame();
clock.advanceTime(/* timeDiffMs= */ 40);
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER);
// Late frame should be marked as skipped
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 5_000,
/* positionUs= */ 11_000,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SKIP);
}

@Test
public void
getFrameReleaseAction_lateFrameWhileJoiningWhenRenderingFirstFrameImmediately_returnsDropAfterInitialImmediateRelease()
throws ExoPlaybackException {
VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo =
new VideoFrameReleaseControl.FrameReleaseInfo();
FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
VideoFrameReleaseControl videoFrameReleaseControl =
new VideoFrameReleaseControl(
ApplicationProvider.getApplicationContext(),
new TestFrameTimingEvaluator(
/* shouldForceRelease= */ false,
/* shouldDropFrame= */ true,
/* shouldIgnoreFrame= */ false),
/* allowedJoiningTimeMs= */ 1234);
videoFrameReleaseControl.setClock(clock);
videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
videoFrameReleaseControl.onStarted();

// Start joining.
videoFrameReleaseControl.join();
videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ true);

// Second frame.
// First output is to force render the next frame.
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 5_000,
Expand All @@ -365,7 +424,18 @@ public void getFrameReleaseAction_dropWhileJoining_returnsSkip() throws ExoPlayb
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_SKIP);
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
videoFrameReleaseControl.onFrameReleasedIsFirstFrame();
// Further late frames should be marked as dropped.
assertThat(
videoFrameReleaseControl.getFrameReleaseAction(
/* presentationTimeUs= */ 6_000,
/* positionUs= */ 11_000,
/* elapsedRealtimeUs= */ 0,
/* outputStreamStartPositionUs= */ 0,
/* isLastFrame= */ false,
frameReleaseInfo))
.isEqualTo(VideoFrameReleaseControl.FRAME_RELEASE_DROP);
}

@Test
Expand Down

0 comments on commit e0fa697

Please sign in to comment.