Skip to content

Commit

Permalink
Add support for ultra HDR overlays
Browse files Browse the repository at this point in the history
PiperOrigin-RevId: 637863706
  • Loading branch information
tof-tof authored and Copybara-Service committed May 28, 2024
1 parent 3bb6cf2 commit 9622411
Show file tree
Hide file tree
Showing 9 changed files with 354 additions and 52 deletions.
1 change: 1 addition & 0 deletions RELEASENOTES.md
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@
([#1055](https://github.com/androidx/media/pull/1055)).
* Maintain a consistent luminance range across different HDR content (uses
the HLG range).
* Add support for Ultra HDR (bitmap) overlays on HDR content.
* Muxers:
* IMA extension:
* Promote API that is required for apps to play
Expand Down
72 changes: 72 additions & 0 deletions libraries/effect/src/main/assets/shaders/insert_ultra_hdr.glsl
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
// The value is calculated as targetHdrPeakBrightnessInNits /
// targetSdrWhitePointInNits. In other effect HDR processing and some parts of
// the wider android ecosystem the assumption is
// targetHdrPeakBrightnessInNits=1000 and targetSdrWhitePointInNits=500
const float HDR_SDR_RATIO = 2.0;

// Matrix values are calculated as inverse of RGB_BT2020_TO_XYZ.
const mat3 XYZ_TO_RGB_BT2020 =
mat3(1.71665, -0.666684, 0.0176399, -0.355671, 1.61648, -0.0427706,
-0.253366, 0.0157685, 0.942103);
// Matrix values are calculated as inverse of XYZ_TO_RGB_BT709.
const mat3 RGB_BT709_TO_XYZ =
mat3(0.412391, 0.212639, 0.0193308, 0.357584, 0.715169, 0.119195, 0.180481,
0.0721923, 0.950532);

// Reference:
// https://developer.android.com/reference/android/graphics/Gainmap#applying-a-gainmap-manually
// Reference Implementation:
// https://cs.android.com/android/platform/superproject/main/+/main:frameworks/base/libs/hwui/effects/GainmapRenderer.cpp;l=117-146;drc=fadc20184ccb27fe15bb862e6e03fa6d05d41eac
highp vec3 applyGainmap(vec4 S, vec4 G, int uGainmapIsAlpha, int uNoGamma,
int uSingleChannel, vec4 uLogRatioMin,
vec4 uLogRatioMax, vec4 uEpsilonSdr, vec4 uEpsilonHdr,
vec4 uGainmapGamma, float uDisplayRatioHdr,
float uDisplayRatioSdr) {
float W = clamp((log(HDR_SDR_RATIO) - log(uDisplayRatioSdr)) /
(log(uDisplayRatioHdr) - log(uDisplayRatioSdr)),
0.0, 1.0);
vec3 H;
if (uGainmapIsAlpha == 1) {
G = vec4(G.a, G.a, G.a, 1.0);
}
if (uSingleChannel == 1) {
mediump float L;
if (uNoGamma == 1) {
L = mix(uLogRatioMin.r, uLogRatioMax.r, G.r);
} else {
L = mix(uLogRatioMin.r, uLogRatioMax.r, pow(G.r, uGainmapGamma.r));
}
H = (S.rgb + uEpsilonSdr.rgb) * exp(L * W) - uEpsilonHdr.rgb;
} else {
mediump vec3 L;
if (uNoGamma == 1) {
L = mix(uLogRatioMin.rgb, uLogRatioMax.rgb, G.rgb);
} else {
L = mix(uLogRatioMin.rgb, uLogRatioMax.rgb,
pow(G.rgb, uGainmapGamma.rgb));
}
H = (S.rgb + uEpsilonSdr.rgb) * exp(L * W) - uEpsilonHdr.rgb;
}
return H;
}

highp vec3 bt709ToBt2020(vec3 bt709Color) {
return XYZ_TO_RGB_BT2020 * RGB_BT709_TO_XYZ * bt709Color;
}

vec3 scaleHdrLuminance(vec3 linearColor) {
const float SDR_MAX_LUMINANCE = 500.0;
const float HDR_MAX_LUMINANCE = 1000.0;
return linearColor * SDR_MAX_LUMINANCE / HDR_MAX_LUMINANCE;
}

// sRGB EOTF for one channel.
float srgbEotfSingleChannel(float srgb) {
return srgb <= 0.04045 ? srgb / 12.92 : pow((srgb + 0.055) / 1.055, 2.4);
}

// sRGB EOTF.
vec4 srgbEotf(vec4 srgb) {
return vec4(srgbEotfSingleChannel(srgb.r), srgbEotfSingleChannel(srgb.g),
srgbEotfSingleChannel(srgb.b), srgb.a);
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,8 @@
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR;
import static java.lang.Math.log;

import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.Gainmap;
import android.opengl.GLES20;
import android.opengl.Matrix;
Expand Down Expand Up @@ -694,33 +692,6 @@ private void setGainmapSamplerAndUniforms() throws GlUtil.GlException {
throw new IllegalStateException("Gainmaps not supported under API 34.");
}
glProgram.setSamplerTexIdUniform("uGainmapTexSampler", gainmapTexId, /* texUnitIndex= */ 1);

boolean gainmapIsAlpha = lastGainmap.getGainmapContents().getConfig() == Bitmap.Config.ALPHA_8;
float[] gainmapGamma = lastGainmap.getGamma();
boolean noGamma = gainmapGamma[0] == 1f && gainmapGamma[1] == 1f && gainmapGamma[2] == 1f;
boolean singleChannel =
areAllChannelsEqual(gainmapGamma)
&& areAllChannelsEqual(lastGainmap.getRatioMax())
&& areAllChannelsEqual(lastGainmap.getRatioMin());

glProgram.setIntUniform("uGainmapIsAlpha", gainmapIsAlpha ? GL_TRUE : GL_FALSE);
glProgram.setIntUniform("uNoGamma", noGamma ? GL_TRUE : GL_FALSE);
glProgram.setIntUniform("uSingleChannel", singleChannel ? GL_TRUE : GL_FALSE);
glProgram.setFloatsUniform("uLogRatioMin", logRgb(lastGainmap.getRatioMin()));
glProgram.setFloatsUniform("uLogRatioMax", logRgb(lastGainmap.getRatioMax()));
glProgram.setFloatsUniform("uEpsilonSdr", lastGainmap.getEpsilonSdr());
glProgram.setFloatsUniform("uEpsilonHdr", lastGainmap.getEpsilonHdr());
glProgram.setFloatsUniform("uGainmapGamma", gainmapGamma);
glProgram.setFloatUniform("uDisplayRatioHdr", lastGainmap.getDisplayRatioForFullHdr());
glProgram.setFloatUniform("uDisplayRatioSdr", lastGainmap.getMinDisplayRatioForHdrTransition());
GlUtil.checkGlError();
}

private static boolean areAllChannelsEqual(float[] channels) {
return channels[0] == channels[1] && channels[1] == channels[2];
}

private static float[] logRgb(float[] values) {
return new float[] {(float) log(values[0]), (float) log(values[1]), (float) log(values[2])};
GainmapUtil.setGainmapUniforms(glProgram, lastGainmap, C.INDEX_UNSET);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,16 @@
*/
package androidx.media3.effect;

import static android.opengl.GLES20.GL_FALSE;
import static android.opengl.GLES20.GL_TRUE;
import static java.lang.Math.log;

import android.graphics.Bitmap;
import android.graphics.Gainmap;
import androidx.annotation.RequiresApi;
import androidx.media3.common.C;
import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;

/** Utilities for Gainmaps. */
Expand All @@ -37,4 +45,51 @@ public static boolean equals(Gainmap g1, Gainmap g2) {
&& g1.getGainmapContents() == g2.getGainmapContents()
&& g1.getGainmapContents().getGenerationId() == g2.getGainmapContents().getGenerationId();
}

/**
* Sets the uniforms for applying a gainmap to a base image.
*
* @param glProgram The {@link GlProgram}.
* @param gainmap The {@link Gainmap}.
* @param index The index to add to the end of the uniforms, or {@link C#INDEX_UNSET}, is no index
* is to be added.
*/
@RequiresApi(34)
public static void setGainmapUniforms(GlProgram glProgram, Gainmap gainmap, int index)
throws GlUtil.GlException {
boolean gainmapIsAlpha = gainmap.getGainmapContents().getConfig() == Bitmap.Config.ALPHA_8;
float[] gainmapGamma = gainmap.getGamma();
boolean noGamma = gainmapGamma[0] == 1f && gainmapGamma[1] == 1f && gainmapGamma[2] == 1f;
boolean singleChannel =
areAllChannelsEqual(gainmapGamma)
&& areAllChannelsEqual(gainmap.getRatioMax())
&& areAllChannelsEqual(gainmap.getRatioMin());

glProgram.setIntUniform(
addIndex("uGainmapIsAlpha", index), gainmapIsAlpha ? GL_TRUE : GL_FALSE);
glProgram.setIntUniform(addIndex("uNoGamma", index), noGamma ? GL_TRUE : GL_FALSE);
glProgram.setIntUniform(addIndex("uSingleChannel", index), singleChannel ? GL_TRUE : GL_FALSE);
glProgram.setFloatsUniform(addIndex("uLogRatioMin", index), logRgb(gainmap.getRatioMin()));
glProgram.setFloatsUniform(addIndex("uLogRatioMax", index), logRgb(gainmap.getRatioMax()));
glProgram.setFloatsUniform(addIndex("uEpsilonSdr", index), gainmap.getEpsilonSdr());
glProgram.setFloatsUniform(addIndex("uEpsilonHdr", index), gainmap.getEpsilonHdr());
glProgram.setFloatsUniform(addIndex("uGainmapGamma", index), gainmapGamma);
glProgram.setFloatUniform(
addIndex("uDisplayRatioHdr", index), gainmap.getDisplayRatioForFullHdr());
glProgram.setFloatUniform(
addIndex("uDisplayRatioSdr", index), gainmap.getMinDisplayRatioForHdrTransition());
GlUtil.checkGlError();
}

private static boolean areAllChannelsEqual(float[] channels) {
return channels[0] == channels[1] && channels[1] == channels[2];
}

private static String addIndex(String s, int index) {
return index == C.INDEX_UNSET ? s : s + index;
}

private static float[] logRgb(float[] values) {
return new float[] {(float) log(values[0]), (float) log(values[1]), (float) log(values[2])};
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@
* is displayed on top).
*
* <p>This effect assumes a non-{@linkplain DefaultVideoFrameProcessor#WORKING_COLOR_SPACE_LINEAR
* linear} working color space.
* linear} working color space for SDR input and a {@linkplain
* DefaultVideoFrameProcessor#WORKING_COLOR_SPACE_LINEAR linear} working color space or HDR input.
*/
@UnstableApi
public final class OverlayEffect implements GlEffect {
Expand All @@ -44,6 +45,6 @@ public OverlayEffect(ImmutableList<TextureOverlay> textureOverlays) {
@Override
public BaseGlShaderProgram toGlShaderProgram(Context context, boolean useHdr)
throws VideoFrameProcessingException {
return new OverlayShaderProgram(useHdr, overlays);
return new OverlayShaderProgram(context, useHdr, overlays);
}
}
Loading

0 comments on commit 9622411

Please sign in to comment.