Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add luma & chroma bit depths to Format #491

Merged
merged 15 commits into from
Oct 17, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Updated to use ColorInfo.Builder() and formatting issues
  • Loading branch information
microkatz committed Oct 17, 2023
commit d206be74ef129b5660da9d1df19d70920e6ec1b1
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.util.Arrays;
import org.checkerframework.dataflow.qual.Pure;
// copybara:exo-only import com.google.android.exoplayer2.Bundleable;
// copybara:exo-only import com.google.android.exoplayer2.C;
// copybara:exo-only import com.google.android.exoplayer2.Format;

/**
* Stores color info.
Expand Down Expand Up @@ -320,20 +323,16 @@ public boolean isDataSpaceValid() {
* @see Format#toLogString(Format)
*/
public String toLogString() {
if (!isValid()) {
return "NA";
}

String bitdepthsString = isBitdepthValid() ? lumaBitdepth + "/" + chromaBitdepth : "NA";
String dataspaceString =
isDataSpaceValid()
? Util.formatInvariant(
"%s/%s/%s",
colorSpaceToString(colorSpace),
colorRangeToString(colorRange),
colorTransferToString(colorTransfer))
: "NA";
return bitdepthsString + "/" + dataspaceString;
: "NA/NA/NA";
String bitdepthsString = isBitdepthValid() ? lumaBitdepth + "/" + chromaBitdepth : "NA/NA";
return dataspaceString + "/" + bitdepthsString;
}

@Override
Expand All @@ -353,20 +352,35 @@ public boolean equals(@Nullable Object obj) {
&& chromaBitdepth == other.chromaBitdepth;
}

@Override
public int hashCode() {
if (hashCode == 0) {
int result = 17;
result = 31 * result + colorSpace;
result = 31 * result + colorRange;
result = 31 * result + colorTransfer;
result = 31 * result + Arrays.hashCode(hdrStaticInfo);
result = 31 * result + lumaBitdepth;
result = 31 * result + chromaBitdepth;
hashCode = result;
}
return hashCode;
}

@Override
public String toString() {
return "ColorInfo("
+ lumaBitdepthToString(lumaBitdepth)
+ ", "
+ chromaBitdepthToString(chromaBitdepth)
+ ", "
+ colorSpaceToString(colorSpace)
+ ", "
+ colorRangeToString(colorRange)
+ ", "
+ colorTransferToString(colorTransfer)
+ ", "
+ (hdrStaticInfo != null)
+ ", "
+ lumaBitdepthToString(lumaBitdepth)
+ ", "
+ chromaBitdepthToString(chromaBitdepth)
+ ")";
}

Expand All @@ -392,6 +406,7 @@ private static String colorSpaceToString(@C.ColorSpace int colorSpace) {
default:
return "Undefined color space";
}
// LINT.ThenChange(C.java:color_space)
}

private static String colorTransferToString(@C.ColorTransfer int colorTransfer) {
Expand All @@ -414,6 +429,7 @@ private static String colorTransferToString(@C.ColorTransfer int colorTransfer)
default:
return "Undefined color transfer";
}
// LINT.ThenChange(C.java:color_transfer)
}

private static String colorRangeToString(@C.ColorRange int colorRange) {
Expand All @@ -428,21 +444,7 @@ private static String colorRangeToString(@C.ColorRange int colorRange) {
default:
return "Undefined color range";
}
}

@Override
public int hashCode() {
if (hashCode == 0) {
int result = 17;
result = 31 * result + colorSpace;
result = 31 * result + colorRange;
result = 31 * result + colorTransfer;
result = 31 * result + Arrays.hashCode(hdrStaticInfo);
result = 31 * result + lumaBitdepth;
result = 31 * result + chromaBitdepth;
hashCode = result;
}
return hashCode;
// LINT.ThenChange(C.java:color_range)
}

// Bundleable implementation
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@
import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;
// copybara:exo-only import com.google.android.exoplayer2.drm.DrmInitData;
// copybara:exo-only import com.google.android.exoplayer2.util.MimeTypes;
// copybara:exo-only import com.google.android.exoplayer2.video.ColorInfo;
// copybara:exo-only import com.google.android.exoplayer2.metadata.Metadata;

/** Unit test for {@link Format}. */
@RunWith(AndroidJUnit4.class)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,28 +33,21 @@
public final class NalUnitUtil {

private static final String TAG = "NalUnitUtil";

/** Coded slice of a non-IDR picture. */
public static final int NAL_UNIT_TYPE_NON_IDR = 1;

/** Coded slice data partition A. */
public static final int NAL_UNIT_TYPE_PARTITION_A = 2;

/** Coded slice of an IDR picture. */
public static final int NAL_UNIT_TYPE_IDR = 5;

/** Supplemental enhancement information. */
public static final int NAL_UNIT_TYPE_SEI = 6;

/** Sequence parameter set. */
public static final int NAL_UNIT_TYPE_SPS = 7;

/** Picture parameter set. */
public static final int NAL_UNIT_TYPE_PPS = 8;

/** Access unit delimiter. */
public static final int NAL_UNIT_TYPE_AUD = 9;

/** Holds data parsed from a H.264 sequence parameter set NAL unit. */
public static final class SpsData {

Expand Down
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since you are adding the luma and chroma information to vp9 in the mp4 AtomParsers, if you could please at the parsing logic for vp9 for rtsp in this file as well. It will require parsing the fmtp parameters if they exist.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, looking into this...

Copy link
Contributor Author

@dsparano dsparano Jul 25, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I'm correct from https://www.ietf.org/archive/id/draft-ietf-payload-vp9-16.txt the RTP payload for VP9 is not final yet. In the current draft the fmtp parameters for VP9 seem only optional: "max-fr", "max-fs" and "profile-id". None of them allows setting the bit depths. Incidentally I've played with mediamtx and I see it doesn't add fmtp at all for VP9.
Anyway, I can add a processVP9FmtpAttribute() method.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, in more detail: I could add it, but it seems to me we couldn't get anything useful out of it, the only possibly relevant bit is the profile-id however for values 0 and 1 the bit depth is 8, which is already the default in Format.Builder, while for values of 2 and 3 the bit depth can be either 10 or 12, so we couldn't set any value. So @microkatz can we skip this?

Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@
// Format specific parameter names.
private static final String PARAMETER_PROFILE_LEVEL_ID = "profile-level-id";
private static final String PARAMETER_SPROP_PARAMS = "sprop-parameter-sets";

private static final String PARAMETER_AMR_OCTET_ALIGN = "octet-align";
private static final String PARAMETER_AMR_INTERLEAVING = "interleaving";
private static final String PARAMETER_H265_SPROP_SPS = "sprop-sps";
Expand All @@ -60,16 +59,12 @@
private static final String PARAMETER_H265_SPROP_MAX_DON_DIFF = "sprop-max-don-diff";
private static final String PARAMETER_MP4A_CONFIG = "config";
private static final String PARAMETER_MP4A_C_PRESENT = "cpresent";

/** Prefix for the RFC6381 codecs string for AAC formats. */
private static final String AAC_CODECS_PREFIX = "mp4a.40.";

/** Prefix for the RFC6381 codecs string for AVC formats. */
private static final String H264_CODECS_PREFIX = "avc1.";

/** Prefix for the RFC6416 codecs string for MPEG4V-ES formats. */
private static final String MPEG4_CODECS_PREFIX = "mp4v.";

private static final String GENERIC_CONTROL_ATTR = "*";

/**
Expand Down Expand Up @@ -101,7 +96,6 @@
* software VP8 decoder</a>.
*/
private static final int DEFAULT_VP8_WIDTH = 320;

/**
* Default height for VP8.
*
Expand All @@ -125,7 +119,6 @@
* software VP9 decoder</a>.
*/
private static final int DEFAULT_VP9_WIDTH = 320;

/**
* Default height for VP9.
*
Expand All @@ -146,7 +139,6 @@
* >Android's software H263 decoder</a>.
*/
private static final int DEFAULT_H263_WIDTH = 352;

/**
* Default height for H263.
*
Expand All @@ -159,7 +151,6 @@

/** The track's associated {@link RtpPayloadFormat}. */
public final RtpPayloadFormat payloadFormat;

/** The track's URI. */
public final Uri uri;

Expand Down Expand Up @@ -422,14 +413,13 @@ private static void processH264FmtpAttribute(
formatBuilder.setPixelWidthHeightRatio(spsData.pixelWidthHeightRatio);
formatBuilder.setHeight(spsData.height);
formatBuilder.setWidth(spsData.width);
formatBuilder.setColorInfo(
new ColorInfo(
spsData.colorSpace,
spsData.colorRange,
spsData.colorTransfer,
null,
spsData.bitDepthLumaMinus8 + 8,
spsData.bitDepthChromaMinus8 + 8));
formatBuilder.setColorInfo(new ColorInfo.Builder()
.setColorSpace(spsData.colorSpace)
.setColorRange(spsData.colorRange)
.setColorTransfer(spsData.colorTransfer)
.setLumaBitdepth(spsData.bitDepthLumaMinus8 + 8)
.setChromaBitdepth(spsData.bitDepthChromaMinus8 + 8)
.build());

@Nullable String profileLevel = fmtpAttributes.get(PARAMETER_PROFILE_LEVEL_ID);
if (profileLevel != null) {
Expand Down Expand Up @@ -473,14 +463,13 @@ private static void processH265FmtpAttribute(
spsNalDataWithStartCode, NAL_START_CODE.length, spsNalDataWithStartCode.length);
formatBuilder.setPixelWidthHeightRatio(spsData.pixelWidthHeightRatio);
formatBuilder.setHeight(spsData.height).setWidth(spsData.width);
formatBuilder.setColorInfo(
new ColorInfo(
spsData.colorSpace,
spsData.colorRange,
spsData.colorTransfer,
null,
spsData.bitDepthLumaMinus8 + 8,
spsData.bitDepthChromaMinus8 + 8));
formatBuilder.setColorInfo(new ColorInfo.Builder()
.setColorSpace(spsData.colorSpace)
.setColorRange(spsData.colorRange)
.setColorTransfer(spsData.colorTransfer)
.setLumaBitdepth(spsData.bitDepthLumaMinus8 + 8)
.setChromaBitdepth(spsData.bitDepthChromaMinus8 + 8)
.build());

formatBuilder.setCodecs(
CodecSpecificDataUtil.buildHevcCodecString(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import androidx.media3.container.NalUnitUtil.SpsData;
import java.util.ArrayList;
import java.util.List;
// copybara:exo-only import com.google.android.exoplayer2.util.NalUnitUtil;

/** AVC configuration data. */
@UnstableApi
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import androidx.media3.container.NalUnitUtil;
import java.util.Collections;
import java.util.List;
// copybara:exo-only import com.google.android.exoplayer2.util.NalUnitUtil;

/** HEVC configuration data. */
@UnstableApi
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,6 @@ public class MatroskaExtractor implements Extractor {
flag = true,
value = {FLAG_DISABLE_SEEK_FOR_CUES})
public @interface Flags {}

/**
* Flag to disable seeking for cues.
*
Expand Down Expand Up @@ -260,7 +259,6 @@ public class MatroskaExtractor implements Extractor {
* https://www.matroska.org/technical/codec_specs.html.
*/
private static final int BLOCK_ADD_ID_TYPE_DVCC = 0x64766343;

/**
* BlockAddIdType value for Dolby Vision configuration with profile > 7. See also
* https://www.matroska.org/technical/codec_specs.html.
Expand Down Expand Up @@ -292,10 +290,8 @@ public class MatroskaExtractor implements Extractor {
49, 10, 48, 48, 58, 48, 48, 58, 48, 48, 44, 48, 48, 48, 32, 45, 45, 62, 32, 48, 48, 58, 48,
48, 58, 48, 48, 44, 48, 48, 48, 10
};

/** The byte offset of the end timecode in {@link #SUBRIP_PREFIX}. */
private static final int SUBRIP_PREFIX_END_TIMECODE_OFFSET = 19;

/**
* The value by which to divide a time in microseconds to convert it to the unit of the last value
* in a subrip timecode (milliseconds).
Expand All @@ -310,7 +306,6 @@ public class MatroskaExtractor implements Extractor {
Util.getUtf8Bytes(
"Format: Start, End, "
+ "ReadOrder, Layer, Style, Name, MarginL, MarginR, MarginV, Effect, Text");

/**
* A template for the prefix that must be added to each SSA sample.
*
Expand All @@ -327,16 +322,13 @@ public class MatroskaExtractor implements Extractor {
68, 105, 97, 108, 111, 103, 117, 101, 58, 32, 48, 58, 48, 48, 58, 48, 48, 58, 48, 48, 44,
48, 58, 48, 48, 58, 48, 48, 58, 48, 48, 44
};

/** The byte offset of the end timecode in {@link #SSA_PREFIX}. */
private static final int SSA_PREFIX_END_TIMECODE_OFFSET = 21;

/**
* The value by which to divide a time in microseconds to convert it to the unit of the last value
* in an SSA timecode (1/100ths of a second).
*/
private static final long SSA_TIMECODE_LAST_VALUE_SCALING_FACTOR = 10_000;

/** The format of an SSA timecode. */
private static final String SSA_TIMECODE_FORMAT = "%01d:%02d:%02d:%02d";

Expand All @@ -359,13 +351,11 @@ public class MatroskaExtractor implements Extractor {

/** The byte offset of the end timecode in {@link #VTT_PREFIX}. */
private static final int VTT_PREFIX_END_TIMECODE_OFFSET = 25;

/**
* The value by which to divide a time in microseconds to convert it to the unit of the last value
* in a VTT timecode (milliseconds).
*/
private static final long VTT_TIMECODE_LAST_VALUE_SCALING_FACTOR = 1000;

/** The format of a VTT timecode. */
private static final String VTT_TIMECODE_FORMAT = "%02d:%02d:%02d.%03d";

Expand All @@ -374,10 +364,8 @@ public class MatroskaExtractor implements Extractor {

/** Format tag indicating a WAVEFORMATEXTENSIBLE structure. */
private static final int WAVE_FORMAT_EXTENSIBLE = 0xFFFE;

/** Format tag for PCM. */
private static final int WAVE_FORMAT_PCM = 1;

/** Sub format for PCM. */
private static final UUID WAVE_SUBFORMAT_PCM = new UUID(0x0100000000001000L, 0x800000AA00389B71L);

Expand Down Expand Up @@ -1996,7 +1984,6 @@ protected static final class Track {

private static final int DISPLAY_UNIT_PIXELS = 0;
private static final int MAX_CHROMATICITY = 50_000; // Defined in CTA-861.3.

/** Default max content light level (CLL) that should be encoded into hdrStaticInfo. */
private static final int DEFAULT_MAX_CLL = 1000; // nits.

Expand Down Expand Up @@ -2308,14 +2295,14 @@ public void initializeOutput(ExtractorOutput output, int trackId) throws ParserE
@Nullable ColorInfo colorInfo = null;
if (hasColorInfo) {
@Nullable byte[] hdrStaticInfo = getHdrStaticInfo();
colorInfo =
new ColorInfo(
colorSpace,
colorRange,
colorTransfer,
hdrStaticInfo,
bitsPerChannel,
bitsPerChannel);
colorInfo = new ColorInfo.Builder()
.setColorSpace(colorSpace)
.setColorRange(colorRange)
.setColorTransfer(colorTransfer)
.setHdrStaticInfo(hdrStaticInfo)
.setLumaBitdepth(bitsPerChannel)
.setChromaBitdepth(bitsPerChannel)
.build();
}
int rotationDegrees = Format.NO_VALUE;

Expand Down
Loading