Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for RTSP Opus #53

Merged
merged 5 commits into from
Jun 9, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fixed review comments in RtpOpusReader
  • Loading branch information
ManishaJajoo committed Mar 23, 2022
commit 3a87039ba19a72dc6671573036a1d11a873476fc
Original file line number Diff line number Diff line change
Expand Up @@ -37,19 +37,19 @@
public final class RtpPayloadFormat {

private static final String RTP_MEDIA_AC3 = "AC3";
private static final String RTP_MEDIA_OPUS = "OPUS";
private static final String RTP_MEDIA_MPEG4_GENERIC = "MPEG4-GENERIC";
private static final String RTP_MEDIA_H264 = "H264";
private static final String RTP_MEDIA_H265 = "H265";
private static final String RTP_MEDIA_MPEG4_GENERIC = "MPEG4-GENERIC";
private static final String RTP_MEDIA_OPUS = "OPUS";

/** Returns whether the format of a {@link MediaDescription} is supported. */
public static boolean isFormatSupported(MediaDescription mediaDescription) {
switch (Ascii.toUpperCase(mediaDescription.rtpMapAttribute.mediaEncoding)) {
case RTP_MEDIA_AC3:
case RTP_MEDIA_OPUS:
case RTP_MEDIA_H264:
case RTP_MEDIA_H265:
case RTP_MEDIA_MPEG4_GENERIC:
case RTP_MEDIA_OPUS:
return true;
default:
return false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@

private static final String GENERIC_CONTROL_ATTR = "*";

/** RFC7587 Section 6.1 Sampling rate for OPUS is fixed at 48KHz. */
private static final int OPUS_SAMPLING_RATE = 48000;

/** The track's associated {@link RtpPayloadFormat}. */
public final RtpPayloadFormat payloadFormat;
/** The track's URI. */
Expand Down Expand Up @@ -122,12 +125,11 @@ public int hashCode() {
processAacFmtpAttribute(formatBuilder, fmtpParameters, channelCount, clockRate);
break;
case MimeTypes.AUDIO_OPUS:
// RFC7587 Section 7
checkArgument(channelCount == 2, "Invalid channel count");
// RFC7587 Section 6.1
checkArgument(channelCount != C.INDEX_UNSET);
// RFC7587 Section 6.1.
// the RTP timestamp is incremented with a 48000 Hz clock rate
// for all modes of Opus and all sampling rates.
checkArgument(clockRate == 48000, "Invalid sampling rate");
checkArgument(clockRate == OPUS_SAMPLING_RATE, "Invalid sampling rate");
break;
case MimeTypes.VIDEO_H264:
checkArgument(!fmtpParameters.isEmpty());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,19 +39,18 @@
private static final String TAG = "RtpOpusReader";

private final RtpPayloadFormat payloadFormat;
ManishaJajoo marked this conversation as resolved.
Show resolved Hide resolved
private static final long MEDIA_CLOCK_FREQUENCY = 48_000;

private @MonotonicNonNull TrackOutput trackOutput;
private long firstReceivedTimestamp;
private long startTimeOffsetUs;

private final int sampleRate;
private int previousSequenceNumber;
private boolean foundOpusIDHeader;
private boolean foundOpusCommentHeader;

public RtpOpusReader(RtpPayloadFormat payloadFormat) {
this.payloadFormat = payloadFormat;
this.firstReceivedTimestamp = C.INDEX_UNSET;
this.sampleRate = this.payloadFormat.clockRate;
this.previousSequenceNumber = C.INDEX_UNSET;
this.foundOpusIDHeader = false;
this.foundOpusCommentHeader = false;
Expand All @@ -75,23 +74,20 @@ public void consume(
ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) {
checkStateNotNull(trackOutput);

/* RFC7845 Section 3
/* RFC7845 Section 3.
* +---------+ +----------------+ +--------------------+ +-----
* |ID Header| | Comment Header | |Audio Data Packet 1 | | ...
* +---------+ +----------------+ +--------------------+ +-----
*/
if (!foundOpusIDHeader) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry for not getting back to this sooner, I thought we have merged it.

All the logic here to find Opus ID and comment headers imply that the header and comment header is ONE PACKET long. But that is not what is specified in the RFC.

According to the RFC, the ID header is one PAGE long, and the comment header can span across many pages. And one page can span multiple RTP packets. Also, "Audio data packets might span page boundaries. The first audio data page could have the 'continued packet' flag set"

Although this is working right now, it's not semantically correct. So please try to make it compliant with the RFC by supporting multiple packet pages. There are examples of one access unit spans multiple RTP packets, like in the H264/265 reader. It's probably hard to test, but given you have some tests going on, please add some test based on your interpretation to the RFC.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry for not getting back to this sooner, I thought we have merged it.

All the logic here to find Opus ID and comment headers imply that the header and comment header is ONE PACKET long. But that is not what is specified in the RFC.

According to the RFC, the ID header is one PAGE long, and the comment header can span across many pages. And one page can span multiple RTP packets. Also, "Audio data packets might span page boundaries. The first audio data page could have the 'continued packet' flag set"

Although this is working right now, it's not semantically correct. So please try to make it compliant with the RFC by supporting multiple packet pages. There are examples of one access unit spans multiple RTP packets, like in the H264/265 reader. It's probably hard to test, but given you have some tests going on, please add some test based on your interpretation to the RFC.

int currPosition = data.getPosition();
checkArgument(isOpusIDHeader(data), "ID Header missing");

data.setPosition(currPosition);
checkForOpusIdHeader(data);
List<byte[]> initializationData = OpusUtil.buildInitializationData(data.getData());
Format.Builder formatBuilder = payloadFormat.format.buildUpon();
formatBuilder.setInitializationData(initializationData);
trackOutput.format(formatBuilder.build());
foundOpusIDHeader = true;
} else if (!foundOpusCommentHeader) {
claincly marked this conversation as resolved.
Show resolved Hide resolved
// Comment Header RFC7845 Section 5.2
// Comment Header RFC7845 Section 5.2.
String header = data.readString(8);
checkArgument(header.equals("OpusTags"), "Comment Header should follow ID Header");
foundOpusCommentHeader = true;
Expand All @@ -106,12 +102,13 @@ public void consume(
expectedSequenceNumber, sequenceNumber));
}

// sending opus data
// sending opus data.
int size = data.bytesLeft();
trackOutput.sampleData(data, size);
long timeUs =
toSampleTimeUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp, sampleRate);
trackOutput.sampleMetadata(timeUs, C.BUFFER_FLAG_KEY_FRAME, size, 0, null);
toSampleTimeUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp);
trackOutput.sampleMetadata(
timeUs, C.BUFFER_FLAG_KEY_FRAME, size, /* offset*/ 0, /* cryptoData*/ null);
}
previousSequenceNumber = sequenceNumber;
}
Expand All @@ -124,29 +121,23 @@ public void seek(long nextRtpTimestamp, long timeUs) {

// Internal methods.

private static boolean isOpusIDHeader(ParsableByteArray data) {
private static void checkForOpusIdHeader(ParsableByteArray data) {
int currPosition = data.getPosition();
int sampleSize = data.limit();
String header = data.readString(8);
// Identification header RFC7845 Section 5.1
if (sampleSize < 19 || !header.equals("OpusHead")) {
Log.e(
TAG,
Util.formatInvariant(
"first data octet of the RTP packet is not the beginning of a OpusHeader "
+ "Dropping current packet"));
return false;
}
// Identification header RFC7845 Section 5.1.
checkArgument(sampleSize > 18 && header.equals("OpusHead"), "ID Header missing");
checkArgument(data.readUnsignedByte() == 1, "version number must always be 1");
return true;
data.setPosition(currPosition);
}

/** Returns the correct sample time from RTP timestamp, accounting for the OPUS sampling rate. */
private static long toSampleTimeUs(
long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp, int sampleRate) {
long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) {
return startTimeOffsetUs
+ Util.scaleLargeTimestamp(
rtpTimestamp - firstReceivedRtpTimestamp,
/* multiplier= */ C.MICROS_PER_SECOND,
/* divisor= */ sampleRate);
/* divisor= */ MEDIA_CLOCK_FREQUENCY);
}
}