Skip to content

Commit

Permalink
RTC: Support dropping h.264 SEI from NALUs. v5.0.213 (#4057)
Browse files Browse the repository at this point in the history
try to fix #4052.

---------

Co-authored-by: winlin <[email protected]>
  • Loading branch information
suzp1984 and winlinvip committed Jun 3, 2024
1 parent 9995147 commit 3139137
Show file tree
Hide file tree
Showing 11 changed files with 139 additions and 66 deletions.
7 changes: 7 additions & 0 deletions trunk/conf/full.conf
Original file line number Diff line number Diff line change
Expand Up @@ -556,6 +556,13 @@ vhost rtc.vhost.srs.com {
# Overwrite by env SRS_VHOST_RTC_KEEP_BFRAME for all vhosts.
# default: off
keep_bframe off;
# Whether to keep the h.264 SEI type NALU packet.
# DJI drone M30T will send many SEI type NALU packet, while iOS hardware decoder (Video Toolbox)
# dislike to feed it so many SEI NALU between NonIDR and IDR NALU packets.
# @see https://github.com/ossrs/srs/issues/4052
# Overwrite by env SRS_VHOST_RTC_KEEP_AVC_NALU_SEI for all vhosts.
# Default: on
keep_avc_nalu_sei on;
# The transcode audio bitrate, for RTMP to RTC.
# Overwrite by env SRS_VHOST_RTC_OPUS_BITRATE for all vhosts.
# [8000, 320000]
Expand Down
1 change: 1 addition & 0 deletions trunk/doc/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ The changelog for SRS.
<a name="v5-changes"></a>

## SRS 5.0 Changelog
* v5.0, 2024-06-03, Merge [#4057](https://github.com/ossrs/srs/pull/4057): RTC: Support dropping h.264 SEI from NALUs. v5.0.213 (#4057)
* v5.0, 2024-04-23, Merge [#4038](https://github.com/ossrs/srs/pull/4038): RTMP: Do not response publish start message if hooks fail. v5.0.212 (#4038)
* v5.0, 2024-04-22, Merge [#4033](https://github.com/ossrs/srs/pull/4033): issue #3967: support x509 certification chiain in single pem file. v5.0.211 (#4033)
* v5.0, 2024-03-26, Filter JSONP callback function name. v5.0.210
Expand Down
22 changes: 21 additions & 1 deletion trunk/src/app/srs_app_config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2726,7 +2726,7 @@ srs_error_t SrsConfig::check_normal_config()
&& m != "bframe" && m != "aac" && m != "stun_timeout" && m != "stun_strict_check"
&& m != "dtls_role" && m != "dtls_version" && m != "drop_for_pt" && m != "rtc_to_rtmp"
&& m != "pli_for_rtmp" && m != "rtmp_to_rtc" && m != "keep_bframe" && m != "opus_bitrate"
&& m != "aac_bitrate") {
&& m != "aac_bitrate" && m != "keep_avc_nalu_sei") {
return srs_error_new(ERROR_SYSTEM_CONFIG_INVALID, "illegal vhost.rtc.%s of %s", m.c_str(), vhost->arg0().c_str());
}
}
Expand Down Expand Up @@ -4474,6 +4474,26 @@ bool SrsConfig::get_rtc_keep_bframe(string vhost)
return SRS_CONF_PERFER_FALSE(conf->arg0());
}

bool SrsConfig::get_rtc_keep_avc_nalu_sei(std::string vhost)
{
SRS_OVERWRITE_BY_ENV_BOOL2("srs.vhost.rtc.keep_avc_nalu_sei"); // SRS_VHOST_RTC_KEEP_AVC_NALU_SEI

static bool DEFAULT = true;

SrsConfDirective* conf = get_rtc(vhost);

if (!conf) {
return DEFAULT;
}

conf = conf->get("keep_avc_nalu_sei");
if (!conf || conf->arg0().empty()) {
return DEFAULT;
}

return SRS_CONF_PERFER_TRUE(conf->arg0());
}

bool SrsConfig::get_rtc_from_rtmp(string vhost)
{
SRS_OVERWRITE_BY_ENV_BOOL("srs.vhost.rtc.rtmp_to_rtc"); // SRS_VHOST_RTC_RTMP_TO_RTC
Expand Down
1 change: 1 addition & 0 deletions trunk/src/app/srs_app_config.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -533,6 +533,7 @@ class SrsConfig
SrsConfDirective* get_rtc(std::string vhost);
bool get_rtc_enabled(std::string vhost);
bool get_rtc_keep_bframe(std::string vhost);
bool get_rtc_keep_avc_nalu_sei(std::string vhost);
bool get_rtc_from_rtmp(std::string vhost);
srs_utime_t get_rtc_stun_timeout(std::string vhost);
bool get_rtc_stun_strict_check(std::string vhost);
Expand Down
37 changes: 20 additions & 17 deletions trunk/src/app/srs_app_rtc_source.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -724,6 +724,7 @@ SrsRtcFromRtmpBridge::SrsRtcFromRtmpBridge(SrsRtcSource* source)
latest_codec_ = SrsAudioCodecIdForbidden;
rtmp_to_rtc = false;
keep_bframe = false;
keep_avc_nalu_sei = true;
merge_nalus = false;
meta = new SrsMetaCache();
audio_sequence = 0;
Expand Down Expand Up @@ -774,9 +775,10 @@ srs_error_t SrsRtcFromRtmpBridge::initialize(SrsRequest* r)
}

keep_bframe = _srs_config->get_rtc_keep_bframe(req->vhost);
keep_avc_nalu_sei = _srs_config->get_rtc_keep_avc_nalu_sei(req->vhost);
merge_nalus = _srs_config->get_rtc_server_merge_nalus();
srs_trace("RTC bridge from RTMP, rtmp2rtc=%d, keep_bframe=%d, merge_nalus=%d",
rtmp_to_rtc, keep_bframe, merge_nalus);
srs_trace("RTC bridge from RTMP, rtmp2rtc=%d, keep_bframe=%d, keep_avc_nalu_sei=%d, merge_nalus=%d",
rtmp_to_rtc, keep_bframe, keep_avc_nalu_sei, merge_nalus);

return err;
}
Expand Down Expand Up @@ -1025,12 +1027,6 @@ srs_error_t SrsRtcFromRtmpBridge::on_video(SrsSharedPtrMessage* msg)
for (int i = 0; i < nn_samples; i++) {
SrsSample* sample = samples[i];

// We always ignore bframe here, if config to discard bframe,
// the bframe flag will not be set.
if (sample->bframe) {
continue;
}

if (sample->size <= kRtpMaxPayloadSize) {
if ((err = package_single_nalu(msg, sample, pkts)) != srs_success) {
return srs_error_wrap(err, "package single nalu");
Expand Down Expand Up @@ -1062,14 +1058,27 @@ srs_error_t SrsRtcFromRtmpBridge::filter(SrsSharedPtrMessage* msg, SrsFormat* fo
// Update samples to shared frame.
for (int i = 0; i < format->video->nb_samples; ++i) {
SrsSample* sample = &format->video->samples[i];

if (!keep_avc_nalu_sei && format->vcodec->id == SrsVideoCodecIdAVC) {
SrsAvcNaluType avc_nalu_type;
// TODO: FIXME use static method to parse avc nalu type.
if ((err = SrsVideoFrame::parse_avc_nalu_type(sample, avc_nalu_type)) != srs_success) {
return srs_error_wrap(err, "parse avc nalu_type");
}
if (avc_nalu_type == SrsAvcNaluTypeSEI) {
// srs_warn("skip avc nalu type SEI, size=%d", sample->size);
continue;
}
}

// Because RTC does not support B-frame, so we will drop them.
// TODO: Drop B-frame in better way, which not cause picture corruption.
if (!keep_bframe) {
if ((err = sample->parse_bframe()) != srs_success) {
if (!keep_bframe && format->vcodec->id == SrsVideoCodecIdAVC) {
bool is_b_frame;
if ((err = SrsVideoFrame::parse_avc_b_frame(sample, is_b_frame)) != srs_success) {
return srs_error_wrap(err, "parse bframe");
}
if (sample->bframe) {
if (is_b_frame) {
continue;
}
}
Expand Down Expand Up @@ -1149,12 +1158,6 @@ srs_error_t SrsRtcFromRtmpBridge::package_nalus(SrsSharedPtrMessage* msg, const
for (int i = 0; i < (int)samples.size(); i++) {
SrsSample* sample = samples[i];

// We always ignore bframe here, if config to discard bframe,
// the bframe flag will not be set.
if (sample->bframe) {
continue;
}

if (!sample->size) {
continue;
}
Expand Down
1 change: 1 addition & 0 deletions trunk/src/app/srs_app_rtc_source.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,7 @@ class SrsRtcFromRtmpBridge : public ISrsLiveSourceBridge
SrsAudioCodecId latest_codec_;
SrsAudioTranscoder* codec_;
bool keep_bframe;
bool keep_avc_nalu_sei;
bool merge_nalus;
uint16_t audio_sequence;
uint16_t video_sequence;
Expand Down
2 changes: 1 addition & 1 deletion trunk/src/core/srs_core_version5.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@

#define VERSION_MAJOR 5
#define VERSION_MINOR 0
#define VERSION_REVISION 212
#define VERSION_REVISION 213

#endif
99 changes: 58 additions & 41 deletions trunk/src/kernel/srs_kernel_codec.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -403,63 +403,23 @@ SrsSample::SrsSample()
{
size = 0;
bytes = NULL;
bframe = false;
}

SrsSample::SrsSample(char* b, int s)
{
size = s;
bytes = b;
bframe = false;
}

SrsSample::~SrsSample()
{
}

srs_error_t SrsSample::parse_bframe()
{
srs_error_t err = srs_success;

uint8_t header = bytes[0];
SrsAvcNaluType nal_type = (SrsAvcNaluType)(header & kNalTypeMask);

if (nal_type != SrsAvcNaluTypeNonIDR && nal_type != SrsAvcNaluTypeDataPartitionA && nal_type != SrsAvcNaluTypeIDR) {
return err;
}

SrsBuffer* stream = new SrsBuffer(bytes, size);
SrsAutoFree(SrsBuffer, stream);

// Skip nalu header.
stream->skip(1);

SrsBitBuffer bitstream(stream);
int32_t first_mb_in_slice = 0;
if ((err = srs_avc_nalu_read_uev(&bitstream, first_mb_in_slice)) != srs_success) {
return srs_error_wrap(err, "nalu read uev");
}

int32_t slice_type_v = 0;
if ((err = srs_avc_nalu_read_uev(&bitstream, slice_type_v)) != srs_success) {
return srs_error_wrap(err, "nalu read uev");
}
SrsAvcSliceType slice_type = (SrsAvcSliceType)slice_type_v;

if (slice_type == SrsAvcSliceTypeB || slice_type == SrsAvcSliceTypeB1) {
bframe = true;
srs_verbose("nal_type=%d, slice type=%d", nal_type, slice_type);
}

return err;
}

SrsSample* SrsSample::copy()
{
SrsSample* p = new SrsSample();
p->bytes = bytes;
p->size = size;
p->bframe = bframe;
return p;
}

Expand Down Expand Up @@ -554,7 +514,6 @@ srs_error_t SrsFrame::add_sample(char* bytes, int size)
SrsSample* sample = &samples[nb_samples++];
sample->bytes = bytes;
sample->size = size;
sample->bframe = false;

return err;
}
Expand Down Expand Up @@ -625,6 +584,64 @@ SrsVideoCodecConfig* SrsVideoFrame::vcodec()
return (SrsVideoCodecConfig*)codec;
}

srs_error_t SrsVideoFrame::parse_avc_nalu_type(const SrsSample* sample, SrsAvcNaluType& avc_nalu_type)
{
srs_error_t err = srs_success;

if (sample == NULL || sample->size < 1) {
return srs_error_new(ERROR_AVC_NALU_EMPTY, "empty nalu");
}

uint8_t header = sample->bytes[0];
avc_nalu_type = (SrsAvcNaluType)(header & kNalTypeMask);

return err;
}

srs_error_t SrsVideoFrame::parse_avc_b_frame(const SrsSample* sample, bool& is_b_frame)
{
srs_error_t err = srs_success;

if (sample == NULL || sample->size < 1) {
return srs_error_new(ERROR_AVC_NALU_EMPTY, "empty nalu");
}

SrsAvcNaluType nalu_type;
if ((err = parse_avc_nalu_type(sample, nalu_type)) != srs_success) {
return srs_error_wrap(err, "parse avc nalu type error");
}

if (nalu_type != SrsAvcNaluTypeNonIDR && nalu_type != SrsAvcNaluTypeDataPartitionA && nalu_type != SrsAvcNaluTypeIDR) {
is_b_frame = false;
return err;
}

SrsBuffer* stream = new SrsBuffer(sample->bytes, sample->size);
SrsAutoFree(SrsBuffer, stream);

// Skip nalu header.
stream->skip(1);

SrsBitBuffer bitstream(stream);
int32_t first_mb_in_slice = 0;
if ((err = srs_avc_nalu_read_uev(&bitstream, first_mb_in_slice)) != srs_success) {
return srs_error_wrap(err, "nalu read uev");
}

int32_t slice_type_v = 0;
if ((err = srs_avc_nalu_read_uev(&bitstream, slice_type_v)) != srs_success) {
return srs_error_wrap(err, "nalu read uev");
}
SrsAvcSliceType slice_type = (SrsAvcSliceType)slice_type_v;

is_b_frame = slice_type == SrsAvcSliceTypeB || slice_type == SrsAvcSliceTypeB1;
if (is_b_frame) {
srs_verbose("nalu_type=%d, slice type=%d", nalu_type, slice_type);
}

return err;
}

SrsFormat::SrsFormat()
{
acodec = NULL;
Expand Down
9 changes: 4 additions & 5 deletions trunk/src/kernel/srs_kernel_codec.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -522,7 +522,7 @@ std::string srs_avc_level2str(SrsAvcLevel level);

/**
* A sample is the unit of frame.
* It's a NALU for H.264.
* It's a NALU for H.264, H.265.
* It's the whole AAC raw data for AAC.
* @remark Neither SPS/PPS or ASC is sample unit, it's codec sequence header.
*/
Expand All @@ -533,15 +533,11 @@ class SrsSample
int size;
// The ptr of unit, user must free it.
char* bytes;
// Whether is B frame.
bool bframe;
public:
SrsSample();
SrsSample(char* b, int s);
~SrsSample();
public:
// If we need to know whether sample is bframe, we have to parse the NALU payload.
srs_error_t parse_bframe();
// Copy sample, share the bytes pointer.
SrsSample* copy();
};
Expand Down Expand Up @@ -714,6 +710,9 @@ class SrsVideoFrame : public SrsFrame
virtual srs_error_t add_sample(char* bytes, int size);
public:
virtual SrsVideoCodecConfig* vcodec();
public:
static srs_error_t parse_avc_nalu_type(const SrsSample* sample, SrsAvcNaluType& avc_nalu_type);
static srs_error_t parse_avc_b_frame(const SrsSample* sample, bool& is_b_frame);
};

/**
Expand Down
3 changes: 2 additions & 1 deletion trunk/src/kernel/srs_kernel_error.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,8 @@
XX(ERROR_INOTIFY_OPENFD , 3094, "InotifyOpenFd", "Failed to open inotify fd for config listener") \
XX(ERROR_INOTIFY_WATCH , 3095, "InotfyWatch", "Failed to watch inotify for config listener") \
XX(ERROR_HTTP_URL_UNESCAPE , 3096, "HttpUrlUnescape", "Failed to unescape URL for HTTP") \
XX(ERROR_HTTP_WITH_BODY , 3097, "HttpWithBody", "Failed for HTTP body")
XX(ERROR_HTTP_WITH_BODY , 3097, "HttpWithBody", "Failed for HTTP body") \
XX(ERROR_AVC_NALU_EMPTY , 3102, "AvcNaluEmpty", "AVC NALU is empty")

/**************************************************/
/* HTTP/StreamConverter protocol error. */
Expand Down
23 changes: 23 additions & 0 deletions trunk/src/utest/srs_utest_config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4402,6 +4402,29 @@ VOID TEST(ConfigEnvTest, CheckEnvValuesVhostRtc)

SrsSetEnvConfig(rtc_keep_bframe, "SRS_VHOST_RTC_KEEP_BFRAME", "on");
EXPECT_TRUE(conf.get_rtc_keep_bframe("__defaultVhost__"));

{
// make sure the default value is false, if defined incorrect env value.
SrsSetEnvConfig(rtc_keep_bframe, "SRS_VHOST_RTC_KEEP_BFRAME", "onn");
EXPECT_FALSE(conf.get_rtc_keep_bframe("__defaultVhost__"));

}

{
SrsSetEnvConfig(rtc_keep_avc_nalu_sei, "SRS_VHOST_RTC_KEEP_AVC_NALU_SEI", "off");
EXPECT_FALSE(conf.get_rtc_keep_avc_nalu_sei("__defaultVhost__"));
}

{
SrsSetEnvConfig(rtc_keep_avc_nalu_sei, "SRS_VHOST_RTC_KEEP_AVC_NALU_SEI", "on");
EXPECT_TRUE(conf.get_rtc_keep_avc_nalu_sei("__defaultVhost__"));
}

{
// make sure the default value is true, if defined incorrect env value.
SrsSetEnvConfig(rtc_keep_avc_nalu_sei, "SRS_VHOST_RTC_KEEP_AVC_NALU_SEI", "xx");
EXPECT_TRUE(conf.get_rtc_keep_avc_nalu_sei("__defaultVhost__"));
}
}

if (true) {
Expand Down

0 comments on commit 3139137

Please sign in to comment.