diff --git a/include/gpac/ietf.h b/include/gpac/ietf.h index 4920e68dc1..faf4a51553 100644 --- a/include/gpac/ietf.h +++ b/include/gpac/ietf.h @@ -1413,6 +1413,8 @@ enum #endif /*use VVC transport (no RFC yet)*/ GF_RTP_PAYT_VVC, + /*use opus audio format*/ + GF_RTP_PAYT_OPUS, }; diff --git a/include/gpac/internal/ietf_dev.h b/include/gpac/internal/ietf_dev.h index d8642a689e..069d57eac4 100644 --- a/include/gpac/internal/ietf_dev.h +++ b/include/gpac/internal/ietf_dev.h @@ -447,6 +447,7 @@ GF_Err gp_rtp_builder_do_ac3(GP_RTPPacketizer *builder, u8 *data, u32 data_size, GF_Err gp_rtp_builder_do_hevc(GP_RTPPacketizer *builder, u8 *data, u32 data_size, u8 IsAUEnd, u32 FullAUSize); GF_Err gp_rtp_builder_do_mp2t(GP_RTPPacketizer *builder, u8 *data, u32 data_size, u8 IsAUEnd, u32 FullAUSize); GF_Err gp_rtp_builder_do_vvc(GP_RTPPacketizer *builder, u8 *data, u32 data_size, u8 IsAUEnd, u32 FullAUSize); +GF_Err gp_rtp_builder_do_opus(GP_RTPPacketizer *builder, u8 *data, u32 data_size, u8 IsAUEnd, u32 FullAUSize); #define RTP_VVC_AGG_NAL 0x1C //28 #define RTP_VVC_FRAG_NAL 0x1D //29 @@ -477,6 +478,8 @@ struct __tag_rtp_depacketizer GP_RTPSLMap sl_map; /*! RTP clock rate*/ u32 clock_rate; + /*! audio channels from RTP map*/ + u32 audio_channels; //! clip rect X u32 x; diff --git a/include/gpac/rtp_streamer.h b/include/gpac/rtp_streamer.h index fc88dc9cd3..c45a71180a 100644 --- a/include/gpac/rtp_streamer.h +++ b/include/gpac/rtp_streamer.h @@ -134,11 +134,12 @@ Gets the SDP asscoiated with all media in the streaming session (only media part \param tx text window horizontal offset \param ty text window vertical offset \param tl text window z-index +\param nb_chan number of audio channels, 0 if unknown \param for_rtsp if GF_TRUE, produces the SDP for an RTSP describe (no port info) \param out_sdp_buffer location to the SDP buffer to allocate and fill \return error if any */ -GF_Err gf_rtp_streamer_append_sdp_extended(GF_RTPStreamer *rtp, u16 ESID, const u8 *dsi, u32 dsi_len, const u8 *dsi_enh, u32 dsi_enh_len, char *KMS_URI, u32 width, u32 height, u32 tw, u32 th, s32 tx, s32 ty, s16 tl, Bool for_rtsp, char **out_sdp_buffer); +GF_Err gf_rtp_streamer_append_sdp_extended(GF_RTPStreamer *rtp, u16 ESID, const u8 *dsi, u32 dsi_len, const u8 *dsi_enh, u32 dsi_enh_len, char *KMS_URI, u32 width, u32 height, u32 tw, u32 th, s32 tx, s32 ty, s16 tl, u32 nb_chan, Bool for_rtsp, char **out_sdp_buffer); /*! sends a full Access Unit over RTP \param rtp the target RTP streamer diff --git a/src/filters/in_rtp.c b/src/filters/in_rtp.c index b13465250c..217b481841 100644 --- a/src/filters/in_rtp.c +++ b/src/filters/in_rtp.c @@ -539,8 +539,6 @@ static GF_Err rtpin_process(GF_Filter *filter) break; } - ctx->eos_probe_start = 0; - i=0; while ((stream = (GF_RTPInStream *)gf_list_enum(ctx->streams, &i))) { if (stream->status==RTP_Running) { @@ -548,15 +546,15 @@ static GF_Err rtpin_process(GF_Filter *filter) read += rtpin_stream_read(stream); } - if (stream->flags & RTP_EOS) { + if ((stream->flags & RTP_EOS) && !ctx->eos_probe_start) ctx->eos_probe_start = gf_sys_clock(); - } } if (!read) { break; } tot_read+=read; + ctx->eos_probe_start = 0; } //we wait max 300ms to detect eos diff --git a/src/filters/in_rtp_stream.c b/src/filters/in_rtp_stream.c index 05bc1e3313..095926cb21 100644 --- a/src/filters/in_rtp_stream.c +++ b/src/filters/in_rtp_stream.c @@ -715,7 +715,6 @@ u32 rtpin_stream_read(GF_RTPInStream *stream) size = gf_rtp_read_rtp(stream->rtp_ch, stream->buffer, stream->rtpin->block_size); if (size) { tot_size += size; - stream->rtpin->udp_timeout = 0; rtpin_stream_on_rtp_pck(stream, stream->buffer, size); } } @@ -728,16 +727,25 @@ u32 rtpin_stream_read(GF_RTPInStream *stream) if (stream->rtpin->udp_timeout) { if (!stream->last_udp_time) { stream->last_udp_time = gf_sys_clock(); - } else if (stream->rtp_ch->net_info.IsUnicast) { + } else if (stream->rtp_bytes || stream->rtp_ch->net_info.IsUnicast) { u32 diff = gf_sys_clock() - stream->last_udp_time; if (diff >= stream->rtpin->udp_timeout) { - GF_LOG(GF_LOG_WARNING, GF_LOG_RTP, ("[RTP] UDP Timeout after %d ms on stream %d%s\n", diff, stream->ES_ID, - (stream->rtpin->session && stream->rtpin->autortsp) ? ", retrying using TCP" : "")); - - if (stream->rtpin->autortsp && stream->rtpin->session && !stream->rtpin->retry_tcp) { - stream->rtpin->retry_tcp = GF_TRUE; + if (stream->rtp_bytes) { + if (! (stream->flags & RTP_EOS)) { + GF_LOG(GF_LOG_WARNING, GF_LOG_RTP, ("[RTP] No packet received for %d ms on stream %d, assuming EOS\n", diff, stream->ES_ID)); + stream->flags |= RTP_EOS; + } } else { - gf_filter_pid_set_eos(stream->opid); + if (! (stream->flags & RTP_EOS) && !stream->rtpin->retry_tcp) { + GF_LOG(GF_LOG_WARNING, GF_LOG_RTP, ("[RTP] UDP Timeout after %d ms on stream %d%s\n", diff, stream->ES_ID, + (stream->rtpin->session && stream->rtpin->autortsp) ? ", retrying using TCP" : "")); + } + + if (stream->rtpin->autortsp && stream->rtpin->session && !stream->rtpin->retry_tcp) { + stream->rtpin->retry_tcp = GF_TRUE; + } else { + stream->flags |= RTP_EOS; + } } } } diff --git a/src/filters/out_rtp.c b/src/filters/out_rtp.c index da5a8d48a0..fca894d620 100644 --- a/src/filters/out_rtp.c +++ b/src/filters/out_rtp.c @@ -182,6 +182,7 @@ GF_Err rtpout_create_sdp(GF_List *streams, Bool is_rtsp, const char *ip, const c s16 tl; u32 dsi_len = 0; u32 dsi_enh_len = 0; + u32 nb_chan = 0; const GF_PropertyValue *p; GF_RTPOutStream *stream = gf_list_get(streams, i); if (!stream->rtp) continue; @@ -208,6 +209,9 @@ GF_Err rtpout_create_sdp(GF_List *streams, Bool is_rtsp, const char *ip, const c p = gf_filter_pid_get_property(stream->pid, GF_PROP_PID_HEIGHT); if (p) h = p->value.uint; + p = gf_filter_pid_get_property(stream->pid, GF_PROP_PID_NUM_CHANNELS); + if (p) nb_chan = p->value.uint; + p = gf_filter_pid_get_property(stream->pid, GF_PROP_PID_PROTECTION_KMS_URI); if (p) KMS = p->value.string; @@ -228,7 +232,7 @@ GF_Err rtpout_create_sdp(GF_List *streams, Bool is_rtsp, const char *ip, const c if (p) h = p->value.uint; } - gf_rtp_streamer_append_sdp_extended(stream->rtp, stream->id, dsi, dsi_len, dsi_enh, dsi_enh_len, (char *)KMS, w, h, tw, th, tx, ty, tl, is_rtsp, &sdp_media); + gf_rtp_streamer_append_sdp_extended(stream->rtp, stream->id, dsi, dsi_len, dsi_enh, dsi_enh_len, (char *)KMS, w, h, tw, th, tx, ty, tl, nb_chan, is_rtsp, &sdp_media); if (sdp_media) { gf_fprintf(sdp_out, "%s", sdp_media); diff --git a/src/ietf/rtp_depacketizer.c b/src/ietf/rtp_depacketizer.c index 4d2337bb27..da604b40e9 100644 --- a/src/ietf/rtp_depacketizer.c +++ b/src/ietf/rtp_depacketizer.c @@ -1186,6 +1186,21 @@ static void gf_rtp_parse_eac3(GF_RTPDepacketizer *rtp, GF_RTPHeader *hdr, u8 *pa { gf_rtp_parse_ac3_eac3(rtp, hdr, payload, size, GF_TRUE); } + + +static void gf_rtp_parse_opus(GF_RTPDepacketizer *rtp, GF_RTPHeader *hdr, u8 *payload, u32 size) +{ + rtp->sl_hdr.compositionTimeStampFlag = 1; + if (rtp->sl_hdr.compositionTimeStamp != hdr->TimeStamp) { + rtp->sl_hdr.accessUnitStartFlag = 1; + rtp->sl_hdr.compositionTimeStamp = hdr->TimeStamp; + } + rtp->sl_hdr.randomAccessPointFlag = 1; + rtp->on_sl_packet(rtp->udta, payload, size, &rtp->sl_hdr, GF_OK); + rtp->flags |= GF_RTP_NEW_AU; + rtp->sl_hdr.accessUnitStartFlag = 0; +} + #endif /*GPAC_DISABLE_AV_PARSERS*/ static u32 gf_rtp_get_payload_type(GF_RTPMap *map, GF_SDPMedia *media) @@ -1233,6 +1248,7 @@ static u32 gf_rtp_get_payload_type(GF_RTPMap *map, GF_SDPMedia *media) else if (!stricmp(map->payload_name, "H265")) return GF_RTP_PAYT_HEVC; else if (!stricmp(map->payload_name, "H265-SHVC")) return GF_RTP_PAYT_LHVC; else if (!stricmp(map->payload_name, "H266")) return GF_RTP_PAYT_VVC; + else if (!stricmp(map->payload_name, "opus")) return GF_RTP_PAYT_OPUS; else return 0; } @@ -1839,6 +1855,25 @@ static GF_Err gf_rtp_payt_setup(GF_RTPDepacketizer *rtp, GF_RTPMap *map, GF_SDPM rtp->sl_map.RandomAccessIndication = GF_TRUE; /*assign depacketizer*/ rtp->depacketize = gf_rtp_parse_eac3; + break; + case GF_RTP_PAYT_OPUS: + rtp->sl_map.StreamType = GF_STREAM_AUDIO; + rtp->sl_map.CodecID = GF_CODECID_OPUS; + rtp->sl_map.RandomAccessIndication = GF_TRUE; + /*assign depacketizer*/ + rtp->depacketize = gf_rtp_parse_opus; + + if (!rtp->sl_map.config) { + GF_OpusConfig opcfg; + opcfg.version = 1; + opcfg.OutputChannelCount = rtp->audio_channels; + opcfg.PreSkip = 0; + opcfg.InputSampleRate = rtp->clock_rate; + opcfg.OutputGain = 0; + opcfg.ChannelMappingFamily = 0; + gf_odf_opus_cfg_write(&opcfg, &rtp->sl_map.config, &rtp->sl_map.configSize); + } + break; #endif /*GPAC_DISABLE_AV_PARSERS*/ default: @@ -1898,7 +1933,7 @@ GF_RTPDepacketizer *gf_rtp_depacketizer_new(GF_SDPMedia *media, u32 hdr_payt, gf GF_RTPMap *map = NULL; u32 payt; GF_RTPDepacketizer *tmp; - u32 clock_rate; + u32 clock_rate, nb_chan=0; const GF_RTPStaticMap *static_map = NULL; /*check RTP map. For now we only support 1 RTPMap*/ @@ -1917,6 +1952,8 @@ GF_RTPDepacketizer *gf_rtp_depacketizer_new(GF_SDPMedia *media, u32 hdr_payt, gf return NULL; } clock_rate = static_map->clock_rate; + if (static_map->stream_type==GF_STREAM_AUDIO) + nb_chan = 1; payt = hdr_payt; } else { /*check payload type*/ @@ -1932,10 +1969,13 @@ GF_RTPDepacketizer *gf_rtp_depacketizer_new(GF_SDPMedia *media, u32 hdr_payt, gf return NULL; } clock_rate = static_map->clock_rate; + if (static_map->stream_type==GF_STREAM_AUDIO) + nb_chan = 1; } else { payt = gf_rtp_get_payload_type(map, media); if (!payt) return NULL; clock_rate = map->ClockRate; + nb_chan = map->AudioChannels; } } @@ -1943,6 +1983,8 @@ GF_RTPDepacketizer *gf_rtp_depacketizer_new(GF_SDPMedia *media, u32 hdr_payt, gf if (!tmp) return NULL; tmp->payt = payt; tmp->static_map = static_map; + tmp->clock_rate = clock_rate; + tmp->audio_channels = nb_chan; e = gf_rtp_payt_setup(tmp, map, media); if (e) { @@ -1951,7 +1993,6 @@ GF_RTPDepacketizer *gf_rtp_depacketizer_new(GF_SDPMedia *media, u32 hdr_payt, gf } assert(tmp->depacketize); - tmp->clock_rate = clock_rate; tmp->on_sl_packet = sl_packet_cbk; tmp->udta = udta; return tmp; diff --git a/src/ietf/rtp_packetizer.c b/src/ietf/rtp_packetizer.c index 639fcb9b37..a667b0e190 100644 --- a/src/ietf/rtp_packetizer.c +++ b/src/ietf/rtp_packetizer.c @@ -127,6 +127,8 @@ GF_Err gf_rtp_builder_process(GP_RTPPacketizer *builder, u8 *data, u32 data_size return gp_rtp_builder_do_vvc(builder, data, data_size, IsAUEnd, FullAUSize); case GF_RTP_PAYT_MP2T: return gp_rtp_builder_do_mp2t(builder, data, data_size, IsAUEnd, FullAUSize); + case GF_RTP_PAYT_OPUS: + return gp_rtp_builder_do_opus(builder, data, data_size, IsAUEnd, FullAUSize); default: return GF_NOT_SUPPORTED; } @@ -542,6 +544,10 @@ Bool gf_rtp_builder_get_payload_name(GP_RTPPacketizer *rtpb, char szPayloadName[ strcpy(szMediaName, "audio"); strcpy(szPayloadName, "eac3"); return GF_TRUE; + case GF_RTP_PAYT_OPUS: + strcpy(szMediaName, "audio"); + strcpy(szPayloadName, "opus"); + return GF_TRUE; case GF_RTP_PAYT_H264_SVC: strcpy(szMediaName, "video"); strcpy(szPayloadName, "H264-SVC"); diff --git a/src/ietf/rtp_pck_3gpp.c b/src/ietf/rtp_pck_3gpp.c index a63a831fb6..7482edc845 100644 --- a/src/ietf/rtp_pck_3gpp.c +++ b/src/ietf/rtp_pck_3gpp.c @@ -817,5 +817,32 @@ GF_Err gp_rtp_builder_do_ac3(GP_RTPPacketizer *builder, u8 *data, u32 data_size, return GF_OK; } +GF_Err gp_rtp_builder_do_opus(GP_RTPPacketizer *builder, u8 *data, u32 data_size, u8 IsAUEnd, u32 FullAUSize) +{ + /*flush*/ + if (!data) return GF_OK; + + /*fits*/ + if (data_size > builder->Path_MTU) { + GF_LOG(GF_LOG_ERROR, GF_LOG_RTP, ("[RTPOpus] Packet size %u larger MTU %u but Opus fragmentation is not yet supported, pacth welcome\n", data_size, builder->Path_MTU )); + return GF_NOT_SUPPORTED; + } + /*send new packet*/ + builder->rtp_header.TimeStamp = (u32) builder->sl_header.compositionTimeStamp; + builder->rtp_header.Marker = 0; + builder->rtp_header.SequenceNumber += 1; + builder->OnNewPacket(builder->cbk_obj, &builder->rtp_header); + + /*add payload*/ + if (builder->OnDataReference) + builder->OnDataReference(builder->cbk_obj, data_size, 0); + else + builder->OnData(builder->cbk_obj, data, data_size, GF_FALSE); + + builder->OnPacketDone(builder->cbk_obj, &builder->rtp_header); + builder->last_au_sn++; + return GF_OK; +} + #endif /*GPAC_DISABLE_STREAMING*/ diff --git a/src/ietf/rtp_streamer.c b/src/ietf/rtp_streamer.c index 06b9863902..6c02c5d64b 100644 --- a/src/ietf/rtp_streamer.c +++ b/src/ietf/rtp_streamer.c @@ -401,6 +401,11 @@ GF_RTPStreamer *gf_rtp_streamer_new(u32 streamType, u32 codecid, u32 timeScale, PayloadType = OfficialPayloadType = GF_RTP_PAYT_MP2T; required_rate = 90000; break; + case GF_CODECID_OPUS: + rtp_type = GF_RTP_PAYT_OPUS; + streamType = GF_STREAM_AUDIO; + has_mpeg4_mapping = GF_FALSE; + break; default: if (!rtp_type) { @@ -527,7 +532,7 @@ void gf_media_format_ttxt_sdp(GP_RTPPacketizer *builder, char *payload_name, cha GF_EXPORT -GF_Err gf_rtp_streamer_append_sdp_extended(GF_RTPStreamer *rtp, u16 ESID, const u8 *dsi, u32 dsi_len, const u8 *dsi_enh, u32 dsi_enh_len, char *KMS_URI, u32 width, u32 height, u32 tw, u32 th, s32 tx, s32 ty, s16 tl, Bool for_rtsp, char **out_sdp_buffer) +GF_Err gf_rtp_streamer_append_sdp_extended(GF_RTPStreamer *rtp, u16 ESID, const u8 *dsi, u32 dsi_len, const u8 *dsi_enh, u32 dsi_enh_len, char *KMS_URI, u32 width, u32 height, u32 tw, u32 th, s32 tx, s32 ty, s16 tl, u32 nb_channels, Bool for_rtsp, char **out_sdp_buffer) { u32 size; u16 port=0; @@ -540,14 +545,18 @@ GF_Err gf_rtp_streamer_append_sdp_extended(GF_RTPStreamer *rtp, u16 ESID, const if (!for_rtsp) gf_rtp_get_ports(rtp->channel, &port, NULL); - sprintf(sdp, "m=%s %d RTP/%s %d\n", mediaName, for_rtsp ? 0 : port, rtp->packetizer->slMap.IV_length ? "SAVP" : "AVP", rtp->packetizer->PayloadType); - sprintf(sdpLine, "a=rtpmap:%d %s/%d\n", rtp->packetizer->PayloadType, payloadName, rtp->packetizer->sl_config.timestampResolution); + sprintf(sdp, "m=%s %d RTP/%s %u\n", mediaName, for_rtsp ? 0 : port, rtp->packetizer->slMap.IV_length ? "SAVP" : "AVP", rtp->packetizer->PayloadType); + if (nb_channels) + sprintf(sdpLine, "a=rtpmap:%u %s/%u/%u\n", rtp->packetizer->PayloadType, payloadName, rtp->packetizer->sl_config.timestampResolution, nb_channels); + else + sprintf(sdpLine, "a=rtpmap:%u %s/%u\n", rtp->packetizer->PayloadType, payloadName, rtp->packetizer->sl_config.timestampResolution); strcat(sdp, sdpLine); if (ESID #if GPAC_ENABLE_3GPP_DIMS_RTP && (rtp->packetizer->rtp_payt != GF_RTP_PAYT_3GPP_DIMS) #endif + && (rtp->packetizer->rtp_payt != GF_RTP_PAYT_OPUS) ) { sprintf(sdpLine, "a=mpeg4-esid:%d\n", ESID); strcat(sdp, sdpLine); @@ -766,7 +775,7 @@ char *gf_rtp_streamer_format_sdp_header(char *app_name, char *ip_dest, char *ses GF_EXPORT GF_Err gf_rtp_streamer_append_sdp(GF_RTPStreamer *rtp, u16 ESID, const u8 *dsi, u32 dsi_len, char *KMS_URI, char **out_sdp_buffer) { - return gf_rtp_streamer_append_sdp_extended(rtp, ESID, dsi, dsi_len, NULL, 0, KMS_URI, 0, 0, 0, 0, 0, 0, 0, GF_FALSE, out_sdp_buffer); + return gf_rtp_streamer_append_sdp_extended(rtp, ESID, dsi, dsi_len, NULL, 0, KMS_URI, 0, 0, 0, 0, 0, 0, 0, 0, GF_FALSE, out_sdp_buffer); } GF_EXPORT diff --git a/src/media_tools/isom_hinter.c b/src/media_tools/isom_hinter.c index 40d067c028..72265b7b97 100644 --- a/src/media_tools/isom_hinter.c +++ b/src/media_tools/isom_hinter.c @@ -537,6 +537,11 @@ GF_RTPHinter *gf_hinter_track_new(GF_ISOFile *file, u32 TrackNum, streamType = GF_STREAM_AUDIO; gf_isom_get_audio_info(file, TrackNum, 1, NULL, &nb_ch, NULL); break; + case GF_ISOM_SUBTYPE_OPUS: + hintType = GF_RTP_PAYT_OPUS; + streamType = GF_STREAM_AUDIO; + gf_isom_get_audio_info(file, TrackNum, 1, NULL, &nb_ch, NULL); + break; case GF_ISOM_SUBTYPE_MP3: { GF_ISOSample *samp = gf_isom_get_sample(file, TrackNum, 1, NULL); diff --git a/src/odf/descriptors.c b/src/odf/descriptors.c index 089a0848f7..9682be48ad 100644 --- a/src/odf/descriptors.c +++ b/src/odf/descriptors.c @@ -1826,6 +1826,7 @@ GF_Err gf_odf_ac3_config_parse(u8 *dsi, u32 dsi_len, Bool is_ec3, GF_AC3Config * GF_Err gf_odf_opus_cfg_parse_bs(GF_BitStream *bs, GF_OpusConfig *cfg) { + memset(cfg, 0, sizeof(GF_OpusConfig)); cfg->version = gf_bs_read_u8(bs); cfg->OutputChannelCount = gf_bs_read_u8(bs); cfg->PreSkip = gf_bs_read_u16_le(bs);