From ad7b28bb0aa7d617b28143e0a8ba0852e5a0fb24 Mon Sep 17 00:00:00 2001 From: Lidaofu <746101210@qq.com> Date: Sat, 15 Jun 2024 15:55:39 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E8=A1=A5=E5=85=85mk=5Ftrack=E3=80=81mk=5Fm?= =?UTF-8?q?edia=E9=83=A8=E5=88=86c=20api=20(#3628)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: lidaofu Co-authored-by: xiongziliang <771730766@qq.com> --- api/include/mk_events_objects.h | 11 +++++--- api/include/mk_track.h | 30 +++++++++++++++++++++ api/source/mk_events_objects.cpp | 26 ++++++++++++++++++ api/source/mk_track.cpp | 45 ++++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 3 deletions(-) diff --git a/api/include/mk_events_objects.h b/api/include/mk_events_objects.h index d28dbc62..5d1127a3 100644 --- a/api/include/mk_events_objects.h +++ b/api/include/mk_events_objects.h @@ -114,19 +114,24 @@ API_EXPORT int API_CALL mk_media_source_get_total_reader_count(const mk_media_so API_EXPORT int API_CALL mk_media_source_get_track_count(const mk_media_source ctx); // copy track reference by index from MediaSource, please use mk_track_unref to release it API_EXPORT mk_track API_CALL mk_media_source_get_track(const mk_media_source ctx, int index); +// MediaSource::Track:loss +API_EXPORT float API_CALL mk_media_source_get_track_loss(const mk_media_source ctx, const mk_track track); // MediaSource::broadcastMessage API_EXPORT int API_CALL mk_media_source_broadcast_msg(const mk_media_source ctx, const char *msg, size_t len); // MediaSource::getOriginUrl() API_EXPORT const char* API_CALL mk_media_source_get_origin_url(const mk_media_source ctx); // MediaSource::getOriginType() API_EXPORT int API_CALL mk_media_source_get_origin_type(const mk_media_source ctx); +// MediaSource::getOriginTypeStr(), 使用后请用mk_free释放返回值 +API_EXPORT const char *API_CALL mk_media_source_get_origin_type_str(const mk_media_source ctx); // MediaSource::getCreateStamp() API_EXPORT uint64_t API_CALL mk_media_source_get_create_stamp(const mk_media_source ctx); // MediaSource::isRecording() 0:hls,1:MP4 API_EXPORT int API_CALL mk_media_source_is_recording(const mk_media_source ctx, int type); - - - +// MediaSource::getBytesSpeed() +API_EXPORT int API_CALL mk_media_source_get_bytes_speed(const mk_media_source ctx); +// MediaSource::getAliveSecond() +API_EXPORT uint64_t API_CALL mk_media_source_get_alive_second(const mk_media_source ctx); /** * 直播源在ZLMediaKit中被称作为MediaSource， * 目前支持3种，分别是RtmpMediaSource、RtspMediaSource、HlsMediaSource diff --git a/api/include/mk_track.h b/api/include/mk_track.h index 430c3cfc..63d34b0f 100644 --- a/api/include/mk_track.h +++ b/api/include/mk_track.h @@ -73,6 +73,21 @@ API_EXPORT const char* API_CALL mk_track_codec_name(mk_track track); */ API_EXPORT int API_CALL mk_track_bit_rate(mk_track track); +/** + * 获取轨道是否已就绪，1: 已就绪，0：未就绪 + */ +API_EXPORT int API_CALL mk_track_ready(mk_track track); + +/** + * 获取累计帧数 + */ +API_EXPORT uint64_t API_CALL mk_track_frames(mk_track track); + +/** + * 获取时间，单位毫秒 + */ +API_EXPORT uint64_t API_CALL mk_track_duration(mk_track track); + /** * 监听frame输出事件 * @param track track对象 @@ -114,6 +129,21 @@ API_EXPORT int API_CALL mk_track_video_height(mk_track track); */ API_EXPORT int API_CALL mk_track_video_fps(mk_track track); +/** + * 获取视频累计关键帧数 + */ +API_EXPORT uint64_t API_CALL mk_track_video_key_frames(mk_track track); + +/** + * 获取视频GOP关键帧间隔 + */ +API_EXPORT int API_CALL mk_track_video_gop_size(mk_track track); + +/** + * 获取视频累计关键帧间隔(毫秒) + */ +API_EXPORT int API_CALL mk_track_video_gop_interval_ms(mk_track track); + /** * 获取音频采样率 */ diff --git a/api/source/mk_events_objects.cpp b/api/source/mk_events_objects.cpp index 3615bbcd..097e4201 100644 --- a/api/source/mk_events_objects.cpp +++ b/api/source/mk_events_objects.cpp @@ -218,6 +218,13 @@ API_EXPORT mk_track API_CALL mk_media_source_get_track(const mk_media_source ctx return (mk_track) new Track::Ptr(std::move(tracks[index])); } +API_EXPORT float API_CALL mk_media_source_get_track_loss(const mk_media_source ctx, const mk_track track) { + assert(ctx); + MediaSource *src = (MediaSource *)ctx; + // rtp推流只有一个统计器，但是可能有多个track，如果短时间多次获取间隔丢包率，第二次会获取为-1 + return src->getLossRate((*((Track::Ptr *)track))->getTrackType()); +} + API_EXPORT int API_CALL mk_media_source_broadcast_msg(const mk_media_source ctx, const char *msg, size_t len) { assert(ctx && msg && len); MediaSource *src = (MediaSource *)ctx; @@ -240,6 +247,12 @@ API_EXPORT int API_CALL mk_media_source_get_origin_type(const mk_media_source c return static_cast(src->getOriginType()); } +API_EXPORT const char* API_CALL mk_media_source_get_origin_type_str(const mk_media_source ctx) { + assert(ctx); + MediaSource *src = (MediaSource *)ctx; + return _strdup(getOriginTypeString(src->getOriginType()).c_str()); +} + API_EXPORT uint64_t API_CALL mk_media_source_get_create_stamp(const mk_media_source ctx) { assert(ctx); MediaSource *src = (MediaSource *)ctx; @@ -252,6 +265,19 @@ API_EXPORT int API_CALL mk_media_source_is_recording(const mk_media_source ctx,i return src->isRecording((Recorder::type)type); } +API_EXPORT int API_CALL mk_media_source_get_bytes_speed(const mk_media_source ctx) { + assert(ctx); + MediaSource *src = (MediaSource *)ctx; + return src->getBytesSpeed(); +} + +API_EXPORT uint64_t API_CALL mk_media_source_get_alive_second(const mk_media_source ctx) { + assert(ctx); + MediaSource *src = (MediaSource *)ctx; + return src->getAliveSecond(); +} + + API_EXPORT int API_CALL mk_media_source_close(const mk_media_source ctx,int force){ assert(ctx); MediaSource *src = (MediaSource *)ctx; diff --git a/api/source/mk_track.cpp b/api/source/mk_track.cpp index c409ef22..dbd94eb4 100644 --- a/api/source/mk_track.cpp +++ b/api/source/mk_track.cpp @@ -109,6 +109,21 @@ API_EXPORT int API_CALL mk_track_bit_rate(mk_track track) { return (*((Track::Ptr *) track))->getBitRate(); } +API_EXPORT int API_CALL mk_track_ready(mk_track track) { + assert(track); + return (*((Track::Ptr *)track))->ready(); +} + +API_EXPORT uint64_t API_CALL mk_track_frames(mk_track track) { + assert(track); + return (*((Track::Ptr *)track))->getFrames(); +} + +API_EXPORT uint64_t API_CALL mk_track_duration(mk_track track) { + assert(track); + return (*((Track::Ptr *)track))->getDuration(); +} + API_EXPORT void *API_CALL mk_track_add_delegate(mk_track track, on_mk_frame_out cb, void *user_data) { return mk_track_add_delegate2(track, cb, user_data, nullptr); } @@ -167,6 +182,36 @@ API_EXPORT int API_CALL mk_track_video_fps(mk_track track) { return 0; } +API_EXPORT uint64_t API_CALL mk_track_video_key_frames(mk_track track) { + assert(track); + auto video = dynamic_pointer_cast((*((Track::Ptr *)track))); + if (video) { + return video->getVideoFps(); + } + WarnL << "not video track"; + return 0; +} + +API_EXPORT int API_CALL mk_track_video_gop_size(mk_track track) { + assert(track); + auto video = dynamic_pointer_cast((*((Track::Ptr *)track))); + if (video) { + return video->getVideoGopSize(); + } + WarnL << "not video track"; + return 0; +} + +API_EXPORT int API_CALL mk_track_video_gop_interval_ms(mk_track track) { + assert(track); + auto video = dynamic_pointer_cast((*((Track::Ptr *)track))); + if (video) { + return video->getVideoGopInterval(); + } + WarnL << "not video track"; + return 0; +} + API_EXPORT int API_CALL mk_track_audio_sample_rate(mk_track track) { assert(track); auto audio = dynamic_pointer_cast((*((Track::Ptr *) track))); From 3de2ee7a2b0c3f484ba5255737c6bc860a5615e1 Mon Sep 17 00:00:00 2001 From: xiongziliang <771730766@qq.com> Date: Sun, 16 Jun 2024 09:34:39 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E4=BC=98=E5=8C=96aac=20rtp=E8=A7=A3?= =?UTF-8?q?=E7=A0=81=E5=99=A8=E5=86=85=E5=AD=98=E5=AE=89=E5=85=A8=E9=99=90?= =?UTF-8?q?=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ext-codec/AACRtp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext-codec/AACRtp.cpp b/ext-codec/AACRtp.cpp index 6ac5fa66..c9e4bfea 100644 --- a/ext-codec/AACRtp.cpp +++ b/ext-codec/AACRtp.cpp @@ -103,7 +103,7 @@ bool AACRtpDecoder::inputRtp(const RtpPacket::Ptr &rtp, bool key_pos) { ptr += len; au_header_ptr += 2; - if (_frame->size() == (size_t)size) { + if (_frame->size() >= (size_t)size) { // 设置当前audio unit时间戳 _frame->_dts = _last_dts + i * dts_inc; flushData(); From 84dbe4b07658e19a717b2e6fcbfaf5dd9571f9ff Mon Sep 17 00:00:00 2001 From: Xiaofeng Wang Date: Sun, 16 Jun 2024 09:37:33 +0800 Subject: [PATCH 3/3] Send config frames once before sending DirectProxy RTP packets (#3611) --- ext-codec/H264.cpp | 22 +++++++------------- ext-codec/H264.h | 14 +++++++++++++ ext-codec/H265.cpp | 32 ++++++++-------------------- ext-codec/H265.h | 2 ++ src/Extension/Track.h | 5 +++++ webrtc/WebRtcPlayer.cpp | 46 +++++++++++++++++++++++++++++++++++++++++ webrtc/WebRtcPlayer.h | 6 ++++++ 7 files changed, 89 insertions(+), 38 deletions(-) diff --git a/ext-codec/H264.cpp b/ext-codec/H264.cpp index 59471775..953c11fe 100644 --- a/ext-codec/H264.cpp +++ b/ext-codec/H264.cpp @@ -117,7 +117,7 @@ size_t prefixSize(const char *ptr, size_t len) { H264Track::H264Track(const string &sps, const string &pps, int sps_prefix_len, int pps_prefix_len) { _sps = sps.substr(sps_prefix_len); _pps = pps.substr(pps_prefix_len); - update(); + H264Track::update(); } CodecId H264Track::getCodecId() const { @@ -235,6 +235,10 @@ void H264Track::setExtraData(const uint8_t *data, size_t bytes) { } bool H264Track::update() { + _config_frames = std::vector{ + createConfigFrame(_sps), + createConfigFrame(_pps) + }; return getAVCInfo(_sps, _width, _height, _fps); } @@ -284,23 +288,11 @@ bool H264Track::inputFrame_l(const Frame::Ptr &frame) { void H264Track::insertConfigFrame(const Frame::Ptr &frame) { if (!_sps.empty()) { - auto spsFrame = FrameImp::create(); - spsFrame->_prefix_size = 4; - spsFrame->_buffer.assign("\x00\x00\x00\x01", 4); - spsFrame->_buffer.append(_sps); - spsFrame->_dts = frame->dts(); - spsFrame->setIndex(frame->getIndex()); - VideoTrack::inputFrame(spsFrame); + VideoTrack::inputFrame(createConfigFrame(_sps, frame->dts(), frame->getIndex())); } if (!_pps.empty()) { - auto ppsFrame = FrameImp::create(); - ppsFrame->_prefix_size = 4; - ppsFrame->_buffer.assign("\x00\x00\x00\x01", 4); - ppsFrame->_buffer.append(_pps); - ppsFrame->_dts = frame->dts(); - ppsFrame->setIndex(frame->getIndex()); - VideoTrack::inputFrame(ppsFrame); + VideoTrack::inputFrame(createConfigFrame(_pps, frame->dts(), frame->getIndex())); } } diff --git a/ext-codec/H264.h b/ext-codec/H264.h index cf054da0..9940854c 100644 --- a/ext-codec/H264.h +++ b/ext-codec/H264.h @@ -115,6 +115,7 @@ public: toolkit::Buffer::Ptr getExtraData() const override; void setExtraData(const uint8_t *data, size_t size) override; bool update() override; + std::vector getConfigFrames() const override { return _config_frames; } private: Sdp::Ptr getSdp(uint8_t payload_type) const override; @@ -129,7 +130,20 @@ private: float _fps = 0; std::string _sps; std::string _pps; + std::vector _config_frames; }; +template +Frame::Ptr createConfigFrame(const std::string &data, uint64_t dts = 0, int index = 0) { + auto frame = FrameType::create(); + frame->_prefix_size = 4; + frame->_buffer.assign("\x00\x00\x00\x01", 4); + frame->_buffer.append(data); + frame->_dts = dts; + frame->setIndex(index); + return frame; +} + }//namespace mediakit + #endif //ZLMEDIAKIT_H264_H diff --git a/ext-codec/H265.cpp b/ext-codec/H265.cpp index 3a999255..cc0831f6 100644 --- a/ext-codec/H265.cpp +++ b/ext-codec/H265.cpp @@ -66,7 +66,7 @@ H265Track::H265Track(const string &vps,const string &sps, const string &pps,int _vps = vps.substr(vps_prefix_len); _sps = sps.substr(sps_prefix_len); _pps = pps.substr(pps_prefix_len); - update(); + H265Track::update(); } CodecId H265Track::getCodecId() const { @@ -182,6 +182,11 @@ void H265Track::setExtraData(const uint8_t *data, size_t bytes) { } bool H265Track::update() { + _config_frames = std::vector{ + createConfigFrame(_vps), + createConfigFrame(_sps), + createConfigFrame(_pps) + }; return getHEVCInfo(_vps, _sps, _width, _height, _fps); } @@ -194,32 +199,13 @@ void H265Track::insertConfigFrame(const Frame::Ptr &frame) { return; } if (!_vps.empty()) { - auto vpsFrame = FrameImp::create(); - vpsFrame->_prefix_size = 4; - vpsFrame->_buffer.assign("\x00\x00\x00\x01", 4); - vpsFrame->_buffer.append(_vps); - vpsFrame->_dts = frame->dts(); - vpsFrame->setIndex(frame->getIndex()); - VideoTrack::inputFrame(vpsFrame); + VideoTrack::inputFrame(createConfigFrame(_vps)); } if (!_sps.empty()) { - auto spsFrame = FrameImp::create(); - spsFrame->_prefix_size = 4; - spsFrame->_buffer.assign("\x00\x00\x00\x01", 4); - spsFrame->_buffer.append(_sps); - spsFrame->_dts = frame->dts(); - spsFrame->setIndex(frame->getIndex()); - VideoTrack::inputFrame(spsFrame); + VideoTrack::inputFrame(createConfigFrame(_sps)); } - if (!_pps.empty()) { - auto ppsFrame = FrameImp::create(); - ppsFrame->_prefix_size = 4; - ppsFrame->_buffer.assign("\x00\x00\x00\x01", 4); - ppsFrame->_buffer.append(_pps); - ppsFrame->_dts = frame->dts(); - ppsFrame->setIndex(frame->getIndex()); - VideoTrack::inputFrame(ppsFrame); + VideoTrack::inputFrame(createConfigFrame(_pps)); } } diff --git a/ext-codec/H265.h b/ext-codec/H265.h index 50775727..8132f444 100644 --- a/ext-codec/H265.h +++ b/ext-codec/H265.h @@ -142,6 +142,7 @@ public: toolkit::Buffer::Ptr getExtraData() const override; void setExtraData(const uint8_t *data, size_t size) override; bool update() override; + std::vector getConfigFrames() const override { return _config_frames; } private: Sdp::Ptr getSdp(uint8_t payload_type) const override; @@ -157,6 +158,7 @@ private: std::string _vps; std::string _sps; std::string _pps; + std::vector _config_frames; }; }//namespace mediakit diff --git a/src/Extension/Track.h b/src/Extension/Track.h index ba101282..006bd87a 100644 --- a/src/Extension/Track.h +++ b/src/Extension/Track.h @@ -109,6 +109,11 @@ public: * 返回视频fps */ virtual float getVideoFps() const { return 0; } + + /** + * 返回相关 sps/pps 等 + */ + virtual std::vector getConfigFrames() const { return std::vector{}; } }; class VideoTrackImp : public VideoTrack { diff --git a/webrtc/WebRtcPlayer.cpp b/webrtc/WebRtcPlayer.cpp index cfafae2b..b410a154 100644 --- a/webrtc/WebRtcPlayer.cpp +++ b/webrtc/WebRtcPlayer.cpp @@ -9,7 +9,10 @@ */ #include "WebRtcPlayer.h" + #include "Common/config.h" +#include "Extension/Factory.h" +#include "Util/base64.h" using namespace std; @@ -32,6 +35,9 @@ WebRtcPlayer::WebRtcPlayer(const EventPoller::Ptr &poller, _media_info = info; _play_src = src; CHECK(src); + + GET_CONFIG(bool, direct_proxy, Rtsp::kDirectProxy); + _send_config_frames_once = direct_proxy; } void WebRtcPlayer::onStartWebRTC() { @@ -56,6 +62,13 @@ void WebRtcPlayer::onStartWebRTC() { if (!strong_self) { return; } + + if (strong_self->_send_config_frames_once && !pkt->empty()) { + const auto &first_rtp = pkt->front(); + strong_self->sendConfigFrames(first_rtp->getSeq(), first_rtp->sample_rate, first_rtp->getStamp(), first_rtp->ntp_stamp); + strong_self->_send_config_frames_once = false; + } + size_t i = 0; pkt->for_each([&](const RtpPacket::Ptr &rtp) { //TraceL<<"send track type:"<type<<" ts:"<getStamp()<<" ntp:"<ntp_stamp<<" size:"<getPayloadSize()<<" i:"<getSdp()); } +void WebRtcPlayer::sendConfigFrames(uint32_t before_seq, uint32_t sample_rate, uint32_t timestamp, uint64_t ntp_timestamp) { + auto play_src = _play_src.lock(); + if (!play_src) { + return; + } + auto video_track = std::dynamic_pointer_cast(play_src->getTrack(mediakit::TrackVideo)); + if (!video_track) { + return; + } + auto frames = video_track->getConfigFrames(); + if (frames.empty()) { + return; + } + auto encoder = mediakit::Factory::getRtpEncoderByCodecId(video_track->getCodecId(), 0); + if (!encoder) { + return; + } + + GET_CONFIG(uint32_t, video_mtu, Rtp::kVideoMtuSize); + encoder->setRtpInfo(0, video_mtu, sample_rate, 0, 0, 0); + + auto seq = before_seq - frames.size(); + for (const auto &frame : video_track->getConfigFrames()) { + auto rtp = encoder->getRtpInfo().makeRtp( + TrackVideo, frame->data() + frame->prefixSize(), frame->size() - frame->prefixSize(), false, 0); + auto header = rtp->getHeader(); + header->seq = htons(seq++); + header->stamp = htonl(timestamp); + rtp->ntp_stamp = ntp_timestamp; + onSendRtp(rtp, false); + } +} + }// namespace mediakit \ No newline at end of file diff --git a/webrtc/WebRtcPlayer.h b/webrtc/WebRtcPlayer.h index ccacd410..96e9e830 100644 --- a/webrtc/WebRtcPlayer.h +++ b/webrtc/WebRtcPlayer.h @@ -31,11 +31,17 @@ protected: private: WebRtcPlayer(const EventPoller::Ptr &poller, const RtspMediaSource::Ptr &src, const MediaInfo &info); + void sendConfigFrames(uint32_t before_seq, uint32_t sample_rate, uint32_t timestamp, uint64_t ntp_timestamp); + private: //媒体相关元数据 MediaInfo _media_info; //播放的rtsp源 std::weak_ptr _play_src; + + // rtp 直接转发情况下通常会缺少 sps/pps, 在转发 rtp 前, 先发送一次相关帧信息, 部分情况下是可以播放的 + bool _send_config_frames_once { false }; + //播放rtsp源的reader对象 RtspMediaSource::RingType::RingReader::Ptr _reader; };