From b455586b3456f83c4241d21075a827a3ec4c5d35 Mon Sep 17 00:00:00 2001 From: cqm Date: Tue, 29 Nov 2022 20:38:59 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=9E=84=20audio=5Ftranscode=20?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=EF=BC=9A=20-=20=E7=8B=AC=E7=AB=8B=E5=87=BA?= =?UTF-8?q?=20RtcMediaSource=EF=BC=8C=E5=B9=B6=E5=8F=AA=E5=AF=B9rtc?= =?UTF-8?q?=E5=BC=80=E6=94=BE=20-=20=E5=A2=9E=E5=8A=A0Rtc=20g711=E8=BD=AC?= =?UTF-8?q?=E7=A0=81=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Conflicts: # src/Common/MultiMediaSourceMuxer.cpp # src/Common/MultiMediaSourceMuxer.h --- src/Common/MultiMediaSourceMuxer.cpp | 127 +----------------- src/Common/MultiMediaSourceMuxer.h | 9 -- webrtc/RtcMediaSource.cpp | 190 +++++++++++++++++++++++++++ webrtc/RtcMediaSource.h | 60 +++++++++ webrtc/WebRtcPusher.cpp | 3 +- webrtc/WebRtcTransport.cpp | 8 +- webrtc/WebRtcTransport.h | 1 + 7 files changed, 266 insertions(+), 132 deletions(-) create mode 100644 webrtc/RtcMediaSource.cpp create mode 100644 webrtc/RtcMediaSource.h diff --git a/src/Common/MultiMediaSourceMuxer.cpp b/src/Common/MultiMediaSourceMuxer.cpp index d1d44949..7a77ac5e 100644 --- a/src/Common/MultiMediaSourceMuxer.cpp +++ b/src/Common/MultiMediaSourceMuxer.cpp @@ -11,9 +11,6 @@ #include #include "Common/config.h" #include "MultiMediaSourceMuxer.h" -#include "Extension/AAC.h" -#include "Extension/Opus.h" -#include "Extension/G711.h" #include "Rtp/RtpSender.h" #include "Record/HlsRecorder.h" #include "Record/HlsMediaSource.h" @@ -21,9 +18,8 @@ #include "Rtmp/RtmpMediaSourceMuxer.h" #include "TS/TSMediaSourceMuxer.h" #include "FMP4/FMP4MediaSourceMuxer.h" - -#ifdef ENABLE_FFMPEG -#include "Codec/Transcode.h" +#if defined(ENABLE_WEBRTC) +#include "webrtc/RtcMediaSource.h" #endif using namespace std; using namespace toolkit; @@ -108,7 +104,7 @@ MultiMediaSourceMuxer::MultiMediaSourceMuxer(const string &vhost, const string & } if (option.enable_rtc) { #if defined(ENABLE_WEBRTC) - _rtc = std::make_shared(vhost, app, stream, option, std::make_shared(dur_sec), RTC_SCHEMA); + _rtc = std::make_shared(vhost, app, stream, option, std::make_shared(dur_sec)); #endif } if (option.enable_hls) { @@ -120,14 +116,6 @@ MultiMediaSourceMuxer::MultiMediaSourceMuxer(const string &vhost, const string & if (option.enable_ts) { _ts = std::make_shared(vhost, app, stream, option); } - if (option.audio_transcode) { -#if defined(ENABLE_FFMPEG) - InfoL << "enable audio_transcode"; -#else - InfoL << "without ffmpeg disable audio_transcode"; - _option.audio_transcode = false; -#endif - } #if defined(ENABLE_MP4) if (option.enable_fmp4) { _fmp4 = std::make_shared(vhost, app, stream, option); @@ -348,79 +336,9 @@ EventPoller::Ptr MultiMediaSourceMuxer::getOwnerPoller(MediaSource &sender) { bool MultiMediaSourceMuxer::onTrackReady(const Track::Ptr &track) { bool ret = false; - auto rtmp = _rtmp; - auto rtc = _rtc; -#if defined(ENABLE_FFMPEG) - if (_option.audio_transcode) { - if (track->getCodecId() == CodecAAC) { - if (rtmp) { - rtmp->addTrack(track); - rtmp = nullptr; - } - _audio_dec = nullptr; - _audio_enc = nullptr; - _opus_mute_maker = nullptr; - if (rtc) { - Track::Ptr newTrack(new OpusTrack()); - GET_CONFIG(int, bitrate, General::kOpusBitrate); - newTrack->setBitRate(bitrate); - rtc->addTrack(newTrack); - rtc = nullptr; - if (!hasMuteAudio()) { - // aac to opus - _audio_dec.reset(new FFmpegDecoder(track)); - _audio_enc.reset(new FFmpegEncoder(newTrack)); - _audio_dec->setOnDecode([this](const FFmpegFrame::Ptr & frame) { - _audio_enc->inputFrame(frame, false); - }); - _audio_enc->setOnEncode([this](const Frame::Ptr& frame) { - // fill data to _rtc - if (_rtc && _rtc->isEnabled()) - _rtc->inputFrame(frame); - }); - } - else { - _opus_mute_maker = std::make_shared(CodecOpus); - _opus_mute_maker->addDelegate([this](const Frame::Ptr &frame) { - if (_rtc && _rtc->isEnabled()) - _rtc->inputFrame(frame); - return true; - }); - } - } - } - else if (track->getTrackType() == TrackAudio) { - if (rtc) { - rtc->addTrack(track); - rtc = nullptr; - } - _audio_dec = nullptr; - _audio_enc = nullptr; - _opus_mute_maker = nullptr; - if (rtmp) { - Track::Ptr newTrack(new AACTrack(44100, std::dynamic_pointer_cast(track)->getAudioChannel())); - GET_CONFIG(int, bitrate, General::kAacBitrate); - newTrack->setBitRate(bitrate); - rtmp->addTrack(newTrack); - rtmp = nullptr; - - _audio_dec.reset(new FFmpegDecoder(track)); - _audio_enc.reset(new FFmpegEncoder(newTrack)); - _audio_dec->setOnDecode([this](const FFmpegFrame::Ptr & frame) { - _audio_enc->inputFrame(frame, false); - }); - _audio_enc->setOnEncode([this](const Frame::Ptr& frame) { - // fill aac frame to rtmp - if (_rtmp && _rtmp->isEnabled()) - _rtmp->inputFrame(frame); - }); - } - } - } -#endif - if (rtc && rtc->addTrack(track)) + if (_rtc && _rtc->addTrack(track)) ret = true; - if (rtmp && rtmp->addTrack(track)) + if (_rtmp && _rtmp->addTrack(track)) ret = true; if (_rtsp && _rtsp->addTrack(track)) ret = true; @@ -484,11 +402,6 @@ void MultiMediaSourceMuxer::resetTracks() { if (_rtc) { _rtc->resetTracks(); } -#if defined(ENABLE_FFMPEG) - _audio_dec = nullptr; - _audio_dec = nullptr; - _opus_mute_maker = nullptr; -#endif #if defined(ENABLE_MP4) if (_fmp4) { _fmp4->resetTracks(); @@ -521,36 +434,10 @@ bool MultiMediaSourceMuxer::onTrackFrame(const Frame::Ptr &frame_in) { } bool ret = false; - RtspMediaSourceMuxer::Ptr rtc; - RtmpMediaSourceMuxer::Ptr rtmp; - if (_rtc && _rtc->isEnabled()) - rtc = _rtc; - if (_rtmp && _rtmp->isEnabled()) - rtmp = _rtmp; -#if defined(ENABLE_FFMPEG) - if (_option.audio_transcode) { - if (frame->getCodecId() == CodecAAC) { - if (rtc) { - if (_audio_dec && rtc->readerCount()) - _audio_dec->inputFrame(frame, true, false, false); - rtc = nullptr; - } - } - else if (frame->getTrackType() == TrackAudio) { - if (rtmp) { - if (_audio_dec && rtmp->readerCount()) - _audio_dec->inputFrame(frame, true, false, false); - rtmp = nullptr; - } - } else if (_opus_mute_maker && rtc) { - _opus_mute_maker->inputFrame(frame); - } - } -#endif - if (rtc && rtc->inputFrame(frame)) + if (_rtc && _rtc->inputFrame(frame)) ret = true; - if (rtmp && rtmp->inputFrame(frame)) + if (_rtmp && _rtmp->inputFrame(frame)) ret = true; if (_rtsp && _rtsp->inputFrame(frame)) diff --git a/src/Common/MultiMediaSourceMuxer.h b/src/Common/MultiMediaSourceMuxer.h index cc6ecb92..dc7418a7 100644 --- a/src/Common/MultiMediaSourceMuxer.h +++ b/src/Common/MultiMediaSourceMuxer.h @@ -22,10 +22,6 @@ class RtmpMediaSourceMuxer; class TSMediaSourceMuxer; class FMP4MediaSourceMuxer; class RtpSender; -#ifdef ENABLE_FFMPEG -class FFmpegDecoder; -class FFmpegEncoder; -#endif class MultiMediaSourceMuxer : public MediaSourceEventInterceptor, public MediaSink, public std::enable_shared_from_this{ public: @@ -174,11 +170,6 @@ private: std::shared_ptr _rtsp; std::shared_ptr _ts; std::shared_ptr _rtc; -#if defined(ENABLE_FFMPEG) - MuteAudioMaker::Ptr _opus_mute_maker; - std::shared_ptr _audio_dec; - std::shared_ptr _audio_enc; -#endif MediaSinkInterface::Ptr _mp4; std::shared_ptr _hls; toolkit::EventPoller::Ptr _poller; diff --git a/webrtc/RtcMediaSource.cpp b/webrtc/RtcMediaSource.cpp new file mode 100644 index 00000000..c1abae34 --- /dev/null +++ b/webrtc/RtcMediaSource.cpp @@ -0,0 +1,190 @@ +#include "RtcMediaSource.h" +#include "Common/config.h" +#include "Codec/Transcode.h" +#include "Extension/AAC.h" +#include "Extension/Opus.h" +#include "Extension/G711.h" +// for RTC configure +#include "WebRtcTransport.h" +namespace mediakit { + +bool needTransToOpus(CodecId codec) { + GET_CONFIG(int, transG711, Rtc::kTranscodeG711); + switch (codec) + { + case CodecG711U: + case CodecG711A: + return transG711; + case CodecAAC: + return true; + default: + return false; + } +} + +bool needTransToAac(CodecId codec) { + GET_CONFIG(int, transG711, Rtc::kTranscodeG711); + switch (codec) + { + case CodecG711U: + case CodecG711A: + return transG711; + case CodecOpus: + return true; + default: + return false; + } +} + +RtcMediaSourceMuxer::RtcMediaSourceMuxer(const std::string &vhost, const std::string &strApp, const std::string &strId, const ProtocolOption &option, const TitleSdp::Ptr &title) + : RtspMediaSourceMuxer(vhost, strApp, strId, option, title, RTC_SCHEMA) +{ + if (_option.audio_transcode) { +#ifndef ENABLE_FFMPEG + WarnL << "without ffmpeg, skip transcode setting"; + _option.audio_transcode = false; +#endif + } +} + +bool RtcMediaSourceMuxer::inputFrame(const Frame::Ptr &frame) +{ + if (_clear_cache && _on_demand) { + _clear_cache = false; + _media_src->clearCache(); + } + if (_enabled || !_on_demand) { +#if defined(ENABLE_FFMPEG) + if (_option.audio_transcode && needTransToOpus(frame->getCodecId())) { + if (!_audio_dec) { // addTrack可能没调, 这边根据情况再调一次 + Track::Ptr track; + switch (frame->getCodecId()) + { + case CodecAAC: + if (frame->prefixSize()) { + std::string cfg = makeAacConfig((uint8_t *)(frame->data()), frame->prefixSize()); + track = std::make_shared(cfg); + } + else { + track = std::make_shared(44100, 2); + } + break; + case CodecG711A: + case CodecG711U: + track.reset(new G711Track(frame->getCodecId())); + break; + default: + break; + } + if (track) + addTrack(track); + if (!_audio_dec) return false; + } + if (readerCount()) { + _audio_dec->inputFrame(frame, true, false); + if (!_count) + InfoL << "start transcode " << frame->getCodecName() << "," << frame->pts() << "->Opus"; + _count++; + } + else if (_count) { + InfoL << "stop transcode with " << _count << " items"; + _count = 0; + } + return true; + } +#endif + return RtspMuxer::inputFrame(frame); + } + return false; +} + +#if defined(ENABLE_FFMPEG) + +bool RtcMediaSourceMuxer::addTrack(const Track::Ptr & track) +{ + Track::Ptr newTrack = track; + if (_option.audio_transcode && needTransToOpus(track->getCodecId())) { + newTrack = std::make_shared(); + GET_CONFIG(int, bitrate, General::kOpusBitrate); + newTrack->setBitRate(bitrate); + _audio_dec.reset(new FFmpegDecoder(track)); + _audio_enc.reset(new FFmpegEncoder(newTrack)); + // aac to opus + _audio_dec->setOnDecode([this](const FFmpegFrame::Ptr & frame) { + _audio_enc->inputFrame(frame, false); + }); + _audio_enc->setOnEncode([this](const Frame::Ptr& frame) { + RtspMuxer::inputFrame(frame); + }); + } + return RtspMuxer::addTrack(newTrack); +} + + +void RtcMediaSourceMuxer::resetTracks() +{ + RtspMuxer::resetTracks(); + _audio_dec = nullptr; + _audio_enc = nullptr; + if (_count) { + InfoL << "stop transcode with " << _count << " items"; + _count = 0; + } +} + +bool RtcMediaSourceImp::addTrack(const Track::Ptr &track) +{ + if (_muxer) { + Track::Ptr newTrack = track; + if (_option.audio_transcode && needTransToAac(track->getCodecId())) { + newTrack.reset(new AACTrack(44100, 2)); + GET_CONFIG(int, bitrate, General::kAacBitrate); + newTrack->setBitRate(bitrate); + _audio_dec.reset(new FFmpegDecoder(track)); + _audio_enc.reset(new FFmpegEncoder(newTrack)); + // hook data to newTack + track->addDelegate([this](const Frame::Ptr &frame) -> bool { + if (_all_track_ready && 0 == _muxer->totalReaderCount()) { + if (_count) { + InfoL << "stop transcode with " << _count << " items"; + _count = 0; + } + return true; + } + if (_audio_dec) { + if (!_count) + InfoL << "start transcode " << frame->getCodecName() << "," << frame->pts() << "->AAC"; + _count++; + _audio_dec->inputFrame(frame, true, false); + } + return true; + }); + _audio_dec->setOnDecode([this](const FFmpegFrame::Ptr & frame) { + _audio_enc->inputFrame(frame, false); + }); + _audio_enc->setOnEncode([newTrack](const Frame::Ptr& frame) { + newTrack->inputFrame(frame); + }); + } + + if (_muxer->addTrack(newTrack)) { + newTrack->addDelegate(_muxer); + return true; + } + } + return false; +} + +void RtcMediaSourceImp::resetTracks() +{ + RtspMediaSourceImp::resetTracks(); + _audio_dec = nullptr; + _audio_enc = nullptr; + if (_count) { + InfoL << "stop transcode with " << _count << " items"; + _count = 0; + } +} + +#endif +} diff --git a/webrtc/RtcMediaSource.h b/webrtc/RtcMediaSource.h new file mode 100644 index 00000000..d3f20a5b --- /dev/null +++ b/webrtc/RtcMediaSource.h @@ -0,0 +1,60 @@ +#ifndef ZLMEDIAKIT_RTCMEDIASOURCE_H +#define ZLMEDIAKIT_RTCMEDIASOURCE_H + +#include "Rtsp/RtspMediaSourceMuxer.h" +#include "Rtsp/RtspMediaSourceImp.h" +namespace mediakit { +class FFmpegDecoder; +class FFmpegEncoder; +bool needTransToOpus(CodecId codec); +bool needTransToAac(CodecId codec); + +class RtcMediaSourceImp : public RtspMediaSourceImp { +public: + typedef std::shared_ptr Ptr; + + RtcMediaSourceImp(const std::string &vhost, const std::string &app, const std::string &id, int ringSize = RTP_GOP_SIZE) + : RtspMediaSourceImp(vhost, app, id, RTC_SCHEMA, ringSize) { + } +#if defined(ENABLE_FFMPEG) + ~RtcMediaSourceImp() override { resetTracks(); } + /** + * _demuxer触发的添加Track事件 + */ + bool addTrack(const Track::Ptr &track) override; + void resetTracks() override; +private: + int _count = 0; + std::shared_ptr _audio_dec; + std::shared_ptr _audio_enc; +#endif +}; + +class RtcMediaSourceMuxer : public RtspMediaSourceMuxer { +public: + typedef std::shared_ptr Ptr; + + RtcMediaSourceMuxer(const std::string &vhost, + const std::string &strApp, + const std::string &strId, + const ProtocolOption &option, + const TitleSdp::Ptr &title = nullptr); + + + bool inputFrame(const Frame::Ptr &frame) override; + +#if defined(ENABLE_FFMPEG) + ~RtcMediaSourceMuxer() override{resetTracks();} + + bool addTrack(const Track::Ptr & track) override; + void resetTracks() override; + +private: + int _count = 0; + std::shared_ptr _audio_dec; + std::shared_ptr _audio_enc; +#endif +}; + +} +#endif \ No newline at end of file diff --git a/webrtc/WebRtcPusher.cpp b/webrtc/WebRtcPusher.cpp index d5c6d063..4e3e9091 100644 --- a/webrtc/WebRtcPusher.cpp +++ b/webrtc/WebRtcPusher.cpp @@ -10,6 +10,7 @@ #include "WebRtcPusher.h" #include "Common/config.h" +#include "RtcMediaSource.h" using namespace std; @@ -98,7 +99,7 @@ void WebRtcPusher::onRecvRtp(MediaTrack &track, const string &rid, RtpPacket::Pt auto &src = _push_src_sim[rid]; if (!src) { auto stream_id = rid.empty() ? _push_src->getId() : _push_src->getId() + "_" + rid; - auto src_imp = std::make_shared(_push_src->getVhost(), _push_src->getApp(), stream_id, _push_src->getSchema()); + auto src_imp = std::make_shared(_push_src->getVhost(), _push_src->getApp(), stream_id); _push_src_sim_ownership[rid] = src_imp->getOwnership(); src_imp->setSdp(_push_src->getSdp()); src_imp->setProtocolOption(_push_src->getProtocolOption()); diff --git a/webrtc/WebRtcTransport.cpp b/webrtc/WebRtcTransport.cpp index 09d0d466..0c3d7e49 100644 --- a/webrtc/WebRtcTransport.cpp +++ b/webrtc/WebRtcTransport.cpp @@ -18,7 +18,7 @@ #include "Rtsp/Rtsp.h" #include "Rtsp/RtpReceiver.h" #include "WebRtcTransport.h" - +#include "RtcMediaSource.h" #include "WebRtcEchoTest.h" #include "WebRtcPlayer.h" #include "WebRtcPusher.h" @@ -43,6 +43,9 @@ const string kTimeOutSec = RTC_FIELD "timeoutSec"; const string kExternIP = RTC_FIELD "externIP"; // 设置remb比特率，非0时关闭twcc并开启remb。该设置在rtc推流时有效，可以控制推流画质 const string kRembBitRate = RTC_FIELD "rembBitRate"; +// 是否转码G711音频，做到: 出rtc将g711转成aac，入rtc将g711转成opus +const string kTranscodeG711 = RTC_FIELD "transcodeG711"; + // webrtc单端口udp服务器 const string kPort = RTC_FIELD "port"; @@ -54,6 +57,7 @@ static onceToken token([]() { mINI::Instance()[kRembBitRate] = 0; mINI::Instance()[kPort] = 8000; mINI::Instance()[kTcpPort] = 8000; + mINI::Instance()[kTranscodeG711] = 0; }); } // namespace RTC @@ -1185,7 +1189,7 @@ void push_plugin(Session &sender, const WebRtcArgs &args, const WebRtcPluginMana } if (!push_src) { - push_src = std::make_shared(info._vhost, info._app, info._streamid, schema); + push_src = std::make_shared(info._vhost, info._app, info._streamid); push_src_ownership = push_src->getOwnership(); push_src->setProtocolOption(option); } diff --git a/webrtc/WebRtcTransport.h b/webrtc/WebRtcTransport.h index dfae8012..6a59fe6c 100644 --- a/webrtc/WebRtcTransport.h +++ b/webrtc/WebRtcTransport.h @@ -33,6 +33,7 @@ namespace Rtc { extern const std::string kPort; extern const std::string kTcpPort; extern const std::string kTimeOutSec; +extern const std::string kTranscodeG711; }//namespace RTC class WebRtcInterface {