diff --git a/CMakeLists.txt b/CMakeLists.txt index cb85679b..39c6c517 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,7 @@ option(ENABLE_FAAC "Enable FAAC" OFF) option(ENABLE_FFMPEG "Enable FFmpeg" OFF) option(ENABLE_HLS "Enable HLS" ON) option(ENABLE_JEMALLOC_STATIC "Enable static linking to the jemalloc library" OFF) +option(ENABLE_JEMALLOC_DUMP "Enable jemalloc to dump malloc statistics" OFF) option(ENABLE_MEM_DEBUG "Enable Memory Debug" OFF) option(ENABLE_MP4 "Enable MP4" ON) option(ENABLE_HLS_FMP4 "Enable HLS-FMP4" ON) @@ -335,7 +336,11 @@ if(ENABLE_JEMALLOC_STATIC) if(NOT EXISTS ${DEP_ROOT_DIR}) file(MAKE_DIRECTORY ${DEP_ROOT_DIR}) endif() - + if (ENABLE_JEMALLOC_DUMP) + set(ENABLE_JEMALLOC_STAT ON) + else () + set(ENABLE_JEMALLOC_STAT OFF) + endif () include(Jemalloc) include_directories(SYSTEM ${DEP_ROOT_DIR}/${JEMALLOC_NAME}/include/jemalloc) link_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/lib) @@ -349,6 +354,12 @@ if(JEMALLOC_FOUND) message(STATUS "found library: ${JEMALLOC_LIBRARIES}") include_directories(${JEMALLOC_INCLUDE_DIR}) update_cached_list(MK_LINK_LIBRARIES ${JEMALLOC_LIBRARIES}) + add_definitions(-DUSE_JEMALLOC) + message(STATUS "jemalloc will be used to avoid memory fragmentation") + if (ENABLE_JEMALLOC_DUMP) + add_definitions(-DENABLE_JEMALLOC_DUMP) + message(STATUS "jemalloc will save memory usage statistics when the program exits") + endif () endif() # 查找 openssl 是否安装 diff --git a/cmake/Jemalloc.cmake b/cmake/Jemalloc.cmake index 94d46b64..38847e58 100644 --- a/cmake/Jemalloc.cmake +++ b/cmake/Jemalloc.cmake @@ -1,21 +1,34 @@ # Download and build Jemalloc -set(JEMALLOC_VERSION 5.2.1) +set(JEMALLOC_VERSION 5.3.0) set(JEMALLOC_NAME jemalloc-${JEMALLOC_VERSION}) set(JEMALLOC_TAR_PATH ${DEP_ROOT_DIR}/${JEMALLOC_NAME}.tar.bz2) list(APPEND jemalloc_CONFIG_ARGS --disable-initial-exec-tls) -list(APPEND jemalloc_CONFIG_ARGS --without-export) +#list(APPEND jemalloc_CONFIG_ARGS --without-export) +if (ENABLE_JEMALLOC_STAT) + list(APPEND jemalloc_CONFIG_ARGS --enable-stats) + message(STATUS "Jemalloc stats enabled") +else () list(APPEND jemalloc_CONFIG_ARGS --disable-stats) + message(STATUS "Jemalloc stats disabled") +endif () list(APPEND jemalloc_CONFIG_ARGS --disable-libdl) #list(APPEND jemalloc_CONFIG_ARGS --disable-cxx) #list(APPEND jemalloc_CONFIG_ARGS --with-jemalloc-prefix=je_) #list(APPEND jemalloc_CONFIG_ARGS --enable-debug) if(NOT EXISTS ${JEMALLOC_TAR_PATH}) - message(STATUS "Downloading ${JEMALLOC_NAME}...") - file(DOWNLOAD https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/${JEMALLOC_NAME}.tar.bz2 - ${JEMALLOC_TAR_PATH}) + set(JEMALLOC_URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/${JEMALLOC_NAME}.tar.bz2) + message(STATUS "Downloading ${JEMALLOC_NAME} from ${JEMALLOC_URL}") + file(DOWNLOAD ${JEMALLOC_URL} ${JEMALLOC_TAR_PATH} SHOW_PROGRESS STATUS JEMALLOC_DOWNLOAD_STATUS LOG JEMALLOC_DOWNLOAD_LOG) + list(GET JEMALLOC_DOWNLOAD_STATUS 0 JEMALLOC_DOWNLOAD_STATUS_CODE) + if(NOT JEMALLOC_DOWNLOAD_STATUS_CODE EQUAL 0) + file(REMOVE ${JEMALLOC_TAR_PATH}) + message(STATUS "${JEMALLOC_DOWNLOAD_LOG}") + message(FATAL_ERROR "${JEMALLOC_NAME} download failed! error is ${JEMALLOC_DOWNLOAD_STATUS}") + return() + endif () endif() SET( DIR_CONTAINING_JEMALLOC ${DEP_ROOT_DIR}/${JEMALLOC_NAME} ) diff --git a/server/System.cpp b/server/System.cpp index bc93ed3c..c06d34e1 100644 --- a/server/System.cpp +++ b/server/System.cpp @@ -22,10 +22,11 @@ #include #include +#include "Common/JemallocUtil.h" +#include "Common/macros.h" +#include "System.h" #include "Util/logger.h" #include "Util/uv_errno.h" -#include "System.h" -#include "Common/macros.h" using namespace std; using namespace toolkit; @@ -55,6 +56,16 @@ string System::execute(const string &cmd) { static constexpr int MAX_STACK_FRAMES = 128; +static void save_jemalloc_stats() { + string jemalloc_status = JemallocUtil::get_malloc_stats(); + if (jemalloc_status.empty()) { + return; + } + ofstream out(StrPrinter << exeDir() << "/jemalloc.json", ios::out | ios::binary | ios::trunc); + out << jemalloc_status; + out.flush(); +} + static void sig_crash(int sig) { signal(sig, SIG_DFL); void *array[MAX_STACK_FRAMES]; @@ -149,6 +160,12 @@ void System::startDaemon(bool &kill_parent_if_failed) { } void System::systemSetup(){ + +#ifdef ENABLE_JEMALLOC_DUMP + //Save memory report when program exits + atexit(save_jemalloc_stats); +#endif //ENABLE_JEMALLOC_DUMP + #if !defined(_WIN32) struct rlimit rlim,rlim_new; if (getrlimit(RLIMIT_CORE, &rlim)==0) { diff --git a/server/WebApi.cpp b/server/WebApi.cpp index e8081756..7c8e35c0 100755 --- a/server/WebApi.cpp +++ b/server/WebApi.cpp @@ -1911,24 +1911,28 @@ void installWebApi() { void unInstallWebApi(){ { lock_guard lck(s_proxyMapMtx); - s_proxyMap.clear(); + auto proxyMap(std::move(s_proxyMap)); + proxyMap.clear(); } { lock_guard lck(s_ffmpegMapMtx); - s_ffmpegMap.clear(); + auto ffmpegMap(std::move(s_ffmpegMap)); + ffmpegMap.clear(); } { lock_guard lck(s_proxyPusherMapMtx); - s_proxyPusherMap.clear(); + auto proxyPusherMap(std::move(s_proxyPusherMap)); + proxyPusherMap.clear(); } { #if defined(ENABLE_RTPPROXY) RtpSelector::Instance().clear(); lock_guard lck(s_rtpServerMapMtx); - s_rtpServerMap.clear(); + auto rtpServerMap(std::move(s_rtpServerMap)); + rtpServerMap.clear(); #endif } NoticeCenter::Instance().delListener(&web_api_tag); diff --git a/src/Common/JemallocUtil.cpp b/src/Common/JemallocUtil.cpp new file mode 100644 index 00000000..fa2de64a --- /dev/null +++ b/src/Common/JemallocUtil.cpp @@ -0,0 +1,74 @@ +/* +* Copyright (c) 2016 The ZLMediaKit project authors. All Rights Reserved. +* +* This file is part of ZLMediaKit(https://github.com/xia-chu/ZLMediaKit). +* +* Use of this source code is governed by MIT license that can be found in the +* LICENSE file in the root of the source tree. All contributing project authors +* may be found in the AUTHORS file in the root of the source tree. +*/ + + +#include "JemallocUtil.h" +#include "Util/logger.h" +#ifdef USE_JEMALLOC +#include +#include +#endif + +namespace mediakit { + +void set_profile_active(bool active) { +#ifdef USE_JEMALLOC + int err = mallctl("prof.active", nullptr, nullptr, (void *)&active, sizeof(active)); + if (err != 0) { + WarnL << "mallctl failed with: " << err; + } +#endif +} + +void JemallocUtil::enable_profiling() { + set_profile_active(true); +} +void JemallocUtil::disable_profiling() { + set_profile_active(false); +} +void JemallocUtil::dump(const std::string &file_name) { +#ifdef USE_JEMALLOC + auto *c_str = file_name.c_str(); + int err = mallctl("prof.dump", nullptr, nullptr, &c_str, sizeof(const char *)); + if (err != 0) { + std::cerr << "mallctl failed with: " << err << std::endl; + } +#endif +} +std::string JemallocUtil::get_malloc_stats() { +#ifdef USE_JEMALLOC + std::string res; + malloc_stats_print([](void *opaque, const char *s) { ((std::string *)opaque)->append(s); }, &res, "J"); + return res; +#else + return ""; +#endif +} + +void JemallocUtil::some_malloc_stats(const std::function &fn) { +#ifdef USE_JEMALLOC + constexpr std::array STATS = { + "stats.allocated", "stats.active", "stats.metadata", "stats.metadata_thp", + "stats.resident", "stats.mapped", "stats.retained", "stats.zero_reallocs", + }; + + for (const char *stat : STATS) { + size_t value; + size_t len = sizeof(value); + auto err = mallctl(stat, &value, &len, nullptr, 0); + if (err != 0) { + ErrorL << "Failed reading " << stat << ": " << err; + continue; + } + fn(stat, value); + } +#endif +} +} // namespace mediakit \ No newline at end of file diff --git a/src/Common/JemallocUtil.h b/src/Common/JemallocUtil.h new file mode 100644 index 00000000..796a58bb --- /dev/null +++ b/src/Common/JemallocUtil.h @@ -0,0 +1,30 @@ +/* +* Copyright (c) 2016 The ZLMediaKit project authors. All Rights Reserved. +* +* This file is part of ZLMediaKit(https://github.com/xia-chu/ZLMediaKit). +* +* Use of this source code is governed by MIT license that can be found in the +* LICENSE file in the root of the source tree. All contributing project authors +* may be found in the AUTHORS file in the root of the source tree. +*/ + +#ifndef ZLMEDIAKIT_JEMALLOCUTIL_H +#define ZLMEDIAKIT_JEMALLOCUTIL_H +#include +#include +namespace mediakit { +class JemallocUtil { +public: + JemallocUtil() = default; + ~JemallocUtil() = default; + + static void enable_profiling(); + + static void disable_profiling(); + + static void dump(const std::string &file_name); + static std::string get_malloc_stats(); + static void some_malloc_stats(const std::function &fn); +}; +} // namespace mediakit +#endif // ZLMEDIAKIT_JEMALLOCUTIL_H diff --git a/src/Common/Stamp.cpp b/src/Common/Stamp.cpp index 87282c39..2656a176 100644 --- a/src/Common/Stamp.cpp +++ b/src/Common/Stamp.cpp @@ -50,7 +50,7 @@ int64_t DeltaStamp::deltaStamp(int64_t stamp) { _last_stamp = stamp; // 如果时间戳回退不多，那么返回负值，否则返回加1 - return -ret < MAX_CTS ? ret : 1; + return -ret < MAX_DELTA_STAMP ? ret : 1; } void Stamp::setPlayBack(bool playback) { diff --git a/src/Http/HttpCookieManager.cpp b/src/Http/HttpCookieManager.cpp index b846a3b9..e829423d 100644 --- a/src/Http/HttpCookieManager.cpp +++ b/src/Http/HttpCookieManager.cpp @@ -61,7 +61,7 @@ bool HttpServerCookie::isExpired() { return _ticker.elapsedTime() > _max_elapsed * 1000; } -void HttpServerCookie::setAttach(std::shared_ptr attach) { +void HttpServerCookie::setAttach(toolkit::Any attach) { _attach = std::move(attach); } @@ -114,8 +114,7 @@ void HttpCookieManager::onManager() { } } -HttpServerCookie::Ptr HttpCookieManager::addCookie(const string &cookie_name, const string &uid_in, - uint64_t max_elapsed, std::shared_ptr attach, int max_client) { +HttpServerCookie::Ptr HttpCookieManager::addCookie(const string &cookie_name, const string &uid_in, uint64_t max_elapsed, toolkit::Any attach, int max_client) { lock_guard lck(_mtx_cookie); auto cookie = _generator.obtain(); auto uid = uid_in.empty() ? cookie : uid_in; diff --git a/src/Http/HttpCookieManager.h b/src/Http/HttpCookieManager.h index e55c2086..53775b79 100644 --- a/src/Http/HttpCookieManager.h +++ b/src/Http/HttpCookieManager.h @@ -85,14 +85,14 @@ public: /** * 设置附加数据 */ - void setAttach(std::shared_ptr attach); + void setAttach(toolkit::Any attach); /* * 获取附加数据 */ template T& getAttach() { - return *static_cast(_attach.get()); + return _attach.get(); } private: @@ -104,7 +104,7 @@ private: std::string _cookie_uuid; uint64_t _max_elapsed; toolkit::Ticker _ticker; - std::shared_ptr _attach; + toolkit::Any _attach; std::weak_ptr _manager; }; @@ -163,7 +163,7 @@ public: */ HttpServerCookie::Ptr addCookie( const std::string &cookie_name, const std::string &uid, uint64_t max_elapsed = COOKIE_DEFAULT_LIFE, - std::shared_ptr attach = nullptr, + toolkit::Any = toolkit::Any{}, int max_client = 1); /** diff --git a/src/Http/HttpFileManager.cpp b/src/Http/HttpFileManager.cpp index c8996d1c..1eaa9ec1 100644 --- a/src/Http/HttpFileManager.cpp +++ b/src/Http/HttpFileManager.cpp @@ -31,13 +31,16 @@ namespace mediakit { // 每次访问一次该cookie，那么将重新刷新cookie有效期 // 假如播放器在60秒内都未访问该cookie，那么将重新触发hls播放鉴权 static int kHlsCookieSecond = 60; +static int kFindSrcIntervalSecond = 3; static const string kCookieName = "ZL_COOKIE"; static const string kHlsSuffix = "/hls.m3u8"; static const string kHlsFMP4Suffix = "/hls.fmp4.m3u8"; struct HttpCookieAttachment { - //是否已经查找到过MediaSource + // 是否已经查找到过MediaSource bool _find_src = false; + // 查找MediaSource计时 + Ticker _find_src_ticker; //cookie生效作用域，本cookie只对该目录下的文件生效 string _path; //上次鉴权失败信息,为空则上次鉴权成功 @@ -414,7 +417,9 @@ static void canAccessPath(Session &sender, const Parser &parser, const MediaInfo // hls相关信息 attach->_hls_data = std::make_shared(media_info, info); } - callback(err_msg, HttpCookieManager::Instance().addCookie(kCookieName, uid, life_second, attach)); + toolkit::Any any; + any.set(std::move(attach)); + callback(err_msg, HttpCookieManager::Instance().addCookie(kCookieName, uid, life_second, std::move(any))); } else { callback(err_msg, nullptr); } @@ -532,14 +537,15 @@ static void accessFile(Session &sender, const Parser &parser, const MediaInfo &m return; } - auto src = cookie->getAttach()._hls_data->getMediaSource(); + auto &attach = cookie->getAttach(); + auto src = attach._hls_data->getMediaSource(); if (src) { - //直接从内存获取m3u8索引文件(而不是从文件系统) + // 直接从内存获取m3u8索引文件(而不是从文件系统) response_file(cookie, cb, file_path, parser, src->getIndexFile()); return; } - if (cookie->getAttach()._find_src) { - //查找过MediaSource，但是流已经注销了，不用再查找 + if (attach._find_src && attach._find_src_ticker.elapsedTime() < kFindSrcIntervalSecond * 1000) { + // 最近已经查找过MediaSource了，为了防止频繁查找导致占用全局互斥锁的问题，我们尝试直接从磁盘返回hls索引文件 response_file(cookie, cb, file_path, parser); return; } @@ -555,11 +561,14 @@ static void accessFile(Session &sender, const Parser &parser, const MediaInfo &m auto &attach = cookie->getAttach(); attach._hls_data->setMediaSource(hls); - //添加HlsMediaSource的观看人数(HLS是按需生成的，这样可以触发HLS文件的生成) + // 添加HlsMediaSource的观看人数(HLS是按需生成的，这样可以触发HLS文件的生成) attach._hls_data->addByteUsage(0); - //标记找到MediaSource + // 标记找到MediaSource attach._find_src = true; + // 重置查找MediaSource计时 + attach._find_src_ticker.resetTime(); + // m3u8文件可能不存在, 等待m3u8索引文件按需生成 hls->getIndexFile([response_file, file_path, cookie, cb, parser](const string &file) { response_file(cookie, cb, file_path, parser, file); diff --git a/src/Record/MP4Demuxer.cpp b/src/Record/MP4Demuxer.cpp index 02c66976..87936137 100644 --- a/src/Record/MP4Demuxer.cpp +++ b/src/Record/MP4Demuxer.cpp @@ -16,8 +16,10 @@ #include "Extension/AAC.h" #include "Extension/G711.h" #include "Extension/Opus.h" -using namespace toolkit; +#include "Extension/JPEG.h" + using namespace std; +using namespace toolkit; namespace mediakit { @@ -105,8 +107,9 @@ void MP4Demuxer::onVideoTrack(uint32_t track, uint8_t object, int width, int hei video->inputFrame(std::make_shared((char *)config, size, 0, 0,4)); } } - } break; + } + case MOV_OBJECT_HEVC: { auto video = std::make_shared(); _track_to_codec.emplace(track,video); @@ -120,11 +123,16 @@ void MP4Demuxer::onVideoTrack(uint32_t track, uint8_t object, int width, int hei video->inputFrame(std::make_shared((char *) config, size, 0, 0,4)); } } + break; } + + case MOV_OBJECT_JPEG: { + auto video = std::make_shared(); + _track_to_codec.emplace(track,video); break; - default: - WarnL << "不支持该编码类型的MP4,已忽略:" << getObjectName(object); - break; + } + + default: WarnL << "不支持该编码类型的MP4,已忽略:" << getObjectName(object); break; } } @@ -243,6 +251,11 @@ Frame::Ptr MP4Demuxer::makeFrame(uint32_t track_id, const Buffer::Ptr &buf, int6 break; } + case CodecJPEG: { + ret = std::make_shared(buf, (uint64_t)dts, 0, DATA_OFFSET); + break; + } + case CodecAAC: { AACTrack::Ptr track = dynamic_pointer_cast(it->second); assert(track); @@ -283,4 +296,4 @@ uint64_t MP4Demuxer::getDurationMS() const { } }//namespace mediakit -#endif// ENABLE_MP4 \ No newline at end of file +#endif// ENABLE_MP4