Merge branch 'ZLMediaKit:master' into master

CMakeLists.txt

@@ -39,6 +39,7 @@ option(ENABLE_FAAC "Enable FAAC" OFF)
 option(ENABLE_FFMPEG "Enable FFmpeg" OFF)
 option(ENABLE_HLS "Enable HLS" ON)
 option(ENABLE_JEMALLOC_STATIC "Enable static linking to the jemalloc library" OFF)
+option(ENABLE_JEMALLOC_DUMP "Enable jemalloc to dump malloc statistics" OFF)
 option(ENABLE_MEM_DEBUG "Enable Memory Debug" OFF)
 option(ENABLE_MP4 "Enable MP4" ON)
 option(ENABLE_HLS_FMP4 "Enable HLS-FMP4" ON)
@@ -335,7 +336,11 @@ if(ENABLE_JEMALLOC_STATIC)
   if(NOT EXISTS ${DEP_ROOT_DIR})
     file(MAKE_DIRECTORY ${DEP_ROOT_DIR})
   endif()
-
+  if (ENABLE_JEMALLOC_DUMP)
+    set(ENABLE_JEMALLOC_STAT ON)
+  else ()
+    set(ENABLE_JEMALLOC_STAT OFF)
+  endif ()
   include(Jemalloc)
   include_directories(SYSTEM ${DEP_ROOT_DIR}/${JEMALLOC_NAME}/include/jemalloc)
   link_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/lib)
@@ -349,6 +354,12 @@ if(JEMALLOC_FOUND)
   message(STATUS "found library: ${JEMALLOC_LIBRARIES}")
   include_directories(${JEMALLOC_INCLUDE_DIR})
   update_cached_list(MK_LINK_LIBRARIES ${JEMALLOC_LIBRARIES})
+  add_definitions(-DUSE_JEMALLOC)
+  message(STATUS "jemalloc will be used to avoid memory fragmentation")
+  if (ENABLE_JEMALLOC_DUMP)
+    add_definitions(-DENABLE_JEMALLOC_DUMP)
+    message(STATUS "jemalloc will save memory usage statistics when the program exits")
+  endif ()
 endif()
 
 # 查找 openssl 是否安装

cmake/Jemalloc.cmake

@@ -1,21 +1,34 @@
 # Download and build Jemalloc
 
-set(JEMALLOC_VERSION 5.2.1)
+set(JEMALLOC_VERSION 5.3.0)
 set(JEMALLOC_NAME jemalloc-${JEMALLOC_VERSION})
 set(JEMALLOC_TAR_PATH ${DEP_ROOT_DIR}/${JEMALLOC_NAME}.tar.bz2)
 
 list(APPEND jemalloc_CONFIG_ARGS --disable-initial-exec-tls)
-list(APPEND jemalloc_CONFIG_ARGS --without-export)
+#list(APPEND jemalloc_CONFIG_ARGS --without-export)
+if (ENABLE_JEMALLOC_STAT)
+    list(APPEND jemalloc_CONFIG_ARGS --enable-stats)
+    message(STATUS "Jemalloc stats enabled")
+else ()
 list(APPEND jemalloc_CONFIG_ARGS --disable-stats)
+    message(STATUS "Jemalloc stats disabled")
+endif ()
 list(APPEND jemalloc_CONFIG_ARGS --disable-libdl)
 #list(APPEND jemalloc_CONFIG_ARGS --disable-cxx)
 #list(APPEND jemalloc_CONFIG_ARGS --with-jemalloc-prefix=je_)
 #list(APPEND jemalloc_CONFIG_ARGS --enable-debug)
 
 if(NOT EXISTS ${JEMALLOC_TAR_PATH})
-    message(STATUS "Downloading ${JEMALLOC_NAME}...")
+    set(JEMALLOC_URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/${JEMALLOC_NAME}.tar.bz2)
-    file(DOWNLOAD https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/${JEMALLOC_NAME}.tar.bz2
+    message(STATUS "Downloading ${JEMALLOC_NAME} from ${JEMALLOC_URL}")
-            ${JEMALLOC_TAR_PATH})
+    file(DOWNLOAD ${JEMALLOC_URL} ${JEMALLOC_TAR_PATH} SHOW_PROGRESS STATUS JEMALLOC_DOWNLOAD_STATUS LOG JEMALLOC_DOWNLOAD_LOG)
+    list(GET JEMALLOC_DOWNLOAD_STATUS 0 JEMALLOC_DOWNLOAD_STATUS_CODE)
+    if(NOT JEMALLOC_DOWNLOAD_STATUS_CODE EQUAL 0)
+        file(REMOVE ${JEMALLOC_TAR_PATH})
+        message(STATUS "${JEMALLOC_DOWNLOAD_LOG}")
+        message(FATAL_ERROR "${JEMALLOC_NAME} download failed! error is ${JEMALLOC_DOWNLOAD_STATUS}")
+        return()
+    endif ()
 endif()
 
 SET( DIR_CONTAINING_JEMALLOC ${DEP_ROOT_DIR}/${JEMALLOC_NAME} )

server/System.cpp

@@ -22,10 +22,11 @@
 #include <map>
 #include <iostream>
 
+#include "Common/JemallocUtil.h"
+#include "Common/macros.h"
+#include "System.h"
 #include "Util/logger.h"
 #include "Util/uv_errno.h"
-#include "System.h"
-#include "Common/macros.h"
 
 using namespace std;
 using namespace toolkit;
@@ -55,6 +56,16 @@ string System::execute(const string &cmd) {
 
 static constexpr int MAX_STACK_FRAMES = 128;
 
+static void save_jemalloc_stats() {
+    string jemalloc_status = JemallocUtil::get_malloc_stats();
+    if (jemalloc_status.empty()) {
+        return;
+    }
+    ofstream out(StrPrinter << exeDir() << "/jemalloc.json", ios::out | ios::binary | ios::trunc);
+    out << jemalloc_status;
+    out.flush();
+}
+
 static void sig_crash(int sig) {
     signal(sig, SIG_DFL);
     void *array[MAX_STACK_FRAMES];
@@ -149,6 +160,12 @@ void System::startDaemon(bool &kill_parent_if_failed) {
 }
 
 void System::systemSetup(){
+
+#ifdef ENABLE_JEMALLOC_DUMP
+    //Save memory report when program exits
+    atexit(save_jemalloc_stats);
+#endif //ENABLE_JEMALLOC_DUMP
+
 #if !defined(_WIN32)
     struct rlimit rlim,rlim_new;
     if (getrlimit(RLIMIT_CORE, &rlim)==0) {

server/WebApi.cpp

@@ -1911,24 +1911,28 @@ void installWebApi() {
 void unInstallWebApi(){
     {
         lock_guard<recursive_mutex> lck(s_proxyMapMtx);
-        s_proxyMap.clear();
+        auto proxyMap(std::move(s_proxyMap));
+        proxyMap.clear();
     }
 
     {
         lock_guard<recursive_mutex> lck(s_ffmpegMapMtx);
-        s_ffmpegMap.clear();
+        auto ffmpegMap(std::move(s_ffmpegMap));
+        ffmpegMap.clear();
     }
 
     {
         lock_guard<recursive_mutex> lck(s_proxyPusherMapMtx);
-        s_proxyPusherMap.clear();
+        auto proxyPusherMap(std::move(s_proxyPusherMap));
+        proxyPusherMap.clear();
     }
 
     {
 #if defined(ENABLE_RTPPROXY)
         RtpSelector::Instance().clear();
         lock_guard<recursive_mutex> lck(s_rtpServerMapMtx);
-        s_rtpServerMap.clear();
+        auto rtpServerMap(std::move(s_rtpServerMap));
+        rtpServerMap.clear();
 #endif
     }
     NoticeCenter::Instance().delListener(&web_api_tag);

src/Common/JemallocUtil.cpp

@@ -0,0 +1,74 @@
+/*
+* Copyright (c) 2016 The ZLMediaKit project authors. All Rights Reserved.
+*
+* This file is part of ZLMediaKit(https://github.com/xia-chu/ZLMediaKit).
+*
+* Use of this source code is governed by MIT license that can be found in the
+* LICENSE file in the root of the source tree. All contributing project authors
+* may be found in the AUTHORS file in the root of the source tree.
+*/
+
+
+#include "JemallocUtil.h"
+#include "Util/logger.h"
+#ifdef USE_JEMALLOC
+#include <iostream>
+#include <jemalloc/jemalloc.h>
+#endif
+
+namespace mediakit {
+
+void set_profile_active(bool active) {
+#ifdef USE_JEMALLOC
+    int err = mallctl("prof.active", nullptr, nullptr, (void *)&active, sizeof(active));
+    if (err != 0) {
+        WarnL << "mallctl failed with: " << err;
+    }
+#endif
+}
+
+void JemallocUtil::enable_profiling() {
+    set_profile_active(true);
+}
+void JemallocUtil::disable_profiling() {
+    set_profile_active(false);
+}
+void JemallocUtil::dump(const std::string &file_name) {
+#ifdef USE_JEMALLOC
+    auto *c_str = file_name.c_str();
+    int err = mallctl("prof.dump", nullptr, nullptr, &c_str, sizeof(const char *));
+    if (err != 0) {
+        std::cerr << "mallctl failed with: " << err << std::endl;
+    }
+#endif
+}
+std::string JemallocUtil::get_malloc_stats() {
+#ifdef USE_JEMALLOC
+    std::string res;
+    malloc_stats_print([](void *opaque, const char *s) { ((std::string *)opaque)->append(s); }, &res, "J");
+    return res;
+#else
+    return "";
+#endif
+}
+
+void JemallocUtil::some_malloc_stats(const std::function<void(const char *, uint64_t)> &fn) {
+#ifdef USE_JEMALLOC
+    constexpr std::array<const char *, 8> STATS = {
+        "stats.allocated", "stats.active", "stats.metadata", "stats.metadata_thp",
+        "stats.resident",  "stats.mapped", "stats.retained", "stats.zero_reallocs",
+    };
+
+    for (const char *stat : STATS) {
+        size_t value;
+        size_t len = sizeof(value);
+        auto err = mallctl(stat, &value, &len, nullptr, 0);
+        if (err != 0) {
+            ErrorL << "Failed reading " << stat << ": " << err;
+            continue;
+        }
+        fn(stat, value);
+    }
+#endif
+}
+} // namespace mediakit

src/Common/JemallocUtil.h

@@ -0,0 +1,30 @@
+/*
+* Copyright (c) 2016 The ZLMediaKit project authors. All Rights Reserved.
+*
+* This file is part of ZLMediaKit(https://github.com/xia-chu/ZLMediaKit).
+*
+* Use of this source code is governed by MIT license that can be found in the
+* LICENSE file in the root of the source tree. All contributing project authors
+* may be found in the AUTHORS file in the root of the source tree.
+*/
+
+#ifndef ZLMEDIAKIT_JEMALLOCUTIL_H
+#define ZLMEDIAKIT_JEMALLOCUTIL_H
+#include <functional>
+#include <string>
+namespace mediakit {
+class JemallocUtil {
+public:
+    JemallocUtil() = default;
+    ~JemallocUtil() = default;
+
+    static void enable_profiling();
+
+    static void disable_profiling();
+
+    static void dump(const std::string &file_name);
+    static std::string get_malloc_stats();
+    static void some_malloc_stats(const std::function<void(const char *, uint64_t)> &fn);
+};
+} // namespace mediakit
+#endif // ZLMEDIAKIT_JEMALLOCUTIL_H

src/Common/Stamp.cpp

@@ -50,7 +50,7 @@ int64_t DeltaStamp::deltaStamp(int64_t stamp) {
     _last_stamp = stamp;
 
     // 如果时间戳回退不多，那么返回负值，否则返回加1
-    return -ret < MAX_CTS ? ret : 1;
+    return -ret < MAX_DELTA_STAMP ? ret : 1;
 }
 
 void Stamp::setPlayBack(bool playback) {

src/Http/HttpCookieManager.cpp

@@ -61,7 +61,7 @@ bool HttpServerCookie::isExpired() {
     return _ticker.elapsedTime() > _max_elapsed * 1000;
 }
 
-void HttpServerCookie::setAttach(std::shared_ptr<void> attach) {
+void HttpServerCookie::setAttach(toolkit::Any attach) {
     _attach = std::move(attach);
 }
 
@@ -114,8 +114,7 @@ void HttpCookieManager::onManager() {
     }
 }
 
-HttpServerCookie::Ptr HttpCookieManager::addCookie(const string &cookie_name, const string &uid_in,
+HttpServerCookie::Ptr HttpCookieManager::addCookie(const string &cookie_name, const string &uid_in, uint64_t max_elapsed, toolkit::Any attach, int max_client) {
-                                                   uint64_t max_elapsed, std::shared_ptr<void> attach, int max_client) {
     lock_guard<recursive_mutex> lck(_mtx_cookie);
     auto cookie = _generator.obtain();
     auto uid = uid_in.empty() ? cookie : uid_in;

src/Http/HttpCookieManager.h

@@ -85,14 +85,14 @@ public:
     /**
      * 设置附加数据
      */
-    void setAttach(std::shared_ptr<void> attach);
+    void setAttach(toolkit::Any attach);
 
     /*
      * 获取附加数据
      */
     template <class T>
     T& getAttach() {
-        return *static_cast<T *>(_attach.get());
+        return _attach.get<T>();
     }
 
 private:
@@ -104,7 +104,7 @@ private:
     std::string _cookie_uuid;
     uint64_t _max_elapsed;
     toolkit::Ticker _ticker;
-    std::shared_ptr<void> _attach;
+    toolkit::Any _attach;
     std::weak_ptr<HttpCookieManager> _manager;
 };
 
@@ -163,7 +163,7 @@ public:
      */
     HttpServerCookie::Ptr addCookie(
         const std::string &cookie_name, const std::string &uid, uint64_t max_elapsed = COOKIE_DEFAULT_LIFE,
-        std::shared_ptr<void> attach = nullptr,
+        toolkit::Any = toolkit::Any{},
         int max_client = 1);
 
     /**

src/Http/HttpFileManager.cpp

@@ -31,13 +31,16 @@ namespace mediakit {
 // 每次访问一次该cookie，那么将重新刷新cookie有效期
 // 假如播放器在60秒内都未访问该cookie，那么将重新触发hls播放鉴权
 static int kHlsCookieSecond = 60;
+static int kFindSrcIntervalSecond = 3;
 static const string kCookieName = "ZL_COOKIE";
 static const string kHlsSuffix = "/hls.m3u8";
 static const string kHlsFMP4Suffix = "/hls.fmp4.m3u8";
 
 struct HttpCookieAttachment {
-    //是否已经查找到过MediaSource
+    // 是否已经查找到过MediaSource
     bool _find_src = false;
+    // 查找MediaSource计时
+    Ticker _find_src_ticker;
     //cookie生效作用域，本cookie只对该目录下的文件生效
     string _path;
     //上次鉴权失败信息,为空则上次鉴权成功
@@ -414,7 +417,9 @@ static void canAccessPath(Session &sender, const Parser &parser, const MediaInfo
                 // hls相关信息
                 attach->_hls_data = std::make_shared<HlsCookieData>(media_info, info);
             }
-            callback(err_msg, HttpCookieManager::Instance().addCookie(kCookieName, uid, life_second, attach));
+           toolkit::Any any;
+           any.set(std::move(attach));
+           callback(err_msg, HttpCookieManager::Instance().addCookie(kCookieName, uid, life_second, std::move(any)));
         } else {
             callback(err_msg, nullptr);
         }
@@ -532,14 +537,15 @@ static void accessFile(Session &sender, const Parser &parser, const MediaInfo &m
             return;
         }
 
-        auto src = cookie->getAttach<HttpCookieAttachment>()._hls_data->getMediaSource();
+        auto &attach = cookie->getAttach<HttpCookieAttachment>();
+        auto src = attach._hls_data->getMediaSource();
         if (src) {
-            //直接从内存获取m3u8索引文件(而不是从文件系统)
+            // 直接从内存获取m3u8索引文件(而不是从文件系统)
             response_file(cookie, cb, file_path, parser, src->getIndexFile());
             return;
         }
-        if (cookie->getAttach<HttpCookieAttachment>()._find_src) {
+        if (attach._find_src && attach._find_src_ticker.elapsedTime() < kFindSrcIntervalSecond * 1000) {
-            //查找过MediaSource，但是流已经注销了，不用再查找
+            // 最近已经查找过MediaSource了，为了防止频繁查找导致占用全局互斥锁的问题，我们尝试直接从磁盘返回hls索引文件
             response_file(cookie, cb, file_path, parser);
             return;
         }
@@ -555,11 +561,14 @@ static void accessFile(Session &sender, const Parser &parser, const MediaInfo &m
 
             auto &attach = cookie->getAttach<HttpCookieAttachment>();
             attach._hls_data->setMediaSource(hls);
-            //添加HlsMediaSource的观看人数(HLS是按需生成的，这样可以触发HLS文件的生成)
+            // 添加HlsMediaSource的观看人数(HLS是按需生成的，这样可以触发HLS文件的生成)
             attach._hls_data->addByteUsage(0);
-            //标记找到MediaSource
+            // 标记找到MediaSource
             attach._find_src = true;
 
+            // 重置查找MediaSource计时
+            attach._find_src_ticker.resetTime();
+
             // m3u8文件可能不存在, 等待m3u8索引文件按需生成
             hls->getIndexFile([response_file, file_path, cookie, cb, parser](const string &file) {
                 response_file(cookie, cb, file_path, parser, file);

src/Record/MP4Demuxer.cpp

@@ -16,8 +16,10 @@
 #include "Extension/AAC.h"
 #include "Extension/G711.h"
 #include "Extension/Opus.h"
-using namespace toolkit;
+#include "Extension/JPEG.h"
+
 using namespace std;
+using namespace toolkit;
 
 namespace mediakit {
 
@@ -105,8 +107,9 @@ void MP4Demuxer::onVideoTrack(uint32_t track, uint8_t object, int width, int hei
                     video->inputFrame(std::make_shared<H264FrameNoCacheAble>((char *)config, size, 0, 0,4));
                 }
             }
-        }
             break;
+        }
+
         case MOV_OBJECT_HEVC: {
             auto video = std::make_shared<H265Track>();
             _track_to_codec.emplace(track,video);
@@ -120,11 +123,16 @@ void MP4Demuxer::onVideoTrack(uint32_t track, uint8_t object, int width, int hei
                     video->inputFrame(std::make_shared<H265FrameNoCacheAble>((char *) config, size, 0, 0,4));
                 }
             }
+            break;
         }
+
+        case MOV_OBJECT_JPEG: {
+            auto video = std::make_shared<JPEGTrack>();
+            _track_to_codec.emplace(track,video);
             break;
-        default:
+        }
-            WarnL << "不支持该编码类型的MP4,已忽略:" << getObjectName(object);
+
-            break;
+        default: WarnL << "不支持该编码类型的MP4,已忽略:" << getObjectName(object); break;
     }
 }
 
@@ -243,6 +251,11 @@ Frame::Ptr MP4Demuxer::makeFrame(uint32_t track_id, const Buffer::Ptr &buf, int6
             break;
         }
 
+        case CodecJPEG: {
+            ret = std::make_shared<JPEGFrame>(buf, (uint64_t)dts, 0, DATA_OFFSET);
+            break;
+        }
+
         case CodecAAC: {
             AACTrack::Ptr track = dynamic_pointer_cast<AACTrack>(it->second);
             assert(track);