http文件服务器mmap方案采用共享方式

src/Http/HttpBody.cpp

@@ -9,16 +9,23 @@
  */
 
 #include <csignal>
-#include "HttpBody.h"
+
-#include "Util/util.h"
-#include "Util/File.h"
-#include "Util/uv_errno.h"
-#include "Util/logger.h"
-#include "Util/onceToken.h"
-#include "HttpClient.h"
 #ifndef _WIN32
 #include <sys/mman.h>
 #endif
+#if defined(__linux__) || defined(__linux)
+#include <sys/sendfile.h>
+#endif
+
+#include "Util/File.h"
+#include "Util/logger.h"
+#include "Util/onceToken.h"
+#include "Util/util.h"
+#include "Util/uv_errno.h"
+
+#include "HttpBody.h"
+#include "HttpClient.h"
+#include "Common/macros.h"
 
 #ifndef _WIN32
 #define ENABLE_MMAP
@@ -50,72 +57,80 @@ Buffer::Ptr HttpStringBody::readData(size_t size) {
 
 //////////////////////////////////////////////////////////////////
 
-HttpFileBody::HttpFileBody(const string &filePath, bool use_mmap) {
+#ifdef ENABLE_MMAP
-    std::shared_ptr<FILE> fp(fopen(filePath.data(), "rb"), [](FILE *fp) {
+static std::shared_ptr<char> getSharedMmap(const string &file_path, const std::shared_ptr<FILE> &fp, uint64_t max_size) {
+    static mutex s_mtx;
+    static unordered_map<string /*file_path*/, weak_ptr<char> /*mmap*/> s_shared_mmap;
+
+    {
+        lock_guard<mutex> lck(s_mtx);
+        auto it = s_shared_mmap.find(file_path);
+        if (it != s_shared_mmap.end()) {
+            auto ret = it->second.lock();
+            if (ret) {
+                //命中mmap缓存
+                return ret;
+            }
+        }
+    }
+
+    int fd = fileno(fp.get());
+    if (fd < 0) {
+        WarnL << "fileno failed:" << get_uv_errmsg(false);
+        return nullptr;
+    }
+    auto ptr = (char *)mmap(NULL, max_size, PROT_READ, MAP_SHARED, fd, 0);
+    if (ptr == MAP_FAILED) {
+        WarnL << "mmap " << file_path << " failed:" << get_uv_errmsg(false);
+        return nullptr;
+    }
+    std::shared_ptr<char> ret(ptr, [max_size, fp](char *ptr) { munmap(ptr, max_size); });
+    {
+        lock_guard<mutex> lck(s_mtx);
+        s_shared_mmap[file_path] = ret;
+    }
+    return ret;
+}
+#endif
+
+HttpFileBody::HttpFileBody(const string &file_path, bool use_mmap) {
+    _fp.reset(fopen(file_path.data(), "rb"), [](FILE *fp) {
         if (fp) {
             fclose(fp);
         }
     });
-    if (!fp) {
+    if (!_fp) {
-        init(fp, 0, 0, use_mmap);
+        //文件不存在
-    } else {
+        _read_to = -1;
-        init(fp, 0, File::fileSize(fp.get()), use_mmap);
+        return;
     }
+    _read_to = File::fileSize(_fp.get());
+#ifdef ENABLE_MMAP
+    if (use_mmap && _read_to) {
+        _map_addr = getSharedMmap(file_path, _fp, _read_to);
     }
-
-HttpFileBody::HttpFileBody(const std::shared_ptr<FILE> &fp, size_t offset, size_t max_size, bool use_mmap) {
-    init(fp, offset, max_size, use_mmap);
-}
-
-#if defined(__linux__) || defined(__linux)
-#include <sys/sendfile.h>
 #endif
+}
+
+void HttpFileBody::setRange(uint64_t offset, uint64_t max_size) {
+    CHECK(offset <= _read_to && max_size + offset <= _read_to);
+    _read_to = max_size + offset;
+    _file_offset = offset;
+    if (_fp && !_map_addr) {
+        fseek64(_fp.get(), _file_offset, SEEK_SET);
+    }
+}
 
 int HttpFileBody::sendFile(int fd) {
 #if defined(__linux__) || defined(__linux)
-    static onceToken s_token([]() {
+    static onceToken s_token([]() { signal(SIGPIPE, SIG_IGN); });
-        signal(SIGPIPE, SIG_IGN);
-    });
     off_t off = _file_offset;
-    return sendfile(fd, fileno(_fp.get()), &off, _max_size);
+    return sendfile(fd, fileno(_fp.get()), &off, _read_to - _file_offset);
 #else
     return -1;
 #endif
 }
 
-void HttpFileBody::init(const std::shared_ptr<FILE> &fp, size_t offset, size_t max_size, bool use_mmap) {
-    _fp = fp;
-    _max_size = max_size;
-    _file_offset = offset;
-#ifdef ENABLE_MMAP
-    if (use_mmap) {
-        do {
-            if (!_fp) {
-                //文件不存在
-                break;
-            }
-            int fd = fileno(fp.get());
-            if (fd < 0) {
-                WarnL << "fileno failed:" << get_uv_errmsg(false);
-                break;
-            }
-            auto ptr = (char *) mmap(NULL, max_size, PROT_READ, MAP_SHARED, fd, offset);
-            if (ptr == MAP_FAILED) {
-                WarnL << "mmap failed:" << get_uv_errmsg(false);
-                break;
-            }
-            _map_addr.reset(ptr, [max_size, fp](char *ptr) {
-                munmap(ptr, max_size);
-            });
-        } while (false);
-    }
-#endif
-    if (!_map_addr && offset && fp.get()) {
-        //未映射,那么fseek设置偏移量
-        fseek64(fp.get(), offset, SEEK_SET);
-    }
-}
-
 class BufferMmap : public Buffer {
 public:
     typedef std::shared_ptr<BufferMmap> Ptr;
@@ -126,20 +141,17 @@ public:
     }
     ~BufferMmap() override {};
     //返回数据长度
-    char *data() const override {
+    char *data() const override { return _data; }
-        return _data;
+    size_t size() const override { return _size; }
-    }
+
-    size_t size() const override{
-        return _size;
-    }
 private:
-    std::shared_ptr<char> _map_addr;
     char *_data;
     size_t _size;
+    std::shared_ptr<char> _map_addr;
 };
 
 ssize_t HttpFileBody::remainSize() {
-    return _max_size - _offset;
+    return _read_to - _file_offset;
 }
 
 Buffer::Ptr HttpFileBody::readData(size_t size) {
@@ -160,32 +172,28 @@ Buffer::Ptr HttpFileBody::readData(size_t size) {
         if (iRead > 0) {
             //读到数据了
             ret->setSize(iRead);
-            _offset += iRead;
+            _file_offset += iRead;
             return std::move(ret);
         }
         //读取文件异常，文件真实长度小于声明长度
-        _offset = _max_size;
+        _file_offset = _read_to;
         WarnL << "read file err:" << get_uv_errmsg();
         return nullptr;
     }
 
     // mmap模式
-    auto ret = std::make_shared<BufferMmap>(_map_addr,_offset,size);
+    auto ret = std::make_shared<BufferMmap>(_map_addr, _file_offset, size);
-    _offset += size;
+    _file_offset += size;
     return ret;
 }
 
 //////////////////////////////////////////////////////////////////
+
 HttpMultiFormBody::HttpMultiFormBody(const HttpArgs &args, const string &filePath, const string &boundary) {
-    std::shared_ptr<FILE> fp(fopen(filePath.data(), "rb"), [](FILE *fp) {
+    _fileBody = std::make_shared<HttpFileBody>(filePath);
-        if(fp){
+    if (_fileBody->remainSize() < 0) {
-            fclose(fp);
-        }
-    });
-    if(!fp){
         throw std::invalid_argument(StrPrinter << "open file failed：" << filePath << " " << get_uv_errmsg());
     }
-    _fileBody = std::make_shared<HttpFileBody>(fp, 0, File::fileSize(fp.get()));
 
     auto fileName = filePath;
     auto pos = filePath.rfind('/');
@@ -251,7 +259,9 @@ string HttpMultiFormBody::multiFormBodyPrefix(const HttpArgs &args,const string
         body << pr.second << "\r\n";
     }
     body << MPboundary << "\r\n";
-    body << "Content-Disposition: form-data; name=\"" << "file" << "\";filename=\"" << fileName << "\"\r\n";
+    body << "Content-Disposition: form-data; name=\""
+         << "file"
+         << "\";filename=\"" << fileName << "\"\r\n";
     body << "Content-Type: application/octet-stream\r\n\r\n";
     return std::move(body);
 }

src/Http/HttpBody.h

@@ -110,26 +110,26 @@ public:
 
     /**
      * 构造函数
-     * @param fp 文件句柄，文件的偏移量必须为0
+     * @param file_path 文件路径
-     * @param offset 相对文件头的偏移量
-     * @param max_size 最大读取字节数，未判断是否大于文件真实大小
      * @param use_mmap 是否使用mmap方式访问文件
      */
-    HttpFileBody(const std::shared_ptr<FILE> &fp, size_t offset, size_t max_size, bool use_mmap = true);
     HttpFileBody(const std::string &file_path, bool use_mmap = true);
     ~HttpFileBody() override = default;
 
+    /**
+     * 设置读取范围
+     * @param offset 相对文件头的偏移量
+     * @param max_size 最大读取字节数
+     */
+    void setRange(uint64_t offset, uint64_t max_size);
+
     ssize_t remainSize() override;
     toolkit::Buffer::Ptr readData(size_t size) override;
     int sendFile(int fd) override;
 
 private:
-    void init(const std::shared_ptr<FILE> &fp,size_t offset,size_t max_size, bool use_mmap);
+    int64_t _read_to = 0;
-
+    uint64_t _file_offset = 0;
-private:
-    size_t _max_size;
-    size_t _offset = 0;
-    size_t _file_offset = 0;
     std::shared_ptr<FILE> _fp;
     std::shared_ptr<char> _map_addr;
     toolkit::ResourcePool<toolkit::BufferRaw> _pool;

src/Http/HttpFileManager.cpp

@@ -582,13 +582,8 @@ void HttpResponseInvokerImp::responseFile(const StrCaseMap &requestHeader,
                                           const string &filePath,
                                           bool use_mmap) const {
     StrCaseMap &httpHeader = const_cast<StrCaseMap &>(responseHeader);
-    std::shared_ptr<FILE> fp(fopen(filePath.data(), "rb"), [](FILE *fp) {
+    auto fileBody = std::make_shared<HttpFileBody>(filePath, use_mmap);
-        if (fp) {
+    if (fileBody->remainSize() < 0) {
-            fclose(fp);
-        }
-    });
-
-    if (!fp) {
         //打开文件失败
         GET_CONFIG(string, notFound, Http::kNotFound);
         GET_CONFIG(string, charSet, Http::kCharSet);
@@ -600,29 +595,23 @@ void HttpResponseInvokerImp::responseFile(const StrCaseMap &requestHeader,
     }
 
     auto &strRange = const_cast<StrCaseMap &>(requestHeader)["Range"];
-    size_t iRangeStart = 0;
+    int code = 200;
-    size_t iRangeEnd = 0;
+    if (!strRange.empty()) {
-    size_t fileSize = File::fileSize(fp.get());
-
-    int code;
-    if (strRange.size() == 0) {
-        //全部下载
-        code = 200;
-        iRangeEnd = fileSize - 1;
-    } else {
         //分节下载
         code = 206;
-        iRangeStart = atoll(FindField(strRange.data(), "bytes=", "-").data());
+        auto iRangeStart = atoll(FindField(strRange.data(), "bytes=", "-").data());
-        iRangeEnd = atoll(FindField(strRange.data(), "-", nullptr).data());
+        auto iRangeEnd = atoll(FindField(strRange.data(), "-", nullptr).data());
+        auto fileSize = fileBody->remainSize();
         if (iRangeEnd == 0) {
             iRangeEnd = fileSize - 1;
         }
+        //设置文件范围
+        fileBody->setRange(iRangeStart, iRangeEnd - iRangeStart + 1);
         //分节下载返回Content-Range头
         httpHeader.emplace("Content-Range", StrPrinter << "bytes " << iRangeStart << "-" << iRangeEnd << "/" << fileSize << endl);
     }
 
     //回复文件
-    HttpBody::Ptr fileBody = std::make_shared<HttpFileBody>(fp, iRangeStart, iRangeEnd - iRangeStart + 1, use_mmap);
     (*this)(code, httpHeader, fileBody);
 }