diff --git a/src/Http/HttpBody.cpp b/src/Http/HttpBody.cpp index 922d4811..a7a83e88 100644 --- a/src/Http/HttpBody.cpp +++ b/src/Http/HttpBody.cpp @@ -9,16 +9,23 @@ */ #include -#include "HttpBody.h" -#include "Util/util.h" -#include "Util/File.h" -#include "Util/uv_errno.h" -#include "Util/logger.h" -#include "Util/onceToken.h" -#include "HttpClient.h" + #ifndef _WIN32 #include #endif +#if defined(__linux__) || defined(__linux) +#include +#endif + +#include "Util/File.h" +#include "Util/logger.h" +#include "Util/onceToken.h" +#include "Util/util.h" +#include "Util/uv_errno.h" + +#include "HttpBody.h" +#include "HttpClient.h" +#include "Common/macros.h" #ifndef _WIN32 #define ENABLE_MMAP @@ -29,7 +36,7 @@ using namespace toolkit; namespace mediakit { -HttpStringBody::HttpStringBody(string str){ +HttpStringBody::HttpStringBody(string str) { _str = std::move(str); } @@ -39,84 +46,92 @@ ssize_t HttpStringBody::remainSize() { Buffer::Ptr HttpStringBody::readData(size_t size) { size = MIN((size_t)remainSize(), size); - if(!size){ + if (!size) { //没有剩余字节了 return nullptr; } - auto ret = std::make_shared(_str,_offset,size); + auto ret = std::make_shared(_str, _offset, size); _offset += size; return ret; } ////////////////////////////////////////////////////////////////// -HttpFileBody::HttpFileBody(const string &filePath, bool use_mmap) { - std::shared_ptr fp(fopen(filePath.data(), "rb"), [](FILE *fp) { +#ifdef ENABLE_MMAP +static std::shared_ptr getSharedMmap(const string &file_path, const std::shared_ptr &fp, uint64_t max_size) { + static mutex s_mtx; + static unordered_map /*mmap*/> s_shared_mmap; + + { + lock_guard lck(s_mtx); + auto it = s_shared_mmap.find(file_path); + if (it != s_shared_mmap.end()) { + auto ret = it->second.lock(); + if (ret) { + //命中mmap缓存 + return ret; + } + } + } + + int fd = fileno(fp.get()); + if (fd < 0) { + WarnL << "fileno failed:" << get_uv_errmsg(false); + return nullptr; + } + auto ptr = (char *)mmap(NULL, max_size, PROT_READ, MAP_SHARED, fd, 0); + if (ptr == MAP_FAILED) { + WarnL << "mmap " << file_path << " failed:" << get_uv_errmsg(false); + return nullptr; + } + std::shared_ptr ret(ptr, [max_size, fp](char *ptr) { munmap(ptr, max_size); }); + { + lock_guard lck(s_mtx); + s_shared_mmap[file_path] = ret; + } + return ret; +} +#endif + +HttpFileBody::HttpFileBody(const string &file_path, bool use_mmap) { + _fp.reset(fopen(file_path.data(), "rb"), [](FILE *fp) { if (fp) { fclose(fp); } }); - if (!fp) { - init(fp, 0, 0, use_mmap); - } else { - init(fp, 0, File::fileSize(fp.get()), use_mmap); + if (!_fp) { + //文件不存在 + _read_to = -1; + return; + } + _read_to = File::fileSize(_fp.get()); +#ifdef ENABLE_MMAP + if (use_mmap && _read_to) { + _map_addr = getSharedMmap(file_path, _fp, _read_to); + } +#endif +} + +void HttpFileBody::setRange(uint64_t offset, uint64_t max_size) { + CHECK(offset <= _read_to && max_size + offset <= _read_to); + _read_to = max_size + offset; + _file_offset = offset; + if (_fp && !_map_addr) { + fseek64(_fp.get(), _file_offset, SEEK_SET); } } -HttpFileBody::HttpFileBody(const std::shared_ptr &fp, size_t offset, size_t max_size, bool use_mmap) { - init(fp, offset, max_size, use_mmap); -} - -#if defined(__linux__) || defined(__linux) -#include -#endif - int HttpFileBody::sendFile(int fd) { #if defined(__linux__) || defined(__linux) - static onceToken s_token([]() { - signal(SIGPIPE, SIG_IGN); - }); + static onceToken s_token([]() { signal(SIGPIPE, SIG_IGN); }); off_t off = _file_offset; - return sendfile(fd, fileno(_fp.get()), &off, _max_size); + return sendfile(fd, fileno(_fp.get()), &off, _read_to - _file_offset); #else return -1; #endif } -void HttpFileBody::init(const std::shared_ptr &fp, size_t offset, size_t max_size, bool use_mmap) { - _fp = fp; - _max_size = max_size; - _file_offset = offset; -#ifdef ENABLE_MMAP - if (use_mmap) { - do { - if (!_fp) { - //文件不存在 - break; - } - int fd = fileno(fp.get()); - if (fd < 0) { - WarnL << "fileno failed:" << get_uv_errmsg(false); - break; - } - auto ptr = (char *) mmap(NULL, max_size, PROT_READ, MAP_SHARED, fd, offset); - if (ptr == MAP_FAILED) { - WarnL << "mmap failed:" << get_uv_errmsg(false); - break; - } - _map_addr.reset(ptr, [max_size, fp](char *ptr) { - munmap(ptr, max_size); - }); - } while (false); - } -#endif - if (!_map_addr && offset && fp.get()) { - //未映射,那么fseek设置偏移量 - fseek64(fp.get(), offset, SEEK_SET); - } -} - -class BufferMmap : public Buffer{ +class BufferMmap : public Buffer { public: typedef std::shared_ptr Ptr; BufferMmap(const std::shared_ptr &map_addr, size_t offset, size_t size) { @@ -124,103 +139,96 @@ public: _data = map_addr.get() + offset; _size = size; } - ~BufferMmap() override{}; + ~BufferMmap() override {}; //返回数据长度 - char *data() const override { - return _data; - } - size_t size() const override{ - return _size; - } + char *data() const override { return _data; } + size_t size() const override { return _size; } + private: - std::shared_ptr _map_addr; char *_data; size_t _size; + std::shared_ptr _map_addr; }; ssize_t HttpFileBody::remainSize() { - return _max_size - _offset; + return _read_to - _file_offset; } Buffer::Ptr HttpFileBody::readData(size_t size) { - size = MIN((size_t)remainSize(),size); - if(!size){ + size = MIN((size_t)remainSize(), size); + if (!size) { //没有剩余字节了 return nullptr; } - if(!_map_addr){ - //fread模式 + if (!_map_addr) { + // fread模式 ssize_t iRead; auto ret = _pool.obtain2(); ret->setCapacity(size + 1); - do{ + do { iRead = fread(ret->data(), 1, size, _fp.get()); - }while(-1 == iRead && UV_EINTR == get_uv_error(false)); + } while (-1 == iRead && UV_EINTR == get_uv_error(false)); - if(iRead > 0){ + if (iRead > 0) { //读到数据了 ret->setSize(iRead); - _offset += iRead; + _file_offset += iRead; return std::move(ret); } //读取文件异常，文件真实长度小于声明长度 - _offset = _max_size; + _file_offset = _read_to; WarnL << "read file err:" << get_uv_errmsg(); return nullptr; } - //mmap模式 - auto ret = std::make_shared(_map_addr,_offset,size); - _offset += size; + // mmap模式 + auto ret = std::make_shared(_map_addr, _file_offset, size); + _file_offset += size; return ret; } ////////////////////////////////////////////////////////////////// -HttpMultiFormBody::HttpMultiFormBody(const HttpArgs &args,const string &filePath,const string &boundary){ - std::shared_ptr fp(fopen(filePath.data(), "rb"), [](FILE *fp) { - if(fp){ - fclose(fp); - } - }); - if(!fp){ + +HttpMultiFormBody::HttpMultiFormBody(const HttpArgs &args, const string &filePath, const string &boundary) { + _fileBody = std::make_shared(filePath); + if (_fileBody->remainSize() < 0) { throw std::invalid_argument(StrPrinter << "open file failed：" << filePath << " " << get_uv_errmsg()); } - _fileBody = std::make_shared(fp, 0, File::fileSize(fp.get())); auto fileName = filePath; auto pos = filePath.rfind('/'); - if(pos != string::npos){ + if (pos != string::npos) { fileName = filePath.substr(pos + 1); } - _bodyPrefix = multiFormBodyPrefix(args,boundary,fileName); + _bodyPrefix = multiFormBodyPrefix(args, boundary, fileName); _bodySuffix = multiFormBodySuffix(boundary); - _totalSize = _bodyPrefix.size() + _bodySuffix.size() + _fileBody->remainSize(); + _totalSize = _bodyPrefix.size() + _bodySuffix.size() + _fileBody->remainSize(); } ssize_t HttpMultiFormBody::remainSize() { return _totalSize - _offset; } -Buffer::Ptr HttpMultiFormBody::readData(size_t size){ - if(_bodyPrefix.size()){ +Buffer::Ptr HttpMultiFormBody::readData(size_t size) { + if (_bodyPrefix.size()) { auto ret = std::make_shared(_bodyPrefix); _offset += _bodyPrefix.size(); _bodyPrefix.clear(); return ret; } - if(_fileBody->remainSize()){ + if (_fileBody->remainSize()) { auto ret = _fileBody->readData(size); - if(!ret){ + if (!ret) { //读取文件出现异常，提前中断 _offset = _totalSize; - }else{ + } else { _offset += ret->size(); } return ret; } - if(_bodySuffix.size()){ + if (_bodySuffix.size()) { auto ret = std::make_shared(_bodySuffix); _offset = _totalSize; _bodySuffix.clear(); @@ -230,7 +238,7 @@ Buffer::Ptr HttpMultiFormBody::readData(size_t size){ return nullptr; } -string HttpMultiFormBody::multiFormBodySuffix(const string &boundary){ +string HttpMultiFormBody::multiFormBodySuffix(const string &boundary) { string MPboundary = string("--") + boundary; string endMPboundary = MPboundary + "--"; _StrPrinter body; @@ -238,21 +246,23 @@ string HttpMultiFormBody::multiFormBodySuffix(const string &boundary){ return std::move(body); } -string HttpMultiFormBody::multiFormContentType(const string &boundary){ +string HttpMultiFormBody::multiFormContentType(const string &boundary) { return StrPrinter << "multipart/form-data; boundary=" << boundary; } -string HttpMultiFormBody::multiFormBodyPrefix(const HttpArgs &args,const string &boundary,const string &fileName){ +string HttpMultiFormBody::multiFormBodyPrefix(const HttpArgs &args, const string &boundary, const string &fileName) { string MPboundary = string("--") + boundary; _StrPrinter body; - for(auto &pr : args){ + for (auto &pr : args) { body << MPboundary << "\r\n"; body << "Content-Disposition: form-data; name=\"" << pr.first << "\"\r\n\r\n"; body << pr.second << "\r\n"; } body << MPboundary << "\r\n"; - body << "Content-Disposition: form-data; name=\"" << "file" << "\";filename=\"" << fileName << "\"\r\n"; - body << "Content-Type: application/octet-stream\r\n\r\n" ; + body << "Content-Disposition: form-data; name=\"" + << "file" + << "\";filename=\"" << fileName << "\"\r\n"; + body << "Content-Type: application/octet-stream\r\n\r\n"; return std::move(body); } @@ -268,4 +278,4 @@ Buffer::Ptr HttpBufferBody::readData(size_t size) { return Buffer::Ptr(std::move(_buffer)); } -}//namespace mediakit +} // namespace mediakit diff --git a/src/Http/HttpBody.h b/src/Http/HttpBody.h index 319bc1bb..1f9e21eb 100644 --- a/src/Http/HttpBody.h +++ b/src/Http/HttpBody.h @@ -104,32 +104,32 @@ private: /** * 文件类型的content */ -class HttpFileBody : public HttpBody{ +class HttpFileBody : public HttpBody { public: typedef std::shared_ptr Ptr; /** * 构造函数 - * @param fp 文件句柄，文件的偏移量必须为0 - * @param offset 相对文件头的偏移量 - * @param max_size 最大读取字节数，未判断是否大于文件真实大小 + * @param file_path 文件路径 * @param use_mmap 是否使用mmap方式访问文件 */ - HttpFileBody(const std::shared_ptr &fp, size_t offset, size_t max_size, bool use_mmap = true); HttpFileBody(const std::string &file_path, bool use_mmap = true); ~HttpFileBody() override = default; - ssize_t remainSize() override ; + /** + * 设置读取范围 + * @param offset 相对文件头的偏移量 + * @param max_size 最大读取字节数 + */ + void setRange(uint64_t offset, uint64_t max_size); + + ssize_t remainSize() override; toolkit::Buffer::Ptr readData(size_t size) override; int sendFile(int fd) override; private: - void init(const std::shared_ptr &fp,size_t offset,size_t max_size, bool use_mmap); - -private: - size_t _max_size; - size_t _offset = 0; - size_t _file_offset = 0; + int64_t _read_to = 0; + uint64_t _file_offset = 0; std::shared_ptr _fp; std::shared_ptr _map_addr; toolkit::ResourcePool _pool; diff --git a/src/Http/HttpFileManager.cpp b/src/Http/HttpFileManager.cpp index 30a70f4b..4b6aaa6b 100644 --- a/src/Http/HttpFileManager.cpp +++ b/src/Http/HttpFileManager.cpp @@ -582,13 +582,8 @@ void HttpResponseInvokerImp::responseFile(const StrCaseMap &requestHeader, const string &filePath, bool use_mmap) const { StrCaseMap &httpHeader = const_cast(responseHeader); - std::shared_ptr fp(fopen(filePath.data(), "rb"), [](FILE *fp) { - if (fp) { - fclose(fp); - } - }); - - if (!fp) { + auto fileBody = std::make_shared(filePath, use_mmap); + if (fileBody->remainSize() < 0) { //打开文件失败 GET_CONFIG(string, notFound, Http::kNotFound); GET_CONFIG(string, charSet, Http::kCharSet); @@ -600,29 +595,23 @@ void HttpResponseInvokerImp::responseFile(const StrCaseMap &requestHeader, } auto &strRange = const_cast(requestHeader)["Range"]; - size_t iRangeStart = 0; - size_t iRangeEnd = 0; - size_t fileSize = File::fileSize(fp.get()); - - int code; - if (strRange.size() == 0) { - //全部下载 - code = 200; - iRangeEnd = fileSize - 1; - } else { + int code = 200; + if (!strRange.empty()) { //分节下载 code = 206; - iRangeStart = atoll(FindField(strRange.data(), "bytes=", "-").data()); - iRangeEnd = atoll(FindField(strRange.data(), "-", nullptr).data()); + auto iRangeStart = atoll(FindField(strRange.data(), "bytes=", "-").data()); + auto iRangeEnd = atoll(FindField(strRange.data(), "-", nullptr).data()); + auto fileSize = fileBody->remainSize(); if (iRangeEnd == 0) { iRangeEnd = fileSize - 1; } + //设置文件范围 + fileBody->setRange(iRangeStart, iRangeEnd - iRangeStart + 1); //分节下载返回Content-Range头 httpHeader.emplace("Content-Range", StrPrinter << "bytes " << iRangeStart << "-" << iRangeEnd << "/" << fileSize << endl); } //回复文件 - HttpBody::Ptr fileBody = std::make_shared(fp, iRangeStart, iRangeEnd - iRangeStart + 1, use_mmap); (*this)(code, httpHeader, fileBody); }