From 486e184bfc4561e92ea6f81c0dbd7d91a5d6afe1 Mon Sep 17 00:00:00 2001 From: An Tao Date: Mon, 29 Mar 2021 16:29:33 +0800 Subject: [PATCH 001/182] Update AsyncFileLogger (#135) --- trantor/utils/AsyncFileLogger.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trantor/utils/AsyncFileLogger.cc b/trantor/utils/AsyncFileLogger.cc index ef8ff069..b5bb0800 100644 --- a/trantor/utils/AsyncFileLogger.cc +++ b/trantor/utils/AsyncFileLogger.cc @@ -56,7 +56,7 @@ AsyncFileLogger::~AsyncFileLogger() { writeBuffers_.push(logBufferPtr_); } - if (writeBuffers_.size() > 0) + while (!writeBuffers_.empty()) { StringPtr tmpPtr = (StringPtr &&) writeBuffers_.front(); writeBuffers_.pop(); @@ -150,7 +150,7 @@ void AsyncFileLogger::logThreadFunc() tmpBuffers_.swap(writeBuffers_); } - while (tmpBuffers_.size() > 0) + while (!tmpBuffers_.empty()) { StringPtr tmpPtr = (StringPtr &&) tmpBuffers_.front(); tmpBuffers_.pop(); From 295213374236584a202e28b56da7230103959da5 Mon Sep 17 00:00:00 2001 From: an-tao Date: Fri, 9 Apr 2021 14:28:48 +0800 Subject: [PATCH 002/182] Bump version to 1.4.0 --- ChangeLog.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/ChangeLog.md b/ChangeLog.md index bb80bc58..160484a6 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -3,6 +3,22 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [1.4.0] - 2021-04-09 + +### API changes list + +- Add isUnspecified() to indicate if IP parsing failed. + +- Add exports macro to allow Shared Library with hidden symbols by default. + +### Changed + +- Modify the AsyncFileLogger destructor. + +### Fixed + +- Recycle TimerID in the TimerQueue. + ## [1.3.0] - 2021-03-05 ### API changes list @@ -248,7 +264,9 @@ All notable changes to this project will be documented in this file. ## [1.0.0-rc1] - 2019-06-11 -[Unreleased]: https://github.com/an-tao/trantor/compare/v1.3.0...HEAD +[Unreleased]: https://github.com/an-tao/trantor/compare/v1.4.0...HEAD + +[1.4.0]: https://github.com/an-tao/trantor/compare/v1.3.0...v1.4.0 [1.3.0]: https://github.com/an-tao/trantor/compare/v1.2.0...v1.3.0 From a3cb939ef237b83b5bd8d9c87f1dec6899c54054 Mon Sep 17 00:00:00 2001 From: An Tao Date: Sat, 10 Apr 2021 18:35:27 +0800 Subject: [PATCH 003/182] Fix a bug in the TcpConnectionImpl class (#136) --- trantor/net/inner/TcpConnectionImpl.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index ffc540b3..f3235b54 100644 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -710,7 +710,6 @@ void TcpConnectionImpl::writeCallback() writeBufferList_.front()->msgBuffer_->peek(), writeBufferList_.front() ->msgBuffer_->readableBytes()); - writeBufferList_.front()->msgBuffer_->retrieve(n); if (n >= 0) { writeBufferList_.front()->msgBuffer_->retrieve( From 381263bbc46cd5a592e64d15b474adce78f9f99e Mon Sep 17 00:00:00 2001 From: An Tao Date: Tue, 20 Apr 2021 00:42:44 +0800 Subject: [PATCH 004/182] Add github actions of Windows (#139) Co-authored-by: Bertrand Darbon --- .github/workflows/cmake.yml | 44 +++++++++++++++++++++++++++++++++++-- conanfile.txt | 11 ++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 conanfile.txt diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 3f21882d..7ad9753b 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -11,7 +11,47 @@ env: BUILD_TYPE: Release jobs: - build: + windows: + name: 'windows/msvc - ${{matrix.link}}' + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + link: [ 'STATIC', 'SHARED' ] + steps: + - name: Checkout Trantor source code + uses: actions/checkout@v2 + with: + submodules: false + + - name: Install dependencies + run: | + pip install conan + + - name: Create build directory + run: | + mkdir build + + - name: Install conan packages + shell: pwsh + working-directory: ./build + run: | + conan install .. -s compiler="Visual Studio" -s compiler.version=16 -sbuild_type=Debug -g cmake_paths + + - name: Create Build Environment & Configure Cmake + shell: bash + working-directory: ./build + run: | + [[ ${{ matrix.link }} == "SHARED" ]] && shared="ON" || shared="OFF" + cmake .. -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=on -DBUILD_TRANTOR_SHARED=$shared -DCMAKE_TOOLCHAIN_FILE="conan_paths.cmake" -DCMAKE_INSTALL_PREFIX=../install + + - name: Build + working-directory: ${{env.GITHUB_WORKSPACE}} + shell: bash + run: | + cd build + cmake --build . --target install --parallel + unix: name: ${{matrix.buildname}} runs-on: ${{matrix.os}} strategy: @@ -92,4 +132,4 @@ jobs: if: matrix.os == 'ubuntu-20.04' working-directory: ${{env.GITHUB_WORKSPACE}} shell: bash - run: ./format.sh && git diff --exit-code \ No newline at end of file + run: ./format.sh && git diff --exit-code diff --git a/conanfile.txt b/conanfile.txt new file mode 100644 index 00000000..59906b3a --- /dev/null +++ b/conanfile.txt @@ -0,0 +1,11 @@ +[requires] +gtest/1.10.0 +openssl/1.1.1j +#c-ares/1.17.1 + +[generators] +cmake_paths + +[options] + +[imports] From 025d7d67130a58e919432b1f642e99ecbeee566f Mon Sep 17 00:00:00 2001 From: tangxinpeng <978831608@qq.com> Date: Tue, 20 Apr 2021 00:45:32 +0800 Subject: [PATCH 005/182] Modify the way the log file is opened (#137) Co-authored-by: linuxserverdev --- trantor/utils/AsyncFileLogger.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/trantor/utils/AsyncFileLogger.cc b/trantor/utils/AsyncFileLogger.cc index b5bb0800..e754d489 100644 --- a/trantor/utils/AsyncFileLogger.cc +++ b/trantor/utils/AsyncFileLogger.cc @@ -184,10 +184,7 @@ AsyncFileLogger::LoggerFile::LoggerFile(const std::string &filePath, #ifndef _MSC_VER fp_ = fopen(fileFullName_.c_str(), "a"); #else - if (fopen_s(&fp_, fileFullName_.c_str(), "a") != 0) - { - fp_ = nullptr; - } + fp_ = _fsopen(fileFullName_.c_str(), "a+", _SH_DENYWR); #endif if (fp_ == nullptr) { From 06642c17ecebb1f8ceaac21d330eb082d578dada Mon Sep 17 00:00:00 2001 From: interfector18 Date: Thu, 22 Apr 2021 03:39:54 +0200 Subject: [PATCH 006/182] Fix constructing Date in a daylight saving timezone (#140) --- trantor/utils/Date.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/trantor/utils/Date.cc b/trantor/utils/Date.cc index ba3d9316..14bc83ee 100644 --- a/trantor/utils/Date.cc +++ b/trantor/utils/Date.cc @@ -344,6 +344,7 @@ Date::Date(unsigned int year, { struct tm tm; memset(&tm, 0, sizeof(tm)); + tm.tm_isdst = -1; time_t epoch; tm.tm_year = year - 1900; tm.tm_mon = month - 1; From 33171881fce47b96e10b5d1def94252ba3ccb072 Mon Sep 17 00:00:00 2001 From: Philip Kovacs <30226827+pkovacs@users.noreply.github.com> Date: Thu, 22 Apr 2021 01:20:50 -0400 Subject: [PATCH 007/182] Add version/soversion to shared library (#141) --- CMakeLists.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 95aae1a5..84335b1c 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,8 +4,8 @@ project(trantor) set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) set(TRANTOR_MAJOR_VERSION 1) -set(TRANTOR_MINOR_VERSION 1) -set(TRANTOR_PATCH_VERSION 1) +set(TRANTOR_MINOR_VERSION 4) +set(TRANTOR_PATCH_VERSION 0) set(TRANTOR_VERSION ${TRANTOR_MAJOR_VERSION}.${TRANTOR_MINOR_VERSION}.${TRANTOR_PATCH_VERSION}) @@ -27,6 +27,9 @@ if(BUILD_TRANTOR_SHARED) set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIB_DIR}") endif("${isSystemDir}" STREQUAL "-1") add_library(${PROJECT_NAME} SHARED) + set_target_properties(${PROJECT_NAME} PROPERTIES + VERSION ${TRANTOR_VERSION} + SOVERSION ${TRANTOR_MAJOR_VERSION}) if(CMAKE_CXX_COMPILER_ID MATCHES MSVC) # Ignore MSVC C4251 and C4275 warning of exporting std objects with no dll export # We export class to facilitate maintenance, thus if you compile From 5103ec795ed352713679922882ee7f5a4f93a938 Mon Sep 17 00:00:00 2001 From: Philip Kovacs <30226827+pkovacs@users.noreply.github.com> Date: Thu, 22 Apr 2021 02:26:04 -0400 Subject: [PATCH 008/182] GNU: -Wall -Wextra -Werror; fix related warnings (#143) --- CMakeLists.txt | 4 ++++ trantor/net/EventLoop.cc | 7 ++++--- trantor/net/inner/NormalResolver.cc | 2 +- trantor/net/inner/TcpConnectionImpl.cc | 8 ++++---- trantor/net/inner/TimerQueue.cc | 2 +- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 84335b1c..523f8546 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,6 +41,10 @@ else(BUILD_TRANTOR_SHARED) add_library(${PROJECT_NAME} STATIC) endif(BUILD_TRANTOR_SHARED) +if (CMAKE_CXX_COMPILER_ID MATCHES GNU) + target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wextra -Werror) +endif() + include(GenerateExportHeader) generate_export_header(${PROJECT_NAME} EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/exports/trantor/exports.h) diff --git a/trantor/net/EventLoop.cc b/trantor/net/EventLoop.cc index 076b2fa8..b4523ad5 100644 --- a/trantor/net/EventLoop.cc +++ b/trantor/net/EventLoop.cc @@ -68,12 +68,11 @@ EventLoop::EventLoop() currentActiveChannel_(nullptr), eventHandling_(false), timerQueue_(new TimerQueue(this)), - threadLocalLoopPtr_(&t_loopInThisThread) #ifdef __linux__ - , wakeupFd_(createEventfd()), - wakeupChannelPtr_(new Channel(this, wakeupFd_)) + wakeupChannelPtr_(new Channel(this, wakeupFd_)), #endif + threadLocalLoopPtr_(&t_loopInThisThread) { if (t_loopInThisThread) { @@ -305,10 +304,12 @@ void EventLoop::wakeup() uint64_t tmp = 1; #ifdef __linux__ int ret = write(wakeupFd_, &tmp, sizeof(tmp)); + (void)ret; #elif defined _WIN32 poller_->postEvent(1); #else int ret = write(wakeupFd_[1], &tmp, sizeof(tmp)); + (void)ret; #endif } void EventLoop::wakeupRead() diff --git a/trantor/net/inner/NormalResolver.cc b/trantor/net/inner/NormalResolver.cc index b95e046b..87c2d7c2 100644 --- a/trantor/net/inner/NormalResolver.cc +++ b/trantor/net/inner/NormalResolver.cc @@ -12,7 +12,7 @@ using namespace trantor; -std::shared_ptr Resolver::newResolver(trantor::EventLoop *loop, +std::shared_ptr Resolver::newResolver(trantor::EventLoop *, size_t timeout) { return std::make_shared(timeout); diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index f3235b54..f10b0005 100644 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -1430,7 +1430,7 @@ void TcpConnectionImpl::sendFileInLoop(const BufferNodePtr &filePtr) { filePtr->fileBytesToSend_ -= nSend; filePtr->offset_ += static_cast(nSend); - if (static_cast(nSend) < n) + if (static_cast(nSend) < static_cast(n)) { if (!ioChannelPtr_->isWriting()) { @@ -1559,12 +1559,12 @@ TcpConnectionImpl::TcpConnectionImpl(EventLoop *loop, bool isServer, bool validateCert, const std::string &hostname) - : loop_(loop), + : isEncrypted_(true), + loop_(loop), ioChannelPtr_(new Channel(loop, socketfd)), socketPtr_(new Socket(socketfd)), localAddr_(localAddr), - peerAddr_(peerAddr), - isEncrypted_(true) + peerAddr_(peerAddr) { LOG_TRACE << "new connection:" << peerAddr.toIpPort() << "->" << localAddr.toIpPort(); diff --git a/trantor/net/inner/TimerQueue.cc b/trantor/net/inner/TimerQueue.cc index 337baf6e..91293a28 100644 --- a/trantor/net/inner/TimerQueue.cc +++ b/trantor/net/inner/TimerQueue.cc @@ -65,7 +65,7 @@ static void resetTimerfd(int timerfd, const TimePoint &expiration) // LOG_SYSERR << "timerfd_settime()"; } } -static void readTimerfd(int timerfd, const TimePoint &now) +static void readTimerfd(int timerfd, const TimePoint &) { uint64_t howmany; ssize_t n = ::read(timerfd, &howmany, sizeof howmany); From 24310aac97e97400ffc25185a5a2425491eaab72 Mon Sep 17 00:00:00 2001 From: An Tao Date: Tue, 27 Apr 2021 13:06:07 +0800 Subject: [PATCH 009/182] Use double instead of long double as the type for timer durations (#144) --- trantor/net/EventLoop.h | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/trantor/net/EventLoop.h b/trantor/net/EventLoop.h index a953a876..cdcefffc 100644 --- a/trantor/net/EventLoop.h +++ b/trantor/net/EventLoop.h @@ -164,12 +164,11 @@ class TRANTOR_EXPORT EventLoop : NonCopyable runAfter(10min, task); @endcode */ - TimerId runAfter(const std::chrono::duration &delay, - const Func &cb) + TimerId runAfter(const std::chrono::duration &delay, const Func &cb) { return runAfter(delay.count(), cb); } - TimerId runAfter(const std::chrono::duration &delay, Func &&cb) + TimerId runAfter(const std::chrono::duration &delay, Func &&cb) { return runAfter(delay.count(), std::move(cb)); } @@ -194,13 +193,12 @@ class TRANTOR_EXPORT EventLoop : NonCopyable runEvery(0.1h, task); @endcode */ - TimerId runEvery(const std::chrono::duration &interval, + TimerId runEvery(const std::chrono::duration &interval, const Func &cb) { return runEvery(interval.count(), cb); } - TimerId runEvery(const std::chrono::duration &interval, - Func &&cb) + TimerId runEvery(const std::chrono::duration &interval, Func &&cb) { return runEvery(interval.count(), std::move(cb)); } From a46d9d4a08d3a3512a09ca365a6a73203feea94a Mon Sep 17 00:00:00 2001 From: Monotlith Date: Mon, 3 May 2021 04:59:03 +0200 Subject: [PATCH 010/182] Add wincrypt.h include for Windows (#145) The following include fixes compilation under Windows 20H2 (19042.928)-x64. I do not know why, but without this include, the build fails. --- trantor/net/inner/TcpConnectionImpl.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index f10b0005..b1a424fd 100644 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -24,6 +24,7 @@ #else #include #include +#include #endif #include #include From 941dc48384bd14a2b6a14f3a0dba7b7453d91ed5 Mon Sep 17 00:00:00 2001 From: an-tao Date: Sat, 15 May 2021 17:40:14 +0800 Subject: [PATCH 011/182] Bump version to 1.4.1 --- CMakeLists.txt | 2 +- ChangeLog.md | 26 +++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 523f8546..def121be 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) set(TRANTOR_MAJOR_VERSION 1) set(TRANTOR_MINOR_VERSION 4) -set(TRANTOR_PATCH_VERSION 0) +set(TRANTOR_PATCH_VERSION 1) set(TRANTOR_VERSION ${TRANTOR_MAJOR_VERSION}.${TRANTOR_MINOR_VERSION}.${TRANTOR_PATCH_VERSION}) diff --git a/ChangeLog.md b/ChangeLog.md index 160484a6..43acc71f 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -3,6 +3,28 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [1.4.1] - 2021-05-15 + +### Changed + +- Add github actions of Windows. + +- Modify the way the log file is opened. + +- Add version/soversion to shared library. + +- Use double instead of long double as the type for timer durations. + +### Fixed + +- Fix a bug in the TcpConnectionImpl class. + +- Fix constructing Date in a daylight saving timezone. + +- GNU: -Wall -Wextra -Werror; fix related warnings. + +- Add wincrypt.h include for Windows. + ## [1.4.0] - 2021-04-09 ### API changes list @@ -264,7 +286,9 @@ All notable changes to this project will be documented in this file. ## [1.0.0-rc1] - 2019-06-11 -[Unreleased]: https://github.com/an-tao/trantor/compare/v1.4.0...HEAD +[Unreleased]: https://github.com/an-tao/trantor/compare/v1.4.1...HEAD + +[1.4.1]: https://github.com/an-tao/trantor/compare/v1.4.0...v1.4.1 [1.4.0]: https://github.com/an-tao/trantor/compare/v1.3.0...v1.4.0 From 8d21e95b6c31c782383fdad14fe6acb815a43486 Mon Sep 17 00:00:00 2001 From: An Tao Date: Sun, 16 May 2021 12:35:31 +0800 Subject: [PATCH 012/182] Enable multiple log files or streams (#147) --- CMakeLists.txt | 2 +- trantor/net/EventLoop.cc | 2 +- trantor/tests/LoggerTest.cc | 2 + trantor/utils/Logger.cc | 42 ++++++++++++--- trantor/utils/Logger.h | 104 ++++++++++++++++++++++++++++++++++-- trantor/utils/MsgBuffer.h | 2 +- 6 files changed, 141 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index def121be..47d1f42e 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.5) project(trantor) -set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) set(TRANTOR_MAJOR_VERSION 1) set(TRANTOR_MINOR_VERSION 4) diff --git a/trantor/net/EventLoop.cc b/trantor/net/EventLoop.cc index b4523ad5..8009db47 100644 --- a/trantor/net/EventLoop.cc +++ b/trantor/net/EventLoop.cc @@ -26,7 +26,7 @@ #include #ifdef _WIN32 #include -using ssize_t = std::intptr_t; +using ssize_t = long long; #else #include #endif diff --git a/trantor/tests/LoggerTest.cc b/trantor/tests/LoggerTest.cc index 89e059bd..408fcb77 100644 --- a/trantor/tests/LoggerTest.cc +++ b/trantor/tests/LoggerTest.cc @@ -1133,5 +1133,7 @@ int main() "" << 123 << 123.123 << "haha" << '\n' << std::string("12356"); + LOG_RAW << "Testing finished\n"; + LOG_RAW_TO(5) << "Testing finished\n"; thread_.join(); } diff --git a/trantor/utils/Logger.cc b/trantor/utils/Logger.cc index b053bfd7..b407ef37 100644 --- a/trantor/utils/Logger.cc +++ b/trantor/utils/Logger.cc @@ -165,15 +165,45 @@ Logger::Logger(SourceFile file, int line, bool) logStream_ << strerror_tl(errno) << " (errno=" << errno << ") "; } } +RawLogger::~RawLogger() +{ + if (index_ < 0) + { + auto &oFunc = Logger::outputFunc_(); + if (!oFunc) + return; + oFunc(logStream_.bufferData(), logStream_.bufferLength()); + } + else + { + auto &oFunc = Logger::outputFunc_(index_); + if (!oFunc) + return; + oFunc(logStream_.bufferData(), logStream_.bufferLength()); + } +} Logger::~Logger() { logStream_ << T(" - ", 3) << sourceFile_ << ':' << fileLine_ << '\n'; - auto oFunc = Logger::outputFunc_(); - if (!oFunc) - return; - oFunc(logStream_.bufferData(), logStream_.bufferLength()); - if (level_ >= kError) - Logger::flushFunc_()(); + if (index_ < 0) + { + auto &oFunc = Logger::outputFunc_(); + if (!oFunc) + return; + oFunc(logStream_.bufferData(), logStream_.bufferLength()); + if (level_ >= kError) + Logger::flushFunc_()(); + } + else + { + auto &oFunc = Logger::outputFunc_(index_); + if (!oFunc) + return; + oFunc(logStream_.bufferData(), logStream_.bufferLength()); + if (level_ >= kError) + Logger::flushFunc_(index_)(); + } + // logStream_.resetBuffer(); } LogStream &Logger::stream() diff --git a/trantor/utils/Logger.h b/trantor/utils/Logger.h index 2bcee3b7..ddb61005 100644 --- a/trantor/utils/Logger.h +++ b/trantor/utils/Logger.h @@ -18,9 +18,10 @@ #include #include #include -#include +#include #include #include +#include namespace trantor { @@ -79,6 +80,11 @@ class TRANTOR_EXPORT Logger : public NonCopyable Logger(SourceFile file, int line, bool isSysErr); Logger(SourceFile file, int line, LogLevel level, const char *func); ~Logger(); + Logger &setIndex(int index) + { + index_ = index; + return *this; + } LogStream &stream(); /** @@ -90,10 +96,19 @@ class TRANTOR_EXPORT Logger : public NonCopyable */ static void setOutputFunction( std::function outputFunc, - std::function flushFunc) + std::function flushFunc, + int index = -1) { - outputFunc_() = outputFunc; - flushFunc_() = flushFunc; + if (index < 0) + { + outputFunc_() = outputFunc; + flushFunc_() = flushFunc; + } + else + { + outputFunc_(index) = outputFunc; + flushFunc_(index) = flushFunc; + } } /** @@ -147,11 +162,60 @@ class TRANTOR_EXPORT Logger : public NonCopyable static std::function flushFunc = Logger::defaultFlushFunction; return flushFunc; } + static std::function + &outputFunc_(size_t index) + { + static std::vector< + std::function> + outputFuncs; + if (index < outputFuncs.size()) + { + return outputFuncs[index]; + } + while (index >= outputFuncs.size()) + { + outputFuncs.emplace_back(outputFunc_()); + } + return outputFuncs[index]; + } + static std::function &flushFunc_(size_t index) + { + static std::vector> flushFuncs; + if (index < flushFuncs.size()) + { + return flushFuncs[index]; + } + while (index >= flushFuncs.size()) + { + flushFuncs.emplace_back(flushFunc_()); + } + return flushFuncs[index]; + } + friend class RawLogger; LogStream logStream_; Date date_{Date::now()}; SourceFile sourceFile_; int fileLine_; LogLevel level_; + int index_{-1}; +}; +class TRANTOR_EXPORT RawLogger : public NonCopyable +{ + public: + ~RawLogger(); + RawLogger &setIndex(int index) + { + index_ = index; + return *this; + } + LogStream &stream() + { + return logStream_; + } + + private: + LogStream logStream_; + int index_{-1}; }; #ifdef NDEBUG #define LOG_TRACE \ @@ -163,21 +227,53 @@ class TRANTOR_EXPORT Logger : public NonCopyable if (trantor::Logger::logLevel() <= trantor::Logger::kTrace) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kTrace, __func__) \ .stream() +#define LOG_TRACE_TO(index) \ + if (trantor::Logger::logLevel() <= trantor::Logger::kTrace) \ + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kTrace, __func__) \ + .setIndex(index) \ + .stream() + #endif + #define LOG_DEBUG \ if (trantor::Logger::logLevel() <= trantor::Logger::kDebug) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kDebug, __func__) \ .stream() +#define LOG_DEBUG_TO(index) \ + if (trantor::Logger::logLevel() <= trantor::Logger::kDebug) \ + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kDebug, __func__) \ + .setIndex(index) \ + .stream() #define LOG_INFO \ if (trantor::Logger::logLevel() <= trantor::Logger::kInfo) \ trantor::Logger(__FILE__, __LINE__).stream() +#define LOG_INFO_TO(index) \ + if (trantor::Logger::logLevel() <= trantor::Logger::kInfo) \ + trantor::Logger(__FILE__, __LINE__).setIndex(index).stream() #define LOG_WARN \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kWarn).stream() +#define LOG_WARN_TO(index) \ + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kWarn) \ + .setIndex(index) \ + .stream() #define LOG_ERROR \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kError).stream() +#define LOG_ERROR_TO(index) \ + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kError) \ + .setIndex(index) \ + .stream() #define LOG_FATAL \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kFatal).stream() +#define LOG_FATAL_TO(index) \ + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kFatal) \ + .setIndex(index) \ + .stream() #define LOG_SYSERR trantor::Logger(__FILE__, __LINE__, true).stream() +#define LOG_SYSERR_TO(index) \ + trantor::Logger(__FILE__, __LINE__, true).setIndex(index).stream() + +#define LOG_RAW trantor::RawLogger().stream() +#define LOG_RAW_TO(index) trantor::RawLogger().setIndex(index).stream() #define LOG_TRACE_IF(cond) \ if ((trantor::Logger::logLevel() <= trantor::Logger::kTrace) && (cond)) \ diff --git a/trantor/utils/MsgBuffer.h b/trantor/utils/MsgBuffer.h index bdd5c23b..33c8633f 100644 --- a/trantor/utils/MsgBuffer.h +++ b/trantor/utils/MsgBuffer.h @@ -22,7 +22,7 @@ #include #include #ifdef _WIN32 -using ssize_t = std::intptr_t; +using ssize_t = long long; #endif namespace trantor From f72bc684a517f9bf9bd277d74d381918d843f45b Mon Sep 17 00:00:00 2001 From: Tommy Chiang Date: Wed, 26 May 2021 15:17:52 +0800 Subject: [PATCH 013/182] Add SSL_CONF_cmd support (#148) --- trantor/net/TcpClient.cc | 2 +- trantor/net/TcpConnection.h | 14 +++-- trantor/net/TcpServer.cc | 10 ++-- trantor/net/TcpServer.h | 4 +- trantor/net/inner/TcpConnectionImpl.cc | 75 +++++++++++++++++++------- trantor/net/inner/TcpConnectionImpl.h | 34 +++++++----- trantor/tests/DelayedSSLServerTest.cc | 2 +- 7 files changed, 98 insertions(+), 43 deletions(-) diff --git a/trantor/net/TcpClient.cc b/trantor/net/TcpClient.cc index 35328e46..e274591a 100644 --- a/trantor/net/TcpClient.cc +++ b/trantor/net/TcpClient.cc @@ -208,7 +208,7 @@ void TcpClient::enableSSL(bool useOldTLS, { #ifdef USE_OPENSSL /* Create a new OpenSSL context */ - sslCtxPtr_ = newSSLContext(useOldTLS, validateCert); + sslCtxPtr_ = newSSLContext(useOldTLS, validateCert, {}); validateCert_ = validateCert; if (!hostname.empty()) { diff --git a/trantor/net/TcpConnection.h b/trantor/net/TcpConnection.h index 123dbfc8..6c7eaa3b 100644 --- a/trantor/net/TcpConnection.h +++ b/trantor/net/TcpConnection.h @@ -29,7 +29,8 @@ class SSLContext; TRANTOR_EXPORT std::shared_ptr newSSLServerContext( const std::string &certPath, const std::string &keyPath, - bool useOldTLS = false); + bool useOldTLS = false, + const std::vector> &sslConfCmds = {}); /** * @brief This class represents a TCP connection. * @@ -230,10 +231,13 @@ class TRANTOR_EXPORT TcpConnection * @param hostname The server hostname for SNI. If it is empty, the SNI is * not used. */ - virtual void startClientEncryption(std::function callback, - bool useOldTLS = false, - bool validateCert = true, - std::string hostname = "") = 0; + virtual void startClientEncryption( + std::function callback, + bool useOldTLS = false, + bool validateCert = true, + std::string hostname = "", + const std::vector> &sslConfCmds = + {}) = 0; /** * @brief Start the SSL encryption on the connection (as a server). diff --git a/trantor/net/TcpServer.cc b/trantor/net/TcpServer.cc index 523301cf..d67488bd 100644 --- a/trantor/net/TcpServer.cc +++ b/trantor/net/TcpServer.cc @@ -185,13 +185,15 @@ const trantor::InetAddress &TcpServer::address() const return acceptorPtr_->addr(); } -void TcpServer::enableSSL(const std::string &certPath, - const std::string &keyPath, - bool useOldTLS) +void TcpServer::enableSSL( + const std::string &certPath, + const std::string &keyPath, + bool useOldTLS, + const std::vector> &sslConfCmds) { #ifdef USE_OPENSSL /* Create a new OpenSSL context */ - sslCtxPtr_ = newSSLServerContext(certPath, keyPath, useOldTLS); + sslCtxPtr_ = newSSLServerContext(certPath, keyPath, useOldTLS, sslConfCmds); #else LOG_FATAL << "OpenSSL is not found in your system!"; abort(); diff --git a/trantor/net/TcpServer.h b/trantor/net/TcpServer.h index 93421e1f..40c921f8 100644 --- a/trantor/net/TcpServer.h +++ b/trantor/net/TcpServer.h @@ -207,7 +207,9 @@ class TRANTOR_EXPORT TcpServer : NonCopyable */ void enableSSL(const std::string &certPath, const std::string &keyPath, - bool useOldTLS = false); + bool useOldTLS = false, + const std::vector> + &sslConfCmds = {}); private: EventLoop *loop_; diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index b1a424fd..af899631 100644 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -197,10 +197,23 @@ void initOpenSSL() class SSLContext { public: - explicit SSLContext(bool useOldTLS, bool enableValidtion) + explicit SSLContext( + bool useOldTLS, + bool enableValidtion, + const std::vector> &sslConfCmds) { #if (OPENSSL_VERSION_NUMBER >= 0x10100000L) ctxPtr_ = SSL_CTX_new(TLS_method()); + SSL_CONF_CTX *cctx = SSL_CONF_CTX_new(); + SSL_CONF_CTX_set_flags(cctx, SSL_CONF_FLAG_SERVER); + SSL_CONF_CTX_set_flags(cctx, SSL_CONF_FLAG_CLIENT); + SSL_CONF_CTX_set_flags(cctx, SSL_CONF_FLAG_CERTIFICATE); + SSL_CONF_CTX_set_ssl_ctx(cctx, ctxPtr_); + for (auto cmd : sslConfCmds) + { + SSL_CONF_cmd(cctx, cmd.first.data(), cmd.second.data()); + } + SSL_CONF_CTX_finish(cctx); if (!useOldTLS) { SSL_CTX_set_min_proto_version(ctxPtr_, TLS1_2_VERSION); @@ -213,6 +226,16 @@ class SSLContext } #else ctxPtr_ = SSL_CTX_new(SSLv23_method()); + SSL_CONF_CTX *cctx = SSL_CONF_CTX_new(); + SSL_CONF_CTX_set_flags(cctx, SSL_CONF_FLAG_SERVER); + SSL_CONF_CTX_set_flags(cctx, SSL_CONF_FLAG_CLIENT); + SSL_CONF_CTX_set_flags(cctx, SSL_CONF_FLAG_CERTIFICATE); + SSL_CONF_CTX_set_ssl_ctx(cctx, ctxPtr_); + for (auto cmd : sslConfCmds) + { + SSL_CONF_cmd(cctx, cmd.first.data(), cmd.second.data()); + } + SSL_CONF_CTX_finish(cctx); if (!useOldTLS) { SSL_CTX_set_options(ctxPtr_, SSL_OP_NO_TLSv1 | SSL_OP_NO_TLSv1_1); @@ -271,16 +294,21 @@ class SSLConn SSL *SSL_; }; -std::shared_ptr newSSLContext(bool useOldTLS, bool validateCert) +std::shared_ptr newSSLContext( + bool useOldTLS, + bool validateCert, + const std::vector> &sslConfCmds) { // init OpenSSL initOpenSSL(); - return std::make_shared(useOldTLS, validateCert); + return std::make_shared(useOldTLS, validateCert, sslConfCmds); } -std::shared_ptr newSSLServerContext(const std::string &certPath, - const std::string &keyPath, - bool useOldTLS) +std::shared_ptr newSSLServerContext( + const std::string &certPath, + const std::string &keyPath, + bool useOldTLS, + const std::vector> &sslConfCmds) { - auto ctx = newSSLContext(useOldTLS, false); + auto ctx = newSSLContext(useOldTLS, false, sslConfCmds); auto r = SSL_CTX_use_certificate_chain_file(ctx->get(), certPath.c_str()); if (!r) { @@ -319,9 +347,11 @@ std::shared_ptr newSSLServerContext(const std::string &certPath, #else namespace trantor { -std::shared_ptr newSSLServerContext(const std::string &certPath, - const std::string &keyPath, - bool useOldTLS) +std::shared_ptr newSSLServerContext( + const std::string &certPath, + const std::string &keyPath, + bool useOldTLS, + const std::vector> &sslConfCmds) { LOG_FATAL << "OpenSSL is not found in your system!"; abort(); @@ -360,7 +390,8 @@ void TcpConnectionImpl::startClientEncryptionInLoop( std::function &&callback, bool useOldTLS, bool validateCert, - const std::string &hostname) + const std::string &hostname, + const std::vector> &sslConfCmds) { validateCert_ = validateCert; loop_->assertInLoopThread(); @@ -371,7 +402,8 @@ void TcpConnectionImpl::startClientEncryptionInLoop( } sslEncryptionPtr_ = std::make_unique(); sslEncryptionPtr_->upgradeCallback_ = std::move(callback); - sslEncryptionPtr_->sslCtxPtr_ = newSSLContext(useOldTLS, validateCert_); + sslEncryptionPtr_->sslCtxPtr_ = + newSSLContext(useOldTLS, validateCert_, sslConfCmds); sslEncryptionPtr_->sslPtr_ = std::make_unique(sslEncryptionPtr_->sslCtxPtr_->get()); if (validateCert) @@ -452,10 +484,12 @@ void TcpConnectionImpl::startServerEncryption( #endif } -void TcpConnectionImpl::startClientEncryption(std::function callback, - bool useOldTLS, - bool validateCert, - std::string hostname) +void TcpConnectionImpl::startClientEncryption( + std::function callback, + bool useOldTLS, + bool validateCert, + std::string hostname, + const std::vector> &sslConfCmds) { #ifndef USE_OPENSSL LOG_FATAL << "OpenSSL is not found in your system!"; @@ -475,7 +509,8 @@ void TcpConnectionImpl::startClientEncryption(std::function callback, startClientEncryptionInLoop(std::move(callback), useOldTLS, validateCert, - hostname); + hostname, + sslConfCmds); } else { @@ -483,11 +518,13 @@ void TcpConnectionImpl::startClientEncryption(std::function callback, callback = std::move(callback), useOldTLS, hostname = std::move(hostname), - validateCert]() mutable { + validateCert, + &sslConfCmds]() mutable { thisPtr->startClientEncryptionInLoop(std::move(callback), useOldTLS, validateCert, - hostname); + hostname, + sslConfCmds); }); } #endif diff --git a/trantor/net/inner/TcpConnectionImpl.h b/trantor/net/inner/TcpConnectionImpl.h index 060d3423..cad3dfa4 100644 --- a/trantor/net/inner/TcpConnectionImpl.h +++ b/trantor/net/inner/TcpConnectionImpl.h @@ -38,10 +38,15 @@ enum class SSLStatus class SSLContext; class SSLConn; -std::shared_ptr newSSLContext(bool useOldTLS, bool validateCert); -std::shared_ptr newSSLServerContext(const std::string &certPath, - const std::string &keyPath, - bool useOldTLS); +std::shared_ptr newSSLContext( + bool useOldTLS, + bool validateCert, + const std::vector> &sslConfCmds); +std::shared_ptr newSSLServerContext( + const std::string &certPath, + const std::string &keyPath, + bool useOldTLS, + const std::vector> &sslConfCmds); // void initServerSSLContext(const std::shared_ptr &ctx, // const std::string &certPath, // const std::string &keyPath); @@ -171,10 +176,13 @@ class TcpConnectionImpl : public TcpConnection, { return bytesReceived_; } - virtual void startClientEncryption(std::function callback, - bool useOldTLS = false, - bool validateCert = true, - std::string hostname = "") override; + virtual void startClientEncryption( + std::function callback, + bool useOldTLS = false, + bool validateCert = true, + std::string hostname = "", + const std::vector> &sslConfCmds = + {}) override; virtual void startServerEncryption(const std::shared_ptr &ctx, std::function callback) override; virtual bool isSSLConnection() const override @@ -320,10 +328,12 @@ class TcpConnectionImpl : public TcpConnection, std::string hostname_; }; std::unique_ptr sslEncryptionPtr_; - void startClientEncryptionInLoop(std::function &&callback, - bool useOldTLS, - bool validateCert, - const std::string &hostname); + void startClientEncryptionInLoop( + std::function &&callback, + bool useOldTLS, + bool validateCert, + const std::string &hostname, + const std::vector> &sslConfCmds); void startServerEncryptionInLoop(const std::shared_ptr &ctx, std::function &&callback); #endif diff --git a/trantor/tests/DelayedSSLServerTest.cc b/trantor/tests/DelayedSSLServerTest.cc index 2a13fa9b..466b7ad1 100644 --- a/trantor/tests/DelayedSSLServerTest.cc +++ b/trantor/tests/DelayedSSLServerTest.cc @@ -17,7 +17,7 @@ int main() InetAddress addr(8888); #endif TcpServer server(loopThread.getLoop(), addr, "test"); - auto ctx = newSSLServerContext("server.pem", "server.pem"); + auto ctx = newSSLServerContext("server.pem", "server.pem", {}); LOG_INFO << "start"; server.setRecvMessageCallback( [](const TcpConnectionPtr &connectionPtr, MsgBuffer *buffer) { From 8b26fd90bcbd9577d7425dc04d1e13070dd8f82b Mon Sep 17 00:00:00 2001 From: An Tao Date: Wed, 26 May 2021 16:31:19 +0800 Subject: [PATCH 014/182] Additional modifications to the final submission (#149) --- trantor/net/TcpClient.cc | 10 ++++++---- trantor/net/TcpClient.h | 10 +++++++--- trantor/net/TcpConnection.h | 2 ++ trantor/net/TcpServer.h | 2 ++ 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/trantor/net/TcpClient.cc b/trantor/net/TcpClient.cc index e274591a..98048a07 100644 --- a/trantor/net/TcpClient.cc +++ b/trantor/net/TcpClient.cc @@ -202,13 +202,15 @@ void TcpClient::removeConnection(const TcpConnectionPtr &conn) } } -void TcpClient::enableSSL(bool useOldTLS, - bool validateCert, - std::string hostname) +void TcpClient::enableSSL( + bool useOldTLS, + bool validateCert, + std::string hostname, + const std::vector> &sslConfCmds) { #ifdef USE_OPENSSL /* Create a new OpenSSL context */ - sslCtxPtr_ = newSSLContext(useOldTLS, validateCert, {}); + sslCtxPtr_ = newSSLContext(useOldTLS, validateCert, sslConfCmds); validateCert_ = validateCert; if (!hostname.empty()) { diff --git a/trantor/net/TcpClient.h b/trantor/net/TcpClient.h index 1a5702ae..045ad063 100644 --- a/trantor/net/TcpClient.h +++ b/trantor/net/TcpClient.h @@ -193,16 +193,20 @@ class TRANTOR_EXPORT TcpClient : NonCopyable * @brief Enable SSL encryption. * @param useOldTLS If true, the TLS 1.0 and 1.1 are supported by the * client. - * @param hostname The server hostname for SNI. If it is empty, the SNI is - * not used. * @param validateCert If true, we try to validate if the peer's SSL cert * is valid. + * @param hostname The server hostname for SNI. If it is empty, the SNI is + * not used. + * @param sslConfCmds The commands used to call the SSL_CONF_cmd function in + * OpenSSL. * @note It's well known that TLS 1.0 and 1.1 are not considered secure in * 2020. And it's a good practice to only use TLS 1.2 and above. */ void enableSSL(bool useOldTLS = false, bool validateCert = true, - std::string hostname = ""); + std::string hostname = "", + const std::vector> + &sslConfCmds = {}); private: /// Not thread safe, but in loop diff --git a/trantor/net/TcpConnection.h b/trantor/net/TcpConnection.h index 6c7eaa3b..90f24465 100644 --- a/trantor/net/TcpConnection.h +++ b/trantor/net/TcpConnection.h @@ -230,6 +230,8 @@ class TRANTOR_EXPORT TcpConnection * established. * @param hostname The server hostname for SNI. If it is empty, the SNI is * not used. + * @param sslConfCmds The commands used to call the SSL_CONF_cmd function in + * OpenSSL. */ virtual void startClientEncryption( std::function callback, diff --git a/trantor/net/TcpServer.h b/trantor/net/TcpServer.h index 40c921f8..194fbb24 100644 --- a/trantor/net/TcpServer.h +++ b/trantor/net/TcpServer.h @@ -202,6 +202,8 @@ class TRANTOR_EXPORT TcpServer : NonCopyable * @param keyPath The path of the private key file. * @param useOldTLS If true, the TLS 1.0 and 1.1 are supported by the * server. + * @param sslConfCmds The commands used to call the SSL_CONF_cmd function in + * OpenSSL. * @note It's well known that TLS 1.0 and 1.1 are not considered secure in * 2020. And it's a good practice to only use TLS 1.2 and above. */ From a7150a1607d20c44eea175d17b874f8f14a39efb Mon Sep 17 00:00:00 2001 From: Tommy Chiang Date: Fri, 28 May 2021 14:51:35 +0800 Subject: [PATCH 015/182] Allow OpenSSL recognize SSL_CONF_cmd options (#150) https://www.openssl.org/docs/manmaster/man3/SSL_CONF_CTX_set_flags.html and https://www.openssl.org/docs/manmaster/man3/SSL_CONF_cmd.html for more detail. --- trantor/net/inner/TcpConnectionImpl.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index af899631..57ff1de6 100644 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -208,8 +208,9 @@ class SSLContext SSL_CONF_CTX_set_flags(cctx, SSL_CONF_FLAG_SERVER); SSL_CONF_CTX_set_flags(cctx, SSL_CONF_FLAG_CLIENT); SSL_CONF_CTX_set_flags(cctx, SSL_CONF_FLAG_CERTIFICATE); + SSL_CONF_CTX_set_flags(cctx, SSL_CONF_FLAG_FILE); SSL_CONF_CTX_set_ssl_ctx(cctx, ctxPtr_); - for (auto cmd : sslConfCmds) + for (const auto &cmd : sslConfCmds) { SSL_CONF_cmd(cctx, cmd.first.data(), cmd.second.data()); } @@ -230,8 +231,9 @@ class SSLContext SSL_CONF_CTX_set_flags(cctx, SSL_CONF_FLAG_SERVER); SSL_CONF_CTX_set_flags(cctx, SSL_CONF_FLAG_CLIENT); SSL_CONF_CTX_set_flags(cctx, SSL_CONF_FLAG_CERTIFICATE); + SSL_CONF_CTX_set_flags(cctx, SSL_CONF_FLAG_FILE); SSL_CONF_CTX_set_ssl_ctx(cctx, ctxPtr_); - for (auto cmd : sslConfCmds) + for (const auto &cmd : sslConfCmds) { SSL_CONF_cmd(cctx, cmd.first.data(), cmd.second.data()); } From fa65b9a1e28f2ff839378a70215026ea359be500 Mon Sep 17 00:00:00 2001 From: An Tao Date: Fri, 4 Jun 2021 14:04:46 +0800 Subject: [PATCH 016/182] Export the FixedBuffer (#151) --- trantor/utils/LogStream.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trantor/utils/LogStream.h b/trantor/utils/LogStream.h index 02a3e483..d6684345 100644 --- a/trantor/utils/LogStream.h +++ b/trantor/utils/LogStream.h @@ -30,7 +30,7 @@ static constexpr size_t kSmallBuffer{4000}; static constexpr size_t kLargeBuffer{4000 * 1000}; template -class FixedBuffer : NonCopyable +class TRANTOR_EXPORT FixedBuffer : NonCopyable { public: FixedBuffer() : cur_(data_) From 698ad72b5dc8cd2d1d3c6db1f67741bf0c73062d Mon Sep 17 00:00:00 2001 From: "Joakim L. Gilje" Date: Thu, 10 Jun 2021 16:09:23 +0200 Subject: [PATCH 017/182] added a try_compile block to detect if we need to link against atomic (as required by at least linux/arm32) (#152) --- CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 47d1f42e..7e02fea8 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,6 +122,15 @@ if(WIN32) endif(OpenSSL_FOUND) endif(WIN32) +file(WRITE ${CMAKE_BINARY_DIR}/test_atomic.cpp + "#include \n" + "int main() { std::atomic i(0); i++; return 0; }\n") +try_compile(ATOMIC_WITHOUT_LINKING ${CMAKE_BINARY_DIR} ${CMAKE_BINARY_DIR}/test_atomic.cpp) +if (NOT ATOMIC_WITHOUT_LINKING) + target_link_libraries(${PROJECT_NAME} PUBLIC atomic) +endif () +file(REMOVE ${CMAKE_BINARY_DIR}/test_atomic.cpp) + set_target_properties(${PROJECT_NAME} PROPERTIES CXX_STANDARD 14) set_target_properties(${PROJECT_NAME} PROPERTIES CXX_STANDARD_REQUIRED ON) set_target_properties(${PROJECT_NAME} PROPERTIES CXX_EXTENSIONS OFF) From 255976d89866d556efd24264f65765f3e02bc32b Mon Sep 17 00:00:00 2001 From: Martin Chang Date: Fri, 11 Jun 2021 20:41:04 +0800 Subject: [PATCH 018/182] Add runOnQuit (#153) --- trantor/net/EventLoop.cc | 18 ++++++++++++++++++ trantor/net/EventLoop.h | 10 ++++++++++ trantor/tests/CMakeLists.txt | 4 +++- trantor/tests/RunOnQuitTest.cc | 27 +++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 1 deletion(-) mode change 100755 => 100644 trantor/tests/CMakeLists.txt create mode 100644 trantor/tests/RunOnQuitTest.cc diff --git a/trantor/net/EventLoop.cc b/trantor/net/EventLoop.cc index 8009db47..8a60a03d 100644 --- a/trantor/net/EventLoop.cc +++ b/trantor/net/EventLoop.cc @@ -111,6 +111,7 @@ void EventLoop::resetAfterFork() } EventLoop::~EventLoop() { + quit(); assert(!looping_); t_loopInThisThread = nullptr; #ifdef __linux__ @@ -147,6 +148,13 @@ void EventLoop::removeChannel(Channel *channel) void EventLoop::quit() { quit_ = true; + + Func f; + while (funcsOnQuit_.dequeue(f)) + { + f(); + } + // There is a chance that loop() just executes while(!quit_) and exits, // then EventLoop destructs, then we are accessing an invalid object. // Can be fixed using mutex_ in both places. @@ -351,4 +359,14 @@ void EventLoop::moveToCurrentThread() threadId_ = std::this_thread::get_id(); } +void EventLoop::runOnQuit(Func &&cb) +{ + funcsOnQuit_.enqueue(std::move(cb)); +} + +void EventLoop::runOnQuit(const Func &cb) +{ + funcsOnQuit_.enqueue(cb); +} + } // namespace trantor diff --git a/trantor/net/EventLoop.h b/trantor/net/EventLoop.h index cdcefffc..d28f6b8b 100644 --- a/trantor/net/EventLoop.h +++ b/trantor/net/EventLoop.h @@ -274,6 +274,15 @@ class TRANTOR_EXPORT EventLoop : NonCopyable return callingFuncs_; } + /** + * @brief Run functions when the event loop quits + * + * @param cb the function to run + * @note the function runs on the thread that quits the EventLoop + */ + void runOnQuit(Func &&cb); + void runOnQuit(const Func &cb); + private: void abortNotInLoopThread(); void wakeup(); @@ -289,6 +298,7 @@ class TRANTOR_EXPORT EventLoop : NonCopyable bool eventHandling_; MpscQueue funcs_; std::unique_ptr timerQueue_; + MpscQueue funcsOnQuit_; bool callingFuncs_{false}; #ifdef __linux__ int wakeupFd_; diff --git a/trantor/tests/CMakeLists.txt b/trantor/tests/CMakeLists.txt old mode 100755 new mode 100644 index 5a9def58..180f5565 --- a/trantor/tests/CMakeLists.txt +++ b/trantor/tests/CMakeLists.txt @@ -18,6 +18,7 @@ add_executable(kickoff_test KickoffTest.cc) add_executable(dns_test DnsTest.cc) add_executable(delayed_ssl_server_test DelayedSSLServerTest.cc) add_executable(delayed_ssl_client_test DelayedSSLClientTest.cc) +add_executable(run_on_quit_test RunOnQuitTest.cc) set(targets_list ssl_server_test ssl_client_test @@ -38,7 +39,8 @@ set(targets_list kickoff_test dns_test delayed_ssl_server_test - delayed_ssl_client_test) + delayed_ssl_client_test + run_on_quit_test) set_property(TARGET ${targets_list} PROPERTY CXX_STANDARD 14) set_property(TARGET ${targets_list} PROPERTY CXX_STANDARD_REQUIRED ON) diff --git a/trantor/tests/RunOnQuitTest.cc b/trantor/tests/RunOnQuitTest.cc new file mode 100644 index 00000000..37f95728 --- /dev/null +++ b/trantor/tests/RunOnQuitTest.cc @@ -0,0 +1,27 @@ +#include +#include +#include +#include +#ifndef _WIN32 +#include +#endif + +int main() +{ + std::atomic flag(false); + { + trantor::EventLoopThread thr; + thr.getLoop()->runOnQuit([&]() { flag = true; }); + thr.run(); + thr.getLoop()->quit(); + } + + if (flag == false) + { + std::cerr << "Test failed\n"; + } + else + { + std::cout << "Success\n"; + } +} From a70a0ab4118100e809c828330e7c07e9b5b4466f Mon Sep 17 00:00:00 2001 From: an-tao Date: Fri, 18 Jun 2021 15:05:33 +0800 Subject: [PATCH 019/182] Bump version to 1.5.0 --- CMakeLists.txt | 4 ++-- ChangeLog.md | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e02fea8..3ad175ab 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,8 +4,8 @@ project(trantor) list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) set(TRANTOR_MAJOR_VERSION 1) -set(TRANTOR_MINOR_VERSION 4) -set(TRANTOR_PATCH_VERSION 1) +set(TRANTOR_MINOR_VERSION 5) +set(TRANTOR_PATCH_VERSION 0) set(TRANTOR_VERSION ${TRANTOR_MAJOR_VERSION}.${TRANTOR_MINOR_VERSION}.${TRANTOR_PATCH_VERSION}) diff --git a/ChangeLog.md b/ChangeLog.md index 43acc71f..a63fe82c 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -3,6 +3,22 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [1.5.0] - 2021-06-18 + +### API changes list + +- Enable multiple log files or streams. + +- Add SSL_CONF_cmd support. + +- Add runOnQuit to the EventLoop class. + +### Changed + +- Export the FixedBuffer. + +- Added a try_compile block to detect if we need to link against atomic. + ## [1.4.1] - 2021-05-15 ### Changed @@ -286,7 +302,9 @@ All notable changes to this project will be documented in this file. ## [1.0.0-rc1] - 2019-06-11 -[Unreleased]: https://github.com/an-tao/trantor/compare/v1.4.1...HEAD +[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.0...HEAD + +[1.5.0]: https://github.com/an-tao/trantor/compare/v1.4.1...v1.5.0 [1.4.1]: https://github.com/an-tao/trantor/compare/v1.4.0...v1.4.1 From 3dfcd704f0f693a245d21ad05f9271254e38592d Mon Sep 17 00:00:00 2001 From: Maxim Teplinsky <52708915+teplinsky-maxim@users.noreply.github.com> Date: Thu, 8 Jul 2021 10:30:38 +0500 Subject: [PATCH 020/182] Fix warning C4244 in MSVC (#155) --- trantor/utils/Logger.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trantor/utils/Logger.h b/trantor/utils/Logger.h index ddb61005..1e3e84ac 100644 --- a/trantor/utils/Logger.h +++ b/trantor/utils/Logger.h @@ -134,7 +134,7 @@ class TRANTOR_EXPORT Logger : public NonCopyable protected: static void defaultOutputFunction(const char *msg, const uint64_t len) { - fwrite(msg, 1, len, stdout); + fwrite(msg, 1, static_cast(len), stdout); } static void defaultFlushFunction() { From 15556700e5ce305c7d3fd5a970a58ef6b8297e0c Mon Sep 17 00:00:00 2001 From: Martin Chang Date: Sun, 11 Jul 2021 11:44:18 +0800 Subject: [PATCH 021/182] Disable strict compiler check on Windows with GCC (#157) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3ad175ab..01b39344 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,7 +41,7 @@ else(BUILD_TRANTOR_SHARED) add_library(${PROJECT_NAME} STATIC) endif(BUILD_TRANTOR_SHARED) -if (CMAKE_CXX_COMPILER_ID MATCHES GNU) +if (NOT ${CMAKE_PLATFORM_NAME} STREQUAL "Windows" AND CMAKE_CXX_COMPILER_ID MATCHES GNU) target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wextra -Werror) endif() From d7f700ac7ca0b6c656755142378bc71cd34ecfed Mon Sep 17 00:00:00 2001 From: Greisberger Christophe Date: Mon, 12 Jul 2021 15:50:02 +0200 Subject: [PATCH 022/182] Added support for paths containing unicode characters on Windows (#156) --- trantor/net/TcpConnection.h | 14 +++++++- trantor/net/inner/TcpConnectionImpl.cc | 45 +++++++++++++++++--------- trantor/net/inner/TcpConnectionImpl.h | 5 +++ trantor/utils/AsyncFileLogger.cc | 37 ++++++++++++++++++++- 4 files changed, 84 insertions(+), 17 deletions(-) mode change 100644 => 100755 trantor/net/TcpConnection.h mode change 100644 => 100755 trantor/net/inner/TcpConnectionImpl.cc mode change 100644 => 100755 trantor/net/inner/TcpConnectionImpl.h diff --git a/trantor/net/TcpConnection.h b/trantor/net/TcpConnection.h old mode 100644 new mode 100755 index 90f24465..24b3c800 --- a/trantor/net/TcpConnection.h +++ b/trantor/net/TcpConnection.h @@ -59,13 +59,25 @@ class TRANTOR_EXPORT TcpConnection /** * @brief Send a file to the peer. * - * @param fileName + * @param fileName in UTF-8 * @param offset * @param length */ virtual void sendFile(const char *fileName, size_t offset = 0, size_t length = 0) = 0; +#ifdef _WIN32 + /** + * @brief Send a file to the peer. + * + * @param fileName in UCS-2 (windows only) + * @param offset + * @param length + */ + virtual void sendFile(const wchar_t *fileName, + size_t offset = 0, + size_t length = 0) = 0; +#endif // WIN32 /** * @brief Get the local address of the connection. diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc old mode 100644 new mode 100755 index 57ff1de6..2565bbf0 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -33,9 +33,6 @@ #include #include #endif -#ifdef _WIN32 -#define stat _stati64 -#endif #include using namespace trantor; @@ -1273,7 +1270,19 @@ void TcpConnectionImpl::sendFile(const char *fileName, size_t length) { assert(fileName); -#ifndef _WIN32 +#ifdef _WIN32 + // Convert UTF-8 file path to UCS-2 + int nSizeNeeded = ::MultiByteToWideChar( + CP_UTF8, 0, fileName, (int)strnlen_s(fileName, MAX_PATH), NULL, 0); + std::wstring wFileName(nSizeNeeded, 0); + ::MultiByteToWideChar(CP_UTF8, + 0, + fileName, + (int)strnlen_s(fileName, MAX_PATH), + &wFileName[0], + nSizeNeeded); + sendFile(wFileName.c_str(), offset, length); +#else // _WIN32 int fd = open(fileName, O_RDONLY); if (fd < 0) @@ -1295,16 +1304,22 @@ void TcpConnectionImpl::sendFile(const char *fileName, } sendFile(fd, offset, length); -#else -#ifndef _MSC_VER - auto fp = fopen(fileName, "rb"); -#else +#endif // _WIN32 +} + +#ifdef _WIN32 +void TcpConnectionImpl::sendFile(const wchar_t *fileName, + size_t offset, + size_t length) +{ + assert(fileName); FILE *fp; - if (fopen_s(&fp, fileName, "rb") != 0) - { +#ifndef _MSC_VER + fp = _wfopen(fileName, L"rb"); +#else // _MSC_VER + if (_wfopen_s(&fp, fileName, L"rb") != 0) fp = nullptr; - } -#endif +#endif // _MSC_VER if (fp == nullptr) { LOG_SYSERR << fileName << " open error"; @@ -1313,8 +1328,8 @@ void TcpConnectionImpl::sendFile(const char *fileName, if (length == 0) { - struct stat filestat; - if (stat(fileName, &filestat) < 0) + struct _stati64 filestat; + if (_wstati64(fileName, &filestat) < 0) { LOG_SYSERR << fileName << " stat error"; fclose(fp); @@ -1324,8 +1339,8 @@ void TcpConnectionImpl::sendFile(const char *fileName, } sendFile(fp, offset, length); -#endif } +#endif // _WIN32 #ifndef _WIN32 void TcpConnectionImpl::sendFile(int sfd, size_t offset, size_t length) diff --git a/trantor/net/inner/TcpConnectionImpl.h b/trantor/net/inner/TcpConnectionImpl.h old mode 100644 new mode 100755 index cad3dfa4..49b568de --- a/trantor/net/inner/TcpConnectionImpl.h +++ b/trantor/net/inner/TcpConnectionImpl.h @@ -115,6 +115,11 @@ class TcpConnectionImpl : public TcpConnection, virtual void sendFile(const char *fileName, size_t offset = 0, size_t length = 0) override; +#ifdef _WIN32 + virtual void sendFile(const wchar_t *fileName, + size_t offset = 0, + size_t length = 0) override; +#endif // WIN32 virtual const InetAddress &localAddr() const override { diff --git a/trantor/utils/AsyncFileLogger.cc b/trantor/utils/AsyncFileLogger.cc index e754d489..d43d7833 100644 --- a/trantor/utils/AsyncFileLogger.cc +++ b/trantor/utils/AsyncFileLogger.cc @@ -18,6 +18,8 @@ #ifdef __linux__ #include #endif +#else +#include #endif #include #include @@ -184,7 +186,17 @@ AsyncFileLogger::LoggerFile::LoggerFile(const std::string &filePath, #ifndef _MSC_VER fp_ = fopen(fileFullName_.c_str(), "a"); #else - fp_ = _fsopen(fileFullName_.c_str(), "a+", _SH_DENYWR); + // Convert UTF-8 file to UCS-2 + int nSizeNeeded = ::MultiByteToWideChar( + CP_UTF8, 0, &fileFullName_[0], (int)fileFullName_.size(), NULL, 0); + std::wstring wFullName(nSizeNeeded, 0); + ::MultiByteToWideChar(CP_UTF8, + 0, + &fileFullName_[0], + (int)fileFullName_.size(), + &wFullName[0], + nSizeNeeded); + fp_ = _wfsopen(wFullName.c_str(), L"a+", _SH_DENYWR); #endif if (fp_ == nullptr) { @@ -232,7 +244,30 @@ AsyncFileLogger::LoggerFile::~LoggerFile() filePath_ + fileBaseName_ + "." + creationDate_.toCustomedFormattedString("%y%m%d-%H%M%S") + std::string(seq) + fileExtName_; +#ifndef _WIN32 rename(fileFullName_.c_str(), newName.c_str()); +#else // _WIN32 + // Convert UTF-8 file to UCS-2 + int nSizeNeeded = ::MultiByteToWideChar( + CP_UTF8, 0, &fileFullName_[0], (int)fileFullName_.size(), NULL, 0); + std::wstring wFullName(nSizeNeeded, 0); + ::MultiByteToWideChar(CP_UTF8, + 0, + &fileFullName_[0], + (int)fileFullName_.size(), + &wFullName[0], + nSizeNeeded); + nSizeNeeded = ::MultiByteToWideChar( + CP_UTF8, 0, &newName[0], (int)newName.size(), NULL, 0); + std::wstring wNewName(nSizeNeeded, 0); + ::MultiByteToWideChar(CP_UTF8, + 0, + &newName[0], + (int)newName.size(), + &wNewName[0], + nSizeNeeded); + _wrename(wFullName.c_str(), wNewName.c_str()); +#endif // _WIN32 } } From 5c8c12039c0d7c0876f2b3bfb580922427efb4e5 Mon Sep 17 00:00:00 2001 From: Greisberger Christophe Date: Thu, 15 Jul 2021 03:57:47 +0200 Subject: [PATCH 023/182] Feature/support windows unicode paths (#158) --- CMakeLists.txt | 56 ++++++- trantor/net/TcpConnection.h | 4 +- trantor/net/inner/TcpConnectionImpl.cc | 19 +-- trantor/net/inner/TcpConnectionImpl.h | 2 - trantor/tests/CMakeLists.txt | 4 +- trantor/tests/PathConversionTest.cc | 89 ++++++++++ trantor/unittests/CMakeLists.txt | 4 +- trantor/unittests/stringEncodingUnittest.cc | 149 +++++++++++++++++ trantor/utils/AsyncFileLogger.cc | 31 +--- trantor/utils/Utilities.cc | 136 +++++++++++++++ trantor/utils/Utilities.h | 176 ++++++++++++++++++++ 11 files changed, 615 insertions(+), 55 deletions(-) create mode 100755 trantor/tests/PathConversionTest.cc create mode 100755 trantor/unittests/stringEncodingUnittest.cc mode change 100644 => 100755 trantor/utils/AsyncFileLogger.cc create mode 100755 trantor/utils/Utilities.cc create mode 100644 trantor/utils/Utilities.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 01b39344..15c9c54c 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,6 +75,7 @@ set(TRANTOR_SOURCES trantor/utils/MsgBuffer.cc trantor/utils/SerialTaskQueue.cc trantor/utils/TimingWheel.cc + trantor/utils/Utilities.cc trantor/net/EventLoop.cc trantor/net/EventLoopThread.cc trantor/net/EventLoopThreadPool.cc @@ -91,10 +92,27 @@ set(TRANTOR_SOURCES trantor/net/inner/TimerQueue.cc trantor/net/inner/poller/EpollPoller.cc trantor/net/inner/poller/KQueue.cc) +set(private_headers + trantor/net/inner/Acceptor.h + trantor/net/inner/Connector.h + trantor/net/inner/Poller.h + trantor/net/inner/Socket.h + trantor/net/inner/TcpConnectionImpl.h + trantor/net/inner/Timer.h + trantor/net/inner/TimerQueue.h + trantor/net/inner/poller/EpollPoller.h + trantor/net/inner/poller/KQueue.h) + if(WIN32) - set(TRANTOR_SOURCES ${TRANTOR_SOURCES} trantor/utils/WindowsSupport.cc) - set(TRANTOR_SOURCES ${TRANTOR_SOURCES} third_party/wepoll/Wepoll.c) + set(TRANTOR_SOURCES + ${TRANTOR_SOURCES} + third_party/wepoll/Wepoll.c + trantor/utils/WindowsSupport.cc) + set(private_headers + ${private_headers} + third_party/wepoll/Wepoll.h + trantor/utils/WindowsSupport.h) endif(WIN32) find_package(OpenSSL) @@ -107,11 +125,20 @@ find_package(c-ares) if(c-ares_FOUND) message(STATUS "c-ares found!") target_link_libraries(${PROJECT_NAME} PRIVATE c-ares_lib) - set(TRANTOR_SOURCES ${TRANTOR_SOURCES} trantor/net/inner/AresResolver.cc) -else() - set(TRANTOR_SOURCES ${TRANTOR_SOURCES} trantor/net/inner/NormalResolver.cc) -endif() -target_sources(${PROJECT_NAME} PRIVATE ${TRANTOR_SOURCES}) + set(TRANTOR_SOURCES + ${TRANTOR_SOURCES} + trantor/net/inner/AresResolver.cc) + set(private_headers + ${private_headers} + trantor/net/inner/AresResolver.h) +else(c-ares_FOUND) + set(TRANTOR_SOURCES + ${TRANTOR_SOURCES} + trantor/net/inner/NormalResolver.cc) + set(private_headers + ${private_headers} + trantor/net/inner/NormalResolver.h) +endif(c-ares_FOUND) find_package(Threads) target_link_libraries(${PROJECT_NAME} PUBLIC Threads::Threads) @@ -170,13 +197,26 @@ set(public_utils_headers trantor/utils/ObjectPool.h trantor/utils/SerialTaskQueue.h trantor/utils/TaskQueue.h - trantor/utils/TimingWheel.h) + trantor/utils/TimingWheel.h + trantor/utils/Utilities.h) + +target_sources(${PROJECT_NAME} PRIVATE + ${TRANTOR_SOURCES} + ${CMAKE_CURRENT_BINARY_DIR}/exports/trantor/exports.h + ${public_net_headers} + ${public_utils_headers} + ${private_headers}) source_group("Public API" FILES + ${CMAKE_CURRENT_BINARY_DIR}/exports/trantor/exports.h ${public_net_headers} ${public_utils_headers}) +source_group("Private Headers" + FILES + ${private_headers}) + install(TARGETS trantor # IMPORTANT: Add the trantor library to the "export-set" EXPORT TrantorTargets diff --git a/trantor/net/TcpConnection.h b/trantor/net/TcpConnection.h index 24b3c800..88151d80 100755 --- a/trantor/net/TcpConnection.h +++ b/trantor/net/TcpConnection.h @@ -66,18 +66,16 @@ class TRANTOR_EXPORT TcpConnection virtual void sendFile(const char *fileName, size_t offset = 0, size_t length = 0) = 0; -#ifdef _WIN32 /** * @brief Send a file to the peer. * - * @param fileName in UCS-2 (windows only) + * @param fileName in wide string (eg. windows native UCS-2) * @param offset * @param length */ virtual void sendFile(const wchar_t *fileName, size_t offset = 0, size_t length = 0) = 0; -#endif // WIN32 /** * @brief Get the local address of the connection. diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index 2565bbf0..e78895f8 100755 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -15,6 +15,7 @@ #include "TcpConnectionImpl.h" #include "Socket.h" #include "Channel.h" +#include #ifdef __linux__ #include #endif @@ -1271,17 +1272,7 @@ void TcpConnectionImpl::sendFile(const char *fileName, { assert(fileName); #ifdef _WIN32 - // Convert UTF-8 file path to UCS-2 - int nSizeNeeded = ::MultiByteToWideChar( - CP_UTF8, 0, fileName, (int)strnlen_s(fileName, MAX_PATH), NULL, 0); - std::wstring wFileName(nSizeNeeded, 0); - ::MultiByteToWideChar(CP_UTF8, - 0, - fileName, - (int)strnlen_s(fileName, MAX_PATH), - &wFileName[0], - nSizeNeeded); - sendFile(wFileName.c_str(), offset, length); + sendFile(utils::toNativePath(fileName).c_str(), offset, length); #else // _WIN32 int fd = open(fileName, O_RDONLY); @@ -1307,12 +1298,14 @@ void TcpConnectionImpl::sendFile(const char *fileName, #endif // _WIN32 } -#ifdef _WIN32 void TcpConnectionImpl::sendFile(const wchar_t *fileName, size_t offset, size_t length) { assert(fileName); +#ifndef _WIN32 + sendFile(utils::toNativePath(fileName).c_str(), offset, length); +#else // _WIN32 FILE *fp; #ifndef _MSC_VER fp = _wfopen(fileName, L"rb"); @@ -1339,8 +1332,8 @@ void TcpConnectionImpl::sendFile(const wchar_t *fileName, } sendFile(fp, offset, length); -} #endif // _WIN32 +} #ifndef _WIN32 void TcpConnectionImpl::sendFile(int sfd, size_t offset, size_t length) diff --git a/trantor/net/inner/TcpConnectionImpl.h b/trantor/net/inner/TcpConnectionImpl.h index 49b568de..baf8dc90 100755 --- a/trantor/net/inner/TcpConnectionImpl.h +++ b/trantor/net/inner/TcpConnectionImpl.h @@ -115,11 +115,9 @@ class TcpConnectionImpl : public TcpConnection, virtual void sendFile(const char *fileName, size_t offset = 0, size_t length = 0) override; -#ifdef _WIN32 virtual void sendFile(const wchar_t *fileName, size_t offset = 0, size_t length = 0) override; -#endif // WIN32 virtual const InetAddress &localAddr() const override { diff --git a/trantor/tests/CMakeLists.txt b/trantor/tests/CMakeLists.txt index 180f5565..052dd49e 100644 --- a/trantor/tests/CMakeLists.txt +++ b/trantor/tests/CMakeLists.txt @@ -19,6 +19,7 @@ add_executable(dns_test DnsTest.cc) add_executable(delayed_ssl_server_test DelayedSSLServerTest.cc) add_executable(delayed_ssl_client_test DelayedSSLClientTest.cc) add_executable(run_on_quit_test RunOnQuitTest.cc) +add_executable(path_conversion_test PathConversionTest.cc) set(targets_list ssl_server_test ssl_client_test @@ -40,7 +41,8 @@ set(targets_list dns_test delayed_ssl_server_test delayed_ssl_client_test - run_on_quit_test) + run_on_quit_test + path_conversion_test) set_property(TARGET ${targets_list} PROPERTY CXX_STANDARD 14) set_property(TARGET ${targets_list} PROPERTY CXX_STANDARD_REQUIRED ON) diff --git a/trantor/tests/PathConversionTest.cc b/trantor/tests/PathConversionTest.cc new file mode 100755 index 00000000..f5653fb9 --- /dev/null +++ b/trantor/tests/PathConversionTest.cc @@ -0,0 +1,89 @@ +#include +#include +#include + +int main() +{ + trantor::Logger::setLogLevel(trantor::Logger::kTrace); + LOG_DEBUG << "PathConversion utils test!"; + +#ifdef _WIN32 + std::string utf8PathStandard("C:/Temp/\xE4\xB8\xAD\xE6\x96\x87"); + std::string utf8PathAlt("C:\\Temp\\\xE4\xB8\xAD\xE6\x96\x87"); + std::wstring widePathStandard(L"C:\\Temp\\\u4E2D\u6587"); + std::wstring widePathAlt(L"C:/Temp/\u4E2D\u6587"); + std::string utf8WidePathStandard{utf8PathAlt}; + std::string utf8WidePathAlt{utf8PathStandard}; +#else // _WIN32 + std::string utf8PathStandard("/tmp/\xE4\xB8\xAD\xE6\x96\x87"); + std::string utf8PathAlt( + "\\tmp\\\xE4\xB8\xAD\xE6\x96\x87"); // Invalid, won't be changed + std::wstring widePathStandard(L"/tmp/\u4E2D\u6587"); + std::wstring widePathAlt(L"\\tmp\\\u4E2D\u6587"); + std::string utf8WidePathStandard{utf8PathStandard}; + std::string utf8WidePathAlt{utf8PathAlt}; +#endif // _WIN32 + + // 1. Check from/to UTF-8 +#ifdef _WIN32 + if (utf8PathAlt != trantor::utils::toUtf8(widePathStandard)) +#else // _WIN32 + if (utf8PathStandard != trantor::utils::toUtf8(widePathStandard)) +#endif // _WIN32 + LOG_ERROR << "Error converting " << utf8WidePathStandard + << " from wide string to utf-8"; +#ifdef _WIN32 + if (utf8PathStandard != trantor::utils::toUtf8(widePathAlt)) +#else // _WIN32 + if (utf8PathAlt != trantor::utils::toUtf8(widePathAlt)) +#endif // _WIN32 + LOG_ERROR << "Error converting " << utf8WidePathAlt + << " from wide string to utf-8"; +#ifdef _WIN32 + if (widePathAlt != trantor::utils::fromUtf8(utf8PathStandard)) +#else // _WIN32 + if (widePathStandard != trantor::utils::fromUtf8(utf8PathStandard)) +#endif // _WIN32 + LOG_ERROR << "Error converting " << utf8PathStandard + << " from utf-8 to wide string"; +#ifdef _WIN32 + if (widePathStandard != trantor::utils::fromUtf8(utf8PathAlt)) +#else // _WIN32 + if (widePathAlt != trantor::utils::fromUtf8(utf8PathAlt)) +#endif // _WIN32 + LOG_ERROR << "Error converting " << utf8PathAlt + << " from utf-8 to wide string"; + + // 2. Check path conversion. Note: The directory separator should be changed + // on Windows only + if (utf8PathStandard != trantor::utils::fromWidePath(widePathStandard)) + LOG_ERROR << "Error converting " << utf8WidePathStandard + << " from wide path to utf-8"; +#ifdef _WIN32 + if (utf8PathStandard != trantor::utils::fromWidePath(widePathAlt)) +#else // _WIN32 + if (utf8PathAlt != trantor::utils::fromWidePath(widePathAlt)) +#endif // _WIN32 + LOG_ERROR << "Error converting " << utf8WidePathAlt + << " from wide path to utf-8"; + if (widePathStandard != trantor::utils::toWidePath(utf8PathStandard)) + LOG_ERROR << "Error converting " << utf8WidePathStandard + << " from utf-8 to wide path"; +#ifdef _WIN32 + if (widePathStandard != trantor::utils::toWidePath(utf8PathAlt)) +#else // _WIN32 + if (widePathAlt != trantor::utils::toWidePath(utf8PathAlt)) +#endif // _WIN32 + LOG_ERROR << "Error converting " << utf8PathAlt + << " from utf-8 to wide path"; + + // 3. From/to native path + auto nativePath1 = trantor::utils::toNativePath(widePathStandard); + auto nativePath2 = trantor::utils::toNativePath(utf8PathStandard); + if (nativePath1 != nativePath2) + LOG_ERROR << "Error converting " << utf8PathStandard + << " to native path"; + if (utf8PathStandard != trantor::utils::fromNativePath(nativePath1)) + LOG_ERROR << "Error converting " << utf8PathStandard + << " from native to utf-8 path"; +} diff --git a/trantor/unittests/CMakeLists.txt b/trantor/unittests/CMakeLists.txt index 4c7f4921..ef643679 100644 --- a/trantor/unittests/CMakeLists.txt +++ b/trantor/unittests/CMakeLists.txt @@ -3,11 +3,13 @@ add_executable(msgbuffer_unittest MsgBufferUnittest.cc) add_executable(inetaddress_unittest InetAddressUnittest.cc) add_executable(date_unittest DateUnittest.cc) add_executable(split_string_unittest splitStringUnittest.cc) +add_executable(string_encoding_unittest stringEncodingUnittest.cc) set(UNITTEST_TARGETS msgbuffer_unittest inetaddress_unittest date_unittest - split_string_unittest) + split_string_unittest + string_encoding_unittest) set_property(TARGET ${UNITTEST_TARGETS} PROPERTY CXX_STANDARD 14) set_property(TARGET ${UNITTEST_TARGETS} PROPERTY CXX_STANDARD_REQUIRED ON) set_property(TARGET ${UNITTEST_TARGETS} PROPERTY CXX_EXTENSIONS OFF) diff --git a/trantor/unittests/stringEncodingUnittest.cc b/trantor/unittests/stringEncodingUnittest.cc new file mode 100755 index 00000000..e50761cb --- /dev/null +++ b/trantor/unittests/stringEncodingUnittest.cc @@ -0,0 +1,149 @@ +#include +#include +#include +using namespace trantor; + +const std::string utf8Path("C:/Temp/\xE4\xB8\xAD\xE6\x96\x87"); +const std::string utf8AltPath("C:\\Temp\\\xE4\xB8\xAD\xE6\x96\x87"); +const std::wstring widePath(L"C:/Temp/\u4E2D\u6587"); +const std::wstring windowsPath(L"C:\\Temp\\\u4E2D\u6587"); +#ifdef _WIN32 +const std::wstring nativePath(windowsPath); +#else // _WIN32 +const std::string nativePath(utf8Path); +#endif // _WIN32 + +TEST(pathConversion, fromUtf8a) +{ + auto out = utils::fromUtf8(utf8Path); + EXPECT_EQ(out, widePath) + << "Error converting " << utf8Path << " to wide string"; +} +TEST(pathConversion, fromUtf8b) +{ + auto out = utils::fromUtf8(utf8AltPath); + EXPECT_EQ(out, windowsPath) + << "Error converting " << utf8AltPath << " to wide string"; +} +TEST(pathConversion, toUtf8a) +{ + auto out = utils::toUtf8(widePath); + EXPECT_EQ(out, utf8Path) + << "Error converting " << widePath << " from wide string to utf-8"; +} +TEST(pathConversion, toUtf8b) +{ + auto out = utils::toUtf8(windowsPath); + EXPECT_EQ(out, utf8AltPath) + << "Error converting " << windowsPath << " from wide string to utf-8"; +} +TEST(pathConversion, fromWidePath1) +{ + auto out = utils::fromWidePath(widePath); + EXPECT_EQ(out, utf8Path) + << "Error converting " << widePath << " from wide path to utf-8"; +} +TEST(pathConversion, fromWidePath2) +{ + auto out = utils::fromWidePath(windowsPath); +#ifdef _WIN32 + EXPECT_EQ(out, utf8Path) +#else // _WIN32 + EXPECT_EQ(out, utf8AltPath) +#endif // _WIN32 + << "Error converting " << windowsPath << " from wide path to utf-8"; +} +TEST(pathConversion, toWidePath1) +{ + auto out = utils::toWidePath(utf8Path); +#ifdef _WIN32 + EXPECT_EQ(out, windowsPath) +#else // _WIN32 + EXPECT_EQ(out, widePath) +#endif // _WIN32 + << "Error converting " << utf8Path << " from utf-8 path to wide string"; +} +TEST(pathConversion, toWidePath2) +{ + auto out = utils::toWidePath(utf8AltPath); + EXPECT_EQ(out, windowsPath) << "Error converting " << utf8AltPath + << " from utf-8 path to wide string"; +} +TEST(pathConversion, toNativePath1) +{ + auto out = utils::toNativePath(utf8Path); + EXPECT_EQ(out, nativePath) + << "Error converting " << utf8Path << " from utf-8 path to native path"; +} +TEST(pathConversion, toNativePath2) +{ + auto out = utils::toNativePath(utf8AltPath); +#ifdef _WIN32 + EXPECT_EQ(out, windowsPath) << "Error converting " << utf8AltPath + << " from utf-8 path to wide string"; +#else // _WIN32 + EXPECT_EQ(out, utf8AltPath) << "Error converting " << utf8AltPath + << " from utf-8 path (should be a noop)"; +#endif // _WIN32 +} +TEST(pathConversion, toNativePath3) +{ + auto out = utils::toNativePath(windowsPath); +#ifdef _WIN32 + EXPECT_EQ(out, windowsPath) << "Error converting " << windowsPath + << " from wide path (should be a noop)"; +#else // _WIN32 + EXPECT_EQ(out, utf8AltPath) + << "Error converting " << windowsPath << " from wide path to utf-8"; +#endif // _WIN32 +} +TEST(pathConversion, toNativePath4) +{ + auto out = utils::toNativePath(widePath); +#ifdef _WIN32 + EXPECT_EQ(out, widePath) << "Error converting " << widePath + << " from wide path (should be a noop)"; +#else // _WIN32 + EXPECT_EQ(out, utf8Path) + << "Error converting " << widePath << " from wide path to utf-8"; +#endif // _WIN32 +} +TEST(pathConversion, fromNativePath1) +{ + auto out = utils::fromNativePath(nativePath); + EXPECT_EQ(out, utf8Path) + << "Error converting " << nativePath << " from native path to utf-8"; +} +TEST(pathConversion, fromNativePath2) +{ + auto out = utils::fromNativePath(utf8Path); + EXPECT_EQ(out, utf8Path) << "Error converting " << utf8Path + << " from utf-8 path (should be a noop)"; +} +TEST(pathConversion, fromNativePath3) +{ + auto out = utils::fromNativePath(utf8AltPath); + EXPECT_EQ(out, utf8AltPath) << "Error converting " << utf8AltPath + << " from utf-8 path (should be a noop)"; +} +TEST(pathConversion, fromNativePath4) +{ + auto out = utils::fromNativePath(windowsPath); +#ifdef _WIN32 + EXPECT_EQ(out, utf8Path) +#else // _WIN32 + EXPECT_EQ(out, utf8AltPath) +#endif // _WIN32 + << "Error converting " << windowsPath << " from wide path to utf-8"; +} +TEST(pathConversion, fromNativePath5) +{ + auto out = utils::fromNativePath(widePath); + EXPECT_EQ(out, utf8Path) + << "Error converting " << widePath << " from wide path to utf-8"; +} +int main(int argc, char **argv) +{ + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/trantor/utils/AsyncFileLogger.cc b/trantor/utils/AsyncFileLogger.cc old mode 100644 new mode 100755 index d43d7833..067364f5 --- a/trantor/utils/AsyncFileLogger.cc +++ b/trantor/utils/AsyncFileLogger.cc @@ -13,6 +13,7 @@ */ #include +#include #ifndef _WIN32 #include #ifdef __linux__ @@ -187,15 +188,7 @@ AsyncFileLogger::LoggerFile::LoggerFile(const std::string &filePath, fp_ = fopen(fileFullName_.c_str(), "a"); #else // Convert UTF-8 file to UCS-2 - int nSizeNeeded = ::MultiByteToWideChar( - CP_UTF8, 0, &fileFullName_[0], (int)fileFullName_.size(), NULL, 0); - std::wstring wFullName(nSizeNeeded, 0); - ::MultiByteToWideChar(CP_UTF8, - 0, - &fileFullName_[0], - (int)fileFullName_.size(), - &wFullName[0], - nSizeNeeded); + auto wFullName{utils::toNativePath(fileFullName_)}; fp_ = _wfsopen(wFullName.c_str(), L"a+", _SH_DENYWR); #endif if (fp_ == nullptr) @@ -248,24 +241,8 @@ AsyncFileLogger::LoggerFile::~LoggerFile() rename(fileFullName_.c_str(), newName.c_str()); #else // _WIN32 // Convert UTF-8 file to UCS-2 - int nSizeNeeded = ::MultiByteToWideChar( - CP_UTF8, 0, &fileFullName_[0], (int)fileFullName_.size(), NULL, 0); - std::wstring wFullName(nSizeNeeded, 0); - ::MultiByteToWideChar(CP_UTF8, - 0, - &fileFullName_[0], - (int)fileFullName_.size(), - &wFullName[0], - nSizeNeeded); - nSizeNeeded = ::MultiByteToWideChar( - CP_UTF8, 0, &newName[0], (int)newName.size(), NULL, 0); - std::wstring wNewName(nSizeNeeded, 0); - ::MultiByteToWideChar(CP_UTF8, - 0, - &newName[0], - (int)newName.size(), - &wNewName[0], - nSizeNeeded); + auto wFullName{utils::toNativePath(fileFullName_)}; + auto wNewName{utils::toNativePath(newName)}; _wrename(wFullName.c_str(), wNewName.c_str()); #endif // _WIN32 } diff --git a/trantor/utils/Utilities.cc b/trantor/utils/Utilities.cc new file mode 100755 index 00000000..469f6eb9 --- /dev/null +++ b/trantor/utils/Utilities.cc @@ -0,0 +1,136 @@ +/** + * + * @file Utilities.cc + * @author An Tao + * + * Copyright 2018, An Tao. All rights reserved. + * https://github.com/an-tao/drogon + * Use of this source code is governed by a MIT license + * that can be found in the License file. + * + * Drogon + * + */ + +#include "Utilities.h" +#ifdef _WIN32 +#include +#include +#else // _WIN32 +#if __cplusplus < 201103L || __cplusplus >= 201703L +#include +#include +#else // __cplusplus +#include +#include +#endif // __cplusplus +#endif // _WIN32 + +namespace trantor +{ +namespace utils +{ +std::string toUtf8(const std::wstring &wstr) +{ + if (wstr.empty()) + return {}; + + std::string strTo; +#ifdef _WIN32 + int nSizeNeeded = ::WideCharToMultiByte( + CP_UTF8, 0, &wstr[0], (int)wstr.size(), NULL, 0, NULL, NULL); + strTo.resize(nSizeNeeded, 0); + ::WideCharToMultiByte(CP_UTF8, + 0, + &wstr[0], + (int)wstr.size(), + &strTo[0], + nSizeNeeded, + NULL, + NULL); +#else // _WIN32 +#if __cplusplus < 201103L || __cplusplus >= 201703L + // Note: Introduced in c++11 and deprecated with c++17. + // Revert to C99 code since there no replacement yet + strTo.resize(3 * wstr.length(), 0); + locale_t utf8 = newlocale(LC_ALL_MASK, "C.UTF-8", NULL); + if (!utf8) + utf8 = newlocale(LC_ALL_MASK, "C.utf-8", NULL); + if (!utf8) + utf8 = newlocale(LC_ALL_MASK, "C.UTF8", NULL); + if (!utf8) + utf8 = newlocale(LC_ALL_MASK, "C.utf8", NULL); + auto nLen = wcstombs_l(&strTo[0], wstr.c_str(), strTo.length(), utf8); + strTo.resize(nLen); + freelocale(utf8); +#else // c++11 to c++14 + std::wstring_convert, wchar_t> utf8conv; + strTo = utf8conv.to_bytes(wstr); +#endif // __cplusplus +#endif // _WIN32 + return strTo; +} +std::wstring fromUtf8(const std::string &str) +{ + if (str.empty()) + return {}; + std::wstring wstrTo; +#ifdef _WIN32 + int nSizeNeeded = + ::MultiByteToWideChar(CP_UTF8, 0, &str[0], (int)str.size(), NULL, 0); + wstrTo.resize(nSizeNeeded, 0); + ::MultiByteToWideChar( + CP_UTF8, 0, &str[0], (int)str.size(), &wstrTo[0], nSizeNeeded); +#else // _WIN32 +#if __cplusplus < 201103L || __cplusplus >= 201703L + // Note: Introduced in c++11 and deprecated with c++17. + // Revert to C99 code since there no replacement yet + wstrTo.resize(str.length(), 0); + locale_t utf8 = newlocale(LC_ALL_MASK, "en_US.UTF-8", NULL); + if (!utf8) + utf8 = newlocale(LC_ALL_MASK, "C.utf-8", NULL); + if (!utf8) + utf8 = newlocale(LC_ALL_MASK, "C.UTF8", NULL); + if (!utf8) + utf8 = newlocale(LC_ALL_MASK, "C.utf8", NULL); + auto nLen = mbstowcs_l(&wstrTo[0], str.c_str(), wstrTo.length(), utf8); + wstrTo.resize(nLen); + freelocale(utf8); +#else // c++11 to c++14 + std::wstring_convert, wchar_t> utf8conv; + try + { + wstrTo = utf8conv.from_bytes(str); + } + catch (...) // Should never fail if str valid UTF-8 + { + } +#endif // __cplusplus +#endif // _WIN32 + return wstrTo; +} + +std::wstring toWidePath(const std::string &strUtf8Path) +{ + auto wstrPath{fromUtf8(strUtf8Path)}; +#ifdef _WIN32 + // Not needed: normalize path (just replaces '/' with '\') + std::replace(wstrPath.begin(), wstrPath.end(), L'/', L'\\'); +#endif // _WIN32 + return wstrPath; +} + +std::string fromWidePath(const std::wstring &wstrPath) +{ +#ifdef _WIN32 + auto srcPath{wstrPath}; + // Not needed: to portable path (just replaces '\' with '/') + std::replace(srcPath.begin(), srcPath.end(), L'\\', L'/'); +#else // _WIN32 + auto &srcPath{wstrPath}; +#endif // _WIN32 + return toUtf8(srcPath); +} + +} // namespace utils +} // namespace trantor diff --git a/trantor/utils/Utilities.h b/trantor/utils/Utilities.h new file mode 100644 index 00000000..a616e2b2 --- /dev/null +++ b/trantor/utils/Utilities.h @@ -0,0 +1,176 @@ +/** + * + * @file Utilities.h + * @author An Tao + * + * Copyright 2018, An Tao. All rights reserved. + * https://github.com/an-tao/drogon + * Use of this source code is governed by a MIT license + * that can be found in the License file. + * + * Drogon + * + */ + +#pragma once + +#include +#include + +namespace trantor +{ +/** + * @brief Trantor helper functions. + * + */ +namespace utils +{ +/** + * @brief Convert a wide string to a UTF-8. + * @details UCS2 on Windows, UTF-32 on Linux & Mac + * + * @param str String to convert + * + * @return converted string. + */ +TRANTOR_EXPORT std::string toUtf8(const std::wstring &wstr); +/** + * @brief Convert a UTF-8 string to a wide string. + * @details UCS2 on Windows, UTF-32 on Linux & Mac + * + * @param str String to convert + * + * @return converted string. + */ +TRANTOR_EXPORT std::wstring fromUtf8(const std::string &str); + +/** + * @details Convert a wide string path with arbitrary directory separators + * to a UTF-8 portable path for use with trantor. + * + * This is a helper, mainly for Windows and multi-platform projects. + * + * @note On Windows, backslash directory separators are converted to slash to + * keep portable paths. + * + * @remarks On other OSes, backslashes are not converted to slash, since they + * are valid characters for directory/file names. + * + * @param strPath Wide string path. + * + * @return std::string UTF-8 path, with slash directory separator. + */ +TRANTOR_EXPORT std::string fromWidePath(const std::wstring &strPath); +/** + * @details Convert a UTF-8 path with arbitrary directory separator to a wide + * string path. + * + * This is a helper, mainly for Windows and multi-platform projects. + * + * @note On Windows, slash directory separators are converted to backslash. + * Although it accepts both slash and backslash as directory separator in its + * API, it is better to stick to its standard. + + * @remarks On other OSes, slashes are not converted to backslashes, since they + * are not interpreted as directory separators and are valid characters for + * directory/file names. + * + * @param strUtf8Path Ascii path considered as being UTF-8. + * + * @return std::wstring path with, on windows, standard backslash directory + * separator to stick to its standard. + */ +TRANTOR_EXPORT std::wstring toWidePath(const std::string &strUtf8Path); + +// Helpers for multi-platform development +// OS dependent +#ifdef _WIN32 +/** + * @details Convert an UTF-8 path to a native path. + * + * This is a helper for Windows and multi-platform projects. + * + * Converts the path from UTF-8 to wide string and replaces slash directory + * separators with backslash. + * + * @remarks Although it accepts both slash and backslash as directory + * separators in its API, it is better to stick to its standard. + * + * @param strPath UTF-8 path. + * + * @return Native path suitable for the Windows unicode API. + */ +inline std::wstring toNativePath(const std::string &strPath) +{ + return toWidePath(strPath); +} +/** + * @details Convert an UTF-8 path to a native path. + * + * This is a helper for non-Windows OSes for multi-platform projects. + * + * Does nothing. + * + * @param strPath UTF-8 path. + * + * @return \p strPath. + */ +inline const std::wstring &toNativePath(const std::wstring &strPath) +{ + return strPath; +} +#else // __WIN32 +/** + * @details Convert an UTF-8 path to a native path. + * + * This is a helper for non-Windows OSes for multi-platform projects. + * + * Does nothing. + * + * @param strPath UTF-8 path. + * + * @return \p strPath. + */ +inline const std::string &toNativePath(const std::string &strPath) +{ + return strPath; +} +/** + * @details Convert an wide string path to a UTF-8 path. + * + * This is a helper, mainly for Windows and multi-platform projects. + * + * @note On Windows, backslash directory separators are converted to slash to + * keep portable paths. + + * @warning On other OSes, backslashes are not converted to slash, since they + * are valid characters for directory/file names. + * + * @param strPath wide string path. + * + * @return Generic path, with slash directory separators + */ +inline std::string toNativePath(const std::wstring &strPath) +{ + return fromWidePath(strPath); +} +#endif // _WIN32 +/** + * @note NoOP on all OSes + */ +inline const std::string &fromNativePath(const std::string &strPath) +{ + return strPath; +} +/** + * @note fromWidePath() on all OSes + */ +// Convert on all systems +inline std::string fromNativePath(const std::wstring &strPath) +{ + return fromWidePath(strPath); +} + +} // namespace utils + +} // namespace trantor From 66cc80e8be004e99d59960525ba688b5802d322f Mon Sep 17 00:00:00 2001 From: Greisberger Christophe Date: Sat, 17 Jul 2021 07:03:06 +0200 Subject: [PATCH 024/182] Add BUILD_DOC cmake option (#159) Co-authored-by: Christophe Greisberger --- CMakeLists.txt | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 15c9c54c..64976d75 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,8 @@ cmake_minimum_required(VERSION 3.5) project(trantor) +option(BUILD_DOC "Build Doxygen documentation" OFF) + list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) set(TRANTOR_MAJOR_VERSION 1) @@ -258,3 +260,36 @@ install(EXPORT TrantorTargets DESTINATION "${INSTALL_TRANTOR_CMAKE_DIR}" NAMESPACE Trantor:: COMPONENT dev) + +# Doxygen documentation +find_package(Doxygen OPTIONAL_COMPONENTS dot dia) +if(DOXYGEN_FOUND) + set(DOXYGEN_PROJECT_BRIEF "Non-blocking I/O cross-platform TCP network library, using C++14") + set(DOXYGEN_OUTPUT_DIRECTORY docs/${PROJECT_NAME}) + set(DOXYGEN_GENERATE_LATEX NO) + set(DOXYGEN_BUILTIN_STL_SUPPORT YES) + set(DOXYGEN_USE_MDFILE_AS_MAINPAGE README.md) + set(DOXYGEN_STRIP_FROM_INC_PATH ${PROJECT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/exports) + if (WIN32) + set(DOXYGEN_PREDEFINED _WIN32) + endif(WIN32) + doxygen_add_docs(doc_${PROJECT_NAME} + README.md + ChangeLog.md + ${public_net_headers} + ${public_utils_headers} + COMMENT "Generate documentation") + if(NOT TARGET doc) + add_custom_target(doc) + endif() + add_dependencies(doc doc_${PROJECT_NAME}) + if (BUILD_DOC) + add_dependencies(${PROJECT_NAME} doc_${PROJECT_NAME}) + # Don't install twice, so limit to Debug (assume developer) + install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/docs/${PROJECT_NAME} + TYPE DOC + CONFIGURATIONS Debug) + endif(BUILD_DOC) +endif(DOXYGEN_FOUND) + From 486f3627b7e54358d804eb2fdb0c4f2a52aed2f0 Mon Sep 17 00:00:00 2001 From: Martin Chang Date: Thu, 22 Jul 2021 16:07:48 +0800 Subject: [PATCH 025/182] Use make_shared instead of shared_ptr(new) (#160) --- trantor/net/inner/TcpConnectionImpl.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index e78895f8..d762a7d7 100755 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -967,7 +967,7 @@ void TcpConnectionImpl::sendInLoop(const char *buffer, size_t length) { if (writeBufferList_.empty()) { - BufferNodePtr node(new BufferNode); + BufferNodePtr node = std::make_shared(); node->msgBuffer_ = std::make_shared(); writeBufferList_.push_back(std::move(node)); } @@ -977,7 +977,7 @@ void TcpConnectionImpl::sendInLoop(const char *buffer, size_t length) else if (writeBufferList_.back()->sendFp_) #endif { - BufferNodePtr node(new BufferNode); + BufferNodePtr node = std::make_shared(); node->msgBuffer_ = std::make_shared(); writeBufferList_.push_back(std::move(node)); } @@ -1344,11 +1344,11 @@ void TcpConnectionImpl::sendFile(FILE *fp, size_t offset, size_t length) assert(length > 0); #ifndef _WIN32 assert(sfd >= 0); - BufferNodePtr node(new BufferNode); + BufferNodePtr node = std::make_shared(); node->sendFd_ = sfd; #else assert(fp); - BufferNodePtr node(new BufferNode); + BufferNodePtr node = std::make_shared(); node->sendFp_ = fp; #endif node->offset_ = static_cast(offset); From 0923f6d926075a707c80fb13d2a7d9e808ec2f07 Mon Sep 17 00:00:00 2001 From: Martin Chang Date: Mon, 26 Jul 2021 23:29:04 +0800 Subject: [PATCH 026/182] Detect and handle MinGW (#161) --- trantor/utils/AsyncFileLogger.cc | 6 +- trantor/utils/Utilities.h | 352 +++++++++++++++---------------- 2 files changed, 179 insertions(+), 179 deletions(-) diff --git a/trantor/utils/AsyncFileLogger.cc b/trantor/utils/AsyncFileLogger.cc index 067364f5..3b97dae0 100755 --- a/trantor/utils/AsyncFileLogger.cc +++ b/trantor/utils/AsyncFileLogger.cc @@ -237,14 +237,14 @@ AsyncFileLogger::LoggerFile::~LoggerFile() filePath_ + fileBaseName_ + "." + creationDate_.toCustomedFormattedString("%y%m%d-%H%M%S") + std::string(seq) + fileExtName_; -#ifndef _WIN32 +#if !defined(_WIN32) || defined(__MINGW32__) rename(fileFullName_.c_str(), newName.c_str()); -#else // _WIN32 +#else // Convert UTF-8 file to UCS-2 auto wFullName{utils::toNativePath(fileFullName_)}; auto wNewName{utils::toNativePath(newName)}; _wrename(wFullName.c_str(), wNewName.c_str()); -#endif // _WIN32 +#endif } } diff --git a/trantor/utils/Utilities.h b/trantor/utils/Utilities.h index a616e2b2..0182b1ef 100644 --- a/trantor/utils/Utilities.h +++ b/trantor/utils/Utilities.h @@ -1,176 +1,176 @@ -/** - * - * @file Utilities.h - * @author An Tao - * - * Copyright 2018, An Tao. All rights reserved. - * https://github.com/an-tao/drogon - * Use of this source code is governed by a MIT license - * that can be found in the License file. - * - * Drogon - * - */ - -#pragma once - -#include -#include - -namespace trantor -{ -/** - * @brief Trantor helper functions. - * - */ -namespace utils -{ -/** - * @brief Convert a wide string to a UTF-8. - * @details UCS2 on Windows, UTF-32 on Linux & Mac - * - * @param str String to convert - * - * @return converted string. - */ -TRANTOR_EXPORT std::string toUtf8(const std::wstring &wstr); -/** - * @brief Convert a UTF-8 string to a wide string. - * @details UCS2 on Windows, UTF-32 on Linux & Mac - * - * @param str String to convert - * - * @return converted string. - */ -TRANTOR_EXPORT std::wstring fromUtf8(const std::string &str); - -/** - * @details Convert a wide string path with arbitrary directory separators - * to a UTF-8 portable path for use with trantor. - * - * This is a helper, mainly for Windows and multi-platform projects. - * - * @note On Windows, backslash directory separators are converted to slash to - * keep portable paths. - * - * @remarks On other OSes, backslashes are not converted to slash, since they - * are valid characters for directory/file names. - * - * @param strPath Wide string path. - * - * @return std::string UTF-8 path, with slash directory separator. - */ -TRANTOR_EXPORT std::string fromWidePath(const std::wstring &strPath); -/** - * @details Convert a UTF-8 path with arbitrary directory separator to a wide - * string path. - * - * This is a helper, mainly for Windows and multi-platform projects. - * - * @note On Windows, slash directory separators are converted to backslash. - * Although it accepts both slash and backslash as directory separator in its - * API, it is better to stick to its standard. - - * @remarks On other OSes, slashes are not converted to backslashes, since they - * are not interpreted as directory separators and are valid characters for - * directory/file names. - * - * @param strUtf8Path Ascii path considered as being UTF-8. - * - * @return std::wstring path with, on windows, standard backslash directory - * separator to stick to its standard. - */ -TRANTOR_EXPORT std::wstring toWidePath(const std::string &strUtf8Path); - -// Helpers for multi-platform development -// OS dependent -#ifdef _WIN32 -/** - * @details Convert an UTF-8 path to a native path. - * - * This is a helper for Windows and multi-platform projects. - * - * Converts the path from UTF-8 to wide string and replaces slash directory - * separators with backslash. - * - * @remarks Although it accepts both slash and backslash as directory - * separators in its API, it is better to stick to its standard. - * - * @param strPath UTF-8 path. - * - * @return Native path suitable for the Windows unicode API. - */ -inline std::wstring toNativePath(const std::string &strPath) -{ - return toWidePath(strPath); -} -/** - * @details Convert an UTF-8 path to a native path. - * - * This is a helper for non-Windows OSes for multi-platform projects. - * - * Does nothing. - * - * @param strPath UTF-8 path. - * - * @return \p strPath. - */ -inline const std::wstring &toNativePath(const std::wstring &strPath) -{ - return strPath; -} -#else // __WIN32 -/** - * @details Convert an UTF-8 path to a native path. - * - * This is a helper for non-Windows OSes for multi-platform projects. - * - * Does nothing. - * - * @param strPath UTF-8 path. - * - * @return \p strPath. - */ -inline const std::string &toNativePath(const std::string &strPath) -{ - return strPath; -} -/** - * @details Convert an wide string path to a UTF-8 path. - * - * This is a helper, mainly for Windows and multi-platform projects. - * - * @note On Windows, backslash directory separators are converted to slash to - * keep portable paths. - - * @warning On other OSes, backslashes are not converted to slash, since they - * are valid characters for directory/file names. - * - * @param strPath wide string path. - * - * @return Generic path, with slash directory separators - */ -inline std::string toNativePath(const std::wstring &strPath) -{ - return fromWidePath(strPath); -} -#endif // _WIN32 -/** - * @note NoOP on all OSes - */ -inline const std::string &fromNativePath(const std::string &strPath) -{ - return strPath; -} -/** - * @note fromWidePath() on all OSes - */ -// Convert on all systems -inline std::string fromNativePath(const std::wstring &strPath) -{ - return fromWidePath(strPath); -} - -} // namespace utils - -} // namespace trantor +/** + * + * @file Utilities.h + * @author An Tao + * + * Copyright 2018, An Tao. All rights reserved. + * https://github.com/an-tao/drogon + * Use of this source code is governed by a MIT license + * that can be found in the License file. + * + * Drogon + * + */ + +#pragma once + +#include +#include + +namespace trantor +{ +/** + * @brief Trantor helper functions. + * + */ +namespace utils +{ +/** + * @brief Convert a wide string to a UTF-8. + * @details UCS2 on Windows, UTF-32 on Linux & Mac + * + * @param str String to convert + * + * @return converted string. + */ +TRANTOR_EXPORT std::string toUtf8(const std::wstring &wstr); +/** + * @brief Convert a UTF-8 string to a wide string. + * @details UCS2 on Windows, UTF-32 on Linux & Mac + * + * @param str String to convert + * + * @return converted string. + */ +TRANTOR_EXPORT std::wstring fromUtf8(const std::string &str); + +/** + * @details Convert a wide string path with arbitrary directory separators + * to a UTF-8 portable path for use with trantor. + * + * This is a helper, mainly for Windows and multi-platform projects. + * + * @note On Windows, backslash directory separators are converted to slash to + * keep portable paths. + * + * @remarks On other OSes, backslashes are not converted to slash, since they + * are valid characters for directory/file names. + * + * @param strPath Wide string path. + * + * @return std::string UTF-8 path, with slash directory separator. + */ +TRANTOR_EXPORT std::string fromWidePath(const std::wstring &strPath); +/** + * @details Convert a UTF-8 path with arbitrary directory separator to a wide + * string path. + * + * This is a helper, mainly for Windows and multi-platform projects. + * + * @note On Windows, slash directory separators are converted to backslash. + * Although it accepts both slash and backslash as directory separator in its + * API, it is better to stick to its standard. + + * @remarks On other OSes, slashes are not converted to backslashes, since they + * are not interpreted as directory separators and are valid characters for + * directory/file names. + * + * @param strUtf8Path Ascii path considered as being UTF-8. + * + * @return std::wstring path with, on windows, standard backslash directory + * separator to stick to its standard. + */ +TRANTOR_EXPORT std::wstring toWidePath(const std::string &strUtf8Path); + +// Helpers for multi-platform development +// OS dependent +#if defined(_WIN32) && !defined(__MINGW32__) +/** + * @details Convert an UTF-8 path to a native path. + * + * This is a helper for Windows and multi-platform projects. + * + * Converts the path from UTF-8 to wide string and replaces slash directory + * separators with backslash. + * + * @remarks Although it accepts both slash and backslash as directory + * separators in its API, it is better to stick to its standard. + * + * @param strPath UTF-8 path. + * + * @return Native path suitable for the Windows unicode API. + */ +inline std::wstring toNativePath(const std::string &strPath) +{ + return toWidePath(strPath); +} +/** + * @details Convert an UTF-8 path to a native path. + * + * This is a helper for non-Windows OSes for multi-platform projects. + * + * Does nothing. + * + * @param strPath UTF-8 path. + * + * @return \p strPath. + */ +inline const std::wstring &toNativePath(const std::wstring &strPath) +{ + return strPath; +} +#else // __WIN32 +/** + * @details Convert an UTF-8 path to a native path. + * + * This is a helper for non-Windows OSes for multi-platform projects. + * + * Does nothing. + * + * @param strPath UTF-8 path. + * + * @return \p strPath. + */ +inline const std::string &toNativePath(const std::string &strPath) +{ + return strPath; +} +/** + * @details Convert an wide string path to a UTF-8 path. + * + * This is a helper, mainly for Windows and multi-platform projects. + * + * @note On Windows, backslash directory separators are converted to slash to + * keep portable paths. + + * @warning On other OSes, backslashes are not converted to slash, since they + * are valid characters for directory/file names. + * + * @param strPath wide string path. + * + * @return Generic path, with slash directory separators + */ +inline std::string toNativePath(const std::wstring &strPath) +{ + return fromWidePath(strPath); +} +#endif // _WIN32 +/** + * @note NoOP on all OSes + */ +inline const std::string &fromNativePath(const std::string &strPath) +{ + return strPath; +} +/** + * @note fromWidePath() on all OSes + */ +// Convert on all systems +inline std::string fromNativePath(const std::wstring &strPath) +{ + return fromWidePath(strPath); +} + +} // namespace utils + +} // namespace trantor From 263f58f87d746b1488a228d4a5e506e297a4ba50 Mon Sep 17 00:00:00 2001 From: An Tao Date: Thu, 29 Jul 2021 22:47:01 +0800 Subject: [PATCH 027/182] Fix the destructor of AresResolver (#162) --- trantor/net/inner/AresResolver.cc | 4 +++- trantor/net/inner/AresResolver.h | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/trantor/net/inner/AresResolver.cc b/trantor/net/inner/AresResolver.cc index 3b77b442..029e0859 100644 --- a/trantor/net/inner/AresResolver.cc +++ b/trantor/net/inner/AresResolver.cc @@ -73,6 +73,8 @@ AresResolver::AresResolver(EventLoop* loop, size_t timeout) { loop_ = getLoop(); } + loopValid_ = std::make_shared(true); + loop_->runOnQuit([loopValid = loopValid_]() { *loopValid = false; }); } void AresResolver::init() { @@ -209,7 +211,7 @@ void AresResolver::onSockStateChange(int sockfd, bool read, bool write) // update // if (write) { } else { } } - else + else if (*loopValid_) { // remove it->second->disableAll(); diff --git a/trantor/net/inner/AresResolver.h b/trantor/net/inner/AresResolver.h index 6f4adac3..e47ef7bb 100644 --- a/trantor/net/inner/AresResolver.h +++ b/trantor/net/inner/AresResolver.h @@ -86,6 +86,7 @@ class AresResolver : public Resolver, void resolveInLoop(const std::string& hostname, const Callback& cb); void init(); trantor::EventLoop* loop_; + std::shared_ptr loopValid_; ares_channel ctx_{nullptr}; bool timerActive_{false}; using ChannelList = std::map>; From cba221a87fadabae0625279b6236556786e2e7aa Mon Sep 17 00:00:00 2001 From: An Tao Date: Sat, 7 Aug 2021 14:04:07 +0800 Subject: [PATCH 028/182] Fix memory leak in NormalResolver (#163) --- trantor/net/inner/NormalResolver.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/trantor/net/inner/NormalResolver.cc b/trantor/net/inner/NormalResolver.cc index 87c2d7c2..3cfd40d0 100644 --- a/trantor/net/inner/NormalResolver.cc +++ b/trantor/net/inner/NormalResolver.cc @@ -65,6 +65,10 @@ void NormalResolver::resolve(const std::string &hostname, if (error == -1 || res == nullptr) { LOG_SYSERR << "InetAddress::resolve"; + if (res != nullptr) + { + freeaddrinfo(res); + } callback(InetAddress{}); return; } @@ -83,6 +87,7 @@ void NormalResolver::resolve(const std::string &hostname, addr = *reinterpret_cast(res->ai_addr); inet = InetAddress(addr); } + freeaddrinfo(res); callback(inet); { std::lock_guard guard(thisPtr->globalMutex()); From bb7c6f380946aaca0fcd31fa9b55820e8b56adad Mon Sep 17 00:00:00 2001 From: an-tao Date: Sun, 8 Aug 2021 16:03:28 +0800 Subject: [PATCH 029/182] Bump version to 1.5.1 --- CMakeLists.txt | 2 +- ChangeLog.md | 28 +++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 64976d75..ad5a1b33 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) set(TRANTOR_MAJOR_VERSION 1) set(TRANTOR_MINOR_VERSION 5) -set(TRANTOR_PATCH_VERSION 0) +set(TRANTOR_PATCH_VERSION 1) set(TRANTOR_VERSION ${TRANTOR_MAJOR_VERSION}.${TRANTOR_MINOR_VERSION}.${TRANTOR_PATCH_VERSION}) diff --git a/ChangeLog.md b/ChangeLog.md index a63fe82c..3d249f54 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -3,6 +3,30 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [1.5.1] - 2021-08-08 + +### API changes list + +### Changed + +- Fix warning C4244 in MSVC + +- Disable strict compiler check on Windows with GCC + +- Added support for paths containing unicode characters on Windows + +- Add BUILD_DOC cmake option (doxygen) + +- Use make_shared instead of shared_ptr(new) + +- Detect and handle MinGW + +### Fixed + +- Fix the destructor of AresResolver + +- Fix memory leak in NormalResolver + ## [1.5.0] - 2021-06-18 ### API changes list @@ -302,7 +326,9 @@ All notable changes to this project will be documented in this file. ## [1.0.0-rc1] - 2019-06-11 -[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.0...HEAD +[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.1...HEAD + +[1.5.1]: https://github.com/an-tao/trantor/compare/v1.5.0...v1.5.1 [1.5.0]: https://github.com/an-tao/trantor/compare/v1.4.1...v1.5.0 From 898ec7f0a0561b6818d44b5e14c8553f3760384d Mon Sep 17 00:00:00 2001 From: Jiannan Liu <1456542+ljn917@users.noreply.github.com> Date: Sun, 15 Aug 2021 21:40:15 -0400 Subject: [PATCH 030/182] Avoid a small memory leak (#164) `SSL_CONF_CTX_free()` needs to be called after `SSL_CONF_CTX_finish()` OpenSSL docs: https://www.openssl.org/docs/man1.1.1/man3/SSL_CONF_CTX_new.html I only tested with OpenSSL 1.1.1, but I believe OpenSSL 1.0.x should be the same. --- trantor/net/inner/TcpConnectionImpl.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index d762a7d7..23bb7670 100755 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -213,6 +213,7 @@ class SSLContext SSL_CONF_cmd(cctx, cmd.first.data(), cmd.second.data()); } SSL_CONF_CTX_finish(cctx); + SSL_CONF_CTX_free(cctx); if (!useOldTLS) { SSL_CTX_set_min_proto_version(ctxPtr_, TLS1_2_VERSION); @@ -236,6 +237,7 @@ class SSLContext SSL_CONF_cmd(cctx, cmd.first.data(), cmd.second.data()); } SSL_CONF_CTX_finish(cctx); + SSL_CONF_CTX_free(cctx); if (!useOldTLS) { SSL_CTX_set_options(ctxPtr_, SSL_OP_NO_TLSv1 | SSL_OP_NO_TLSv1_1); From 2252a2ce2f7e3f9e87b900ac21fff8db7cc83ffa Mon Sep 17 00:00:00 2001 From: Martin Chang Date: Fri, 20 Aug 2021 21:25:19 +0800 Subject: [PATCH 031/182] Disable setting SSL Configs when using LibreSSL (#167) --- trantor/net/inner/TcpConnectionImpl.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index 23bb7670..b7b168d0 100755 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -200,7 +200,17 @@ class SSLContext bool enableValidtion, const std::vector> &sslConfCmds) { -#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) +#ifdef LIBRESSL_VERSION_NUMBER + ctxPtr_ = SSL_CTX_new(TLS_method()); + if (sslConfCmds.size() != 0) + { + LOG_WARN << "LibreSSL does not support SSL configuration commands"; + } + if (!useOldTLS) + { + SSL_CTX_set_min_proto_version(ctxPtr_, TLS1_2_VERSION); + } +#elif (OPENSSL_VERSION_NUMBER >= 0x10100000L) ctxPtr_ = SSL_CTX_new(TLS_method()); SSL_CONF_CTX *cctx = SSL_CONF_CTX_new(); SSL_CONF_CTX_set_flags(cctx, SSL_CONF_FLAG_SERVER); From bef06c4f5e2dfc80303aee06a12770d09d5e8f5b Mon Sep 17 00:00:00 2001 From: tastytea Date: Sun, 22 Aug 2021 12:55:11 +0200 Subject: [PATCH 032/182] cmake: Use GNUInstallDirs to figure out install dirs. (#165) And set initial value for INSTALL_BIN_DIR. --- CMakeLists.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ad5a1b33..b858b51a 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,12 +11,14 @@ set(TRANTOR_PATCH_VERSION 1) set(TRANTOR_VERSION ${TRANTOR_MAJOR_VERSION}.${TRANTOR_MINOR_VERSION}.${TRANTOR_PATCH_VERSION}) +include(GNUInstallDirs) # Offer the user the choice of overriding the installation directories -set(INSTALL_LIB_DIR lib CACHE PATH "Installation directory for libraries") +set(INSTALL_BIN_DIR ${CMAKE_INSTALL_BINDIR} CACHE PATH "Installation directory for binaries") +set(INSTALL_LIB_DIR ${CMAKE_INSTALL_LIBDIR} CACHE PATH "Installation directory for libraries") set(INSTALL_INCLUDE_DIR - include + ${CMAKE_INSTALL_INCLUDEDIR} CACHE PATH "Installation directory for header files") -set(DEF_INSTALL_TRANTOR_CMAKE_DIR lib/cmake/Trantor) +set(DEF_INSTALL_TRANTOR_CMAKE_DIR ${CMAKE_INSTALL_LIBDIR}/cmake/Trantor) set(INSTALL_TRANTOR_CMAKE_DIR ${DEF_INSTALL_TRANTOR_CMAKE_DIR} CACHE PATH "Installation directory for cmake files") From 58a868bde3b914256d0dbd4e04779f3252052269 Mon Sep 17 00:00:00 2001 From: Martin Chang Date: Thu, 26 Aug 2021 14:35:07 +0800 Subject: [PATCH 033/182] HaikuOS support (#169) * HaikuOS support This patch addes HaikuOS support. Including the following: Works on HaikuOS R1 Beta3 * A new PollPoller (a reimplementation of mudio's for trantor) * Preprocessor logic to detect Haiku * Update clang-format to work with clang12 --- .clang-format | 23 ++-- CMakeLists.txt | 13 +- trantor/net/Channel.h | 1 + trantor/net/InetAddress.cc | 2 +- trantor/net/inner/Poller.cc | 8 +- trantor/net/inner/poller/KQueue.h | 2 +- trantor/net/inner/poller/PollPoller.cc | 175 +++++++++++++++++++++++++ trantor/net/inner/poller/PollPoller.h | 44 +++++++ trantor/utils/Logger.cc | 13 +- 9 files changed, 257 insertions(+), 24 deletions(-) create mode 100644 trantor/net/inner/poller/PollPoller.cc create mode 100644 trantor/net/inner/poller/PollPoller.h diff --git a/.clang-format b/.clang-format index ab8dc45c..234f6b58 100644 --- a/.clang-format +++ b/.clang-format @@ -21,23 +21,24 @@ AlwaysBreakTemplateDeclarations: true BinPackArguments: false BinPackParameters: false BraceWrapping: - AfterClass: false - AfterControlStatement: false - AfterEnum: false - AfterFunction: false - AfterNamespace: false + AfterClass: true + AfterControlStatement: Always + AfterEnum: true + AfterFunction: true + AfterNamespace: true AfterObjCDeclaration: false - AfterStruct: false - AfterUnion: false - AfterExternBlock: false - BeforeCatch: false - BeforeElse: false + AfterStruct: true + AfterUnion: true + AfterExternBlock: true + BeforeCatch: true + BeforeElse: true IndentBraces: false SplitEmptyFunction: true SplitEmptyRecord: true SplitEmptyNamespace: true + AfterCaseLabel: true BreakBeforeBinaryOperators: None -BreakBeforeBraces: Allman +BreakBeforeBraces: Custom BreakBeforeInheritanceComma: false BreakInheritanceList: BeforeColon BreakBeforeTernaryOperators: true diff --git a/CMakeLists.txt b/CMakeLists.txt index b858b51a..35bbbe80 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,10 +45,14 @@ else(BUILD_TRANTOR_SHARED) add_library(${PROJECT_NAME} STATIC) endif(BUILD_TRANTOR_SHARED) -if (NOT ${CMAKE_PLATFORM_NAME} STREQUAL "Windows" AND CMAKE_CXX_COMPILER_ID MATCHES GNU) +if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Windows" AND CMAKE_CXX_COMPILER_ID MATCHES GNU) target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wextra -Werror) endif() +if(${CMAKE_SYSTEM_NAME} STREQUAL "Haiku") + target_link_libraries(${PROJECT_NAME} PRIVATE network) +endif() + include(GenerateExportHeader) generate_export_header(${PROJECT_NAME} EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/exports/trantor/exports.h) @@ -95,7 +99,8 @@ set(TRANTOR_SOURCES trantor/net/inner/Timer.cc trantor/net/inner/TimerQueue.cc trantor/net/inner/poller/EpollPoller.cc - trantor/net/inner/poller/KQueue.cc) + trantor/net/inner/poller/KQueue.cc + trantor/net/inner/poller/PollPoller.cc) set(private_headers trantor/net/inner/Acceptor.h trantor/net/inner/Connector.h @@ -105,8 +110,8 @@ set(private_headers trantor/net/inner/Timer.h trantor/net/inner/TimerQueue.h trantor/net/inner/poller/EpollPoller.h - trantor/net/inner/poller/KQueue.h) - + trantor/net/inner/poller/KQueue.h + trantor/net/inner/poller/PollPoller.h) if(WIN32) set(TRANTOR_SOURCES diff --git a/trantor/net/Channel.h b/trantor/net/Channel.h index 7060ee58..77aba1cb 100644 --- a/trantor/net/Channel.h +++ b/trantor/net/Channel.h @@ -280,6 +280,7 @@ class TRANTOR_EXPORT Channel : NonCopyable friend class EventLoop; friend class EpollPoller; friend class KQueue; + friend class PollPoller; void update(); void handleEvent(); void handleEventSafely(); diff --git a/trantor/net/InetAddress.cc b/trantor/net/InetAddress.cc index 18b952a1..0f05ccc1 100644 --- a/trantor/net/InetAddress.cc +++ b/trantor/net/InetAddress.cc @@ -215,7 +215,7 @@ uint32_t InetAddress::ipNetEndian() const const uint32_t *InetAddress::ip6NetEndian() const { // assert(family() == AF_INET6); -#ifdef __linux__ +#if defined __linux__ || defined __HAIKU__ return addr6_.sin6_addr.s6_addr32; #elif defined _WIN32 // TODO is this OK ? diff --git a/trantor/net/inner/Poller.cc b/trantor/net/inner/Poller.cc index 379236ad..e0585889 100644 --- a/trantor/net/inner/Poller.cc +++ b/trantor/net/inner/Poller.cc @@ -18,15 +18,19 @@ #elif defined _WIN32 #include "Wepoll.h" #include "poller/EpollPoller.h" -#else +#elif defined __FreeBSD__ || defined __OpenBSD__ || defined __APPLE__ #include "poller/KQueue.h" +#else +#include "poller/PollPoller.h" #endif using namespace trantor; Poller *Poller::newPoller(EventLoop *loop) { #if defined __linux__ || defined _WIN32 return new EpollPoller(loop); -#else +#elif defined __FreeBSD__ || defined __OpenBSD__ || defined __APPLE__ return new KQueue(loop); +#else + return new PollPoller(loop); #endif } diff --git a/trantor/net/inner/poller/KQueue.h b/trantor/net/inner/poller/KQueue.h index 290082d2..f14cbe9f 100644 --- a/trantor/net/inner/poller/KQueue.h +++ b/trantor/net/inner/poller/KQueue.h @@ -52,4 +52,4 @@ class KQueue : public Poller #endif }; -} // namespace trantor \ No newline at end of file +} // namespace trantor diff --git a/trantor/net/inner/poller/PollPoller.cc b/trantor/net/inner/poller/PollPoller.cc new file mode 100644 index 00000000..9291d11a --- /dev/null +++ b/trantor/net/inner/poller/PollPoller.cc @@ -0,0 +1,175 @@ +/** + * + * PollPoller.h + * Martin Chang + * + * Copyright 2021, An Tao. All rights reserved. + * https://github.com/an-tao/trantor + * Use of this source code is governed by a BSD-style license + * that can be found in the License file. + * + * Trantor + * + */ +#include "PollPoller.h" + +#include +#include "trantor/net/Channel.h" + +#include + +using namespace trantor; + +#if defined __unix__ || defined __HAIKU__ + +#include +#include +static std::once_flag warning_flag; + +PollPoller::PollPoller(EventLoop* loop) : Poller(loop) +{ + std::call_once(warning_flag, []() { + LOG_WARN << "Creating a PollPoller. This poller is slow and should " + "only be used when no other pollers are avaliable"; + }); +} + +PollPoller::~PollPoller() +{ +} + +void PollPoller::poll(int timeoutMs, ChannelList* activeChannels) +{ + // XXX pollfds_ shouldn't change + int numEvents = ::poll(pollfds_.data(), pollfds_.size(), timeoutMs); + int savedErrno = errno; + if (numEvents > 0) + { + // LOG_TRACE << numEvents << " events happened"; + fillActiveChannels(numEvents, activeChannels); + } + else if (numEvents == 0) + { + // LOG_TRACE << " nothing happened"; + } + else + { + if (savedErrno != EINTR) + { + errno = savedErrno; + LOG_SYSERR << "PollPoller::poll()"; + } + } +} + +void PollPoller::fillActiveChannels(int numEvents, + ChannelList* activeChannels) const +{ + int processedEvents = 0; + for (auto pfd : pollfds_) + { + if (pfd.revents > 0) + { + auto ch = channels_.find(pfd.fd); + assert(ch != channels_.end()); + Channel* channel = ch->second; + assert(channel->fd() == pfd.fd); + channel->setRevents(pfd.revents); + // pfd.revents = 0; + activeChannels->push_back(channel); + + processedEvents++; + if (processedEvents == numEvents) + break; + } + } + assert(processedEvents == numEvents); +} + +void PollPoller::updateChannel(Channel* channel) +{ + Poller::assertInLoopThread(); + // LOG_TRACE << "fd = " << channel->fd() << " events = " << + // channel->events(); + if (channel->index() < 0) + { + // a new one, add to pollfds_ + assert(channels_.find(channel->fd()) == channels_.end()); + pollfd pfd; + pfd.fd = channel->fd(); + pfd.events = static_cast(channel->events()); + pfd.revents = 0; + pollfds_.push_back(pfd); + int idx = static_cast(pollfds_.size()) - 1; + channel->setIndex(idx); + channels_[pfd.fd] = channel; + } + else + { + // update existing one + assert(channels_.find(channel->fd()) != channels_.end()); + assert(channels_[channel->fd()] == channel); + int idx = channel->index(); + assert(0 <= idx && idx < static_cast(pollfds_.size())); + pollfd& pfd = pollfds_[idx]; + assert(pfd.fd == channel->fd() || pfd.fd == -channel->fd() - 1); + pfd.fd = channel->fd(); + pfd.events = static_cast(channel->events()); + pfd.revents = 0; + if (channel->isNoneEvent()) + { + // ignore this pollfd + pfd.fd = -channel->fd() - 1; + } + } +} + +void PollPoller::removeChannel(Channel* channel) +{ + Poller::assertInLoopThread(); + // LOG_TRACE << "fd = " << channel->fd(); + assert(channels_.find(channel->fd()) != channels_.end()); + assert(channels_[channel->fd()] == channel); + assert(channel->isNoneEvent()); + int idx = channel->index(); + assert(0 <= idx && idx < static_cast(pollfds_.size())); + const pollfd& pfd = pollfds_[idx]; + (void)pfd; + assert(pfd.fd == -channel->fd() - 1 && pfd.events == channel->events()); + size_t n = channels_.erase(channel->fd()); + assert(n == 1); + (void)n; + if (size_t(idx) == pollfds_.size() - 1) + { + pollfds_.pop_back(); + } + else + { + int channelAtEnd = pollfds_.back().fd; + iter_swap(pollfds_.begin() + idx, pollfds_.end() - 1); + if (channelAtEnd < 0) + { + channelAtEnd = -channelAtEnd - 1; + } + channels_[channelAtEnd]->setIndex(idx); + pollfds_.pop_back(); + } +} +#else +PollPoller::PollPoller(EventLoop *loop) : Poller(loop) +{ + assert(false); +} +PollPoller::~PollPoller() +{ +} +void PollPoller::poll(int, ChannelList *) +{ +} +void PollPoller::updateChannel(Channel *) +{ +} +void PollPoller::removeChannel(Channel *) +{ +} +#endif diff --git a/trantor/net/inner/poller/PollPoller.h b/trantor/net/inner/poller/PollPoller.h new file mode 100644 index 00000000..0914360c --- /dev/null +++ b/trantor/net/inner/poller/PollPoller.h @@ -0,0 +1,44 @@ +/** + * + * PollPoller.h + * Martin Chang + * + * Copyright 2021, An Tao. All rights reserved. + * https://github.com/an-tao/trantor + * Use of this source code is governed by a BSD-style license + * that can be found in the License file. + * + * Trantor + * + */ +#pragma once + +#include "../Poller.h" +#include + +#if defined __unix__ || defined __HAIKU__ +#include +#endif + +namespace trantor +{ +class PollPoller : public Poller +{ + public: + PollPoller(EventLoop* loop); + ~PollPoller() override; + + void poll(int timeoutMs, ChannelList* activeChannels) override; + void updateChannel(Channel* channel) override; + void removeChannel(Channel* channel) override; + + private: + void fillActiveChannels(int numEvents, ChannelList* activeChannels) const; + +#if defined __unix__ || defined __HAIKU__ + std::vector pollfds_; + std::map channels_; +#endif +}; + +} // namespace trantor diff --git a/trantor/utils/Logger.cc b/trantor/utils/Logger.cc index b407ef37..a8c26024 100644 --- a/trantor/utils/Logger.cc +++ b/trantor/utils/Logger.cc @@ -15,13 +15,16 @@ #include #include #include -#ifndef _WIN32 +#ifdef __unix__ #include #include -#else #include -#endif -#ifdef __FreeBSD__ +#elif defined __HAIKU__ +#include +#include +#elif defined _WIN32 +#include +#elif defined __FreeBSD__ #include #endif @@ -114,7 +117,7 @@ void Logger::formatTime() { threadId_ = getthrid(); } -#elif defined _WIN32 +#elif defined _WIN32 || defined __HAIKU__ if (threadId_ == 0) { std::stringstream ss; From c894246344464b163ae73a4c2ca9c6ad9e456692 Mon Sep 17 00:00:00 2001 From: An Tao Date: Thu, 26 Aug 2021 14:35:37 +0800 Subject: [PATCH 034/182] Fix errors in log macros (#170) --- trantor/utils/Logger.h | 82 +++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/trantor/utils/Logger.h b/trantor/utils/Logger.h index 1e3e84ac..a1874cc1 100644 --- a/trantor/utils/Logger.h +++ b/trantor/utils/Logger.h @@ -279,104 +279,104 @@ class TRANTOR_EXPORT RawLogger : public NonCopyable if ((trantor::Logger::logLevel() <= trantor::Logger::kTrace) && (cond)) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kTrace, __func__) \ .stream() -#define LOG_DEBUG_IF(cond) \ - if ((Tensor::Logger::logLevel() <= Tensor::Logger::kDebug) && (cond)) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kDebug, __func__) \ +#define LOG_DEBUG_IF(cond) \ + if ((trantor::Logger::logLevel() <= trantor::Logger::kDebug) && (cond)) \ + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kDebug, __func__) \ .stream() -#define LOG_INFO_IF(cond) \ - if ((Tensor::Logger::logLevel() <= Tensor::Logger::kInfo) && (cond)) \ - Tensor::Logger(__FILE__, __LINE__).stream() +#define LOG_INFO_IF(cond) \ + if ((trantor::Logger::logLevel() <= trantor::Logger::kInfo) && (cond)) \ + trantor::Logger(__FILE__, __LINE__).stream() #define LOG_WARN_IF(cond) \ if (cond) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kWarn).stream() + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kWarn).stream() #define LOG_ERROR_IF(cond) \ if (cond) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kError).stream() + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kError).stream() #define LOG_FATAL_IF(cond) \ if (cond) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kFatal).stream() + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kFatal).stream() #ifdef NDEBUG #define DLOG_TRACE \ if (0) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kTrace, __func__) \ .stream() -#define DLOG_DEBUG \ - if (0) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kDebug, __func__) \ +#define DLOG_DEBUG \ + if (0) \ + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kDebug, __func__) \ .stream() #define DLOG_INFO \ if (0) \ - Tensor::Logger(__FILE__, __LINE__).stream() + trantor::Logger(__FILE__, __LINE__).stream() #define DLOG_WARN \ if (0) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kWarn).stream() + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kWarn).stream() #define DLOG_ERROR \ if (0) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kError).stream() + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kError).stream() #define DLOG_FATAL \ if (0) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kFatal).stream() + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kFatal).stream() #define DLOG_TRACE_IF(cond) \ if (0) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kTrace, __func__) \ .stream() -#define DLOG_DEBUG_IF(cond) \ - if (0) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kDebug, __func__) \ +#define DLOG_DEBUG_IF(cond) \ + if (0) \ + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kDebug, __func__) \ .stream() #define DLOG_INFO_IF(cond) \ if (0) \ - Tensor::Logger(__FILE__, __LINE__).stream() + trantor::Logger(__FILE__, __LINE__).stream() #define DLOG_WARN_IF(cond) \ if (0) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kWarn).stream() + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kWarn).stream() #define DLOG_ERROR_IF(cond) \ if (0) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kError).stream() + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kError).stream() #define DLOG_FATAL_IF(cond) \ if (0) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kFatal).stream() + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kFatal).stream() #else #define DLOG_TRACE \ if (trantor::Logger::logLevel() <= trantor::Logger::kTrace) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kTrace, __func__) \ .stream() -#define DLOG_DEBUG \ - if (Tensor::Logger::logLevel() <= Tensor::Logger::kDebug) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kDebug, __func__) \ +#define DLOG_DEBUG \ + if (trantor::Logger::logLevel() <= trantor::Logger::kDebug) \ + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kDebug, __func__) \ .stream() -#define DLOG_INFO \ - if (Tensor::Logger::logLevel() <= Tensor::Logger::kInfo) \ - Tensor::Logger(__FILE__, __LINE__).stream() +#define DLOG_INFO \ + if (trantor::Logger::logLevel() <= trantor::Logger::kInfo) \ + trantor::Logger(__FILE__, __LINE__).stream() #define DLOG_WARN \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kWarn).stream() + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kWarn).stream() #define DLOG_ERROR \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kError).stream() + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kError).stream() #define DLOG_FATAL \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kFatal).stream() + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kFatal).stream() #define DLOG_TRACE_IF(cond) \ if ((trantor::Logger::logLevel() <= trantor::Logger::kTrace) && (cond)) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kTrace, __func__) \ .stream() -#define DLOG_DEBUG_IF(cond) \ - if ((Tensor::Logger::logLevel() <= Tensor::Logger::kDebug) && (cond)) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kDebug, __func__) \ +#define DLOG_DEBUG_IF(cond) \ + if ((trantor::Logger::logLevel() <= trantor::Logger::kDebug) && (cond)) \ + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kDebug, __func__) \ .stream() -#define DLOG_INFO_IF(cond) \ - if ((Tensor::Logger::logLevel() <= Tensor::Logger::kInfo) && (cond)) \ - Tensor::Logger(__FILE__, __LINE__).stream() +#define DLOG_INFO_IF(cond) \ + if ((trantor::Logger::logLevel() <= trantor::Logger::kInfo) && (cond)) \ + trantor::Logger(__FILE__, __LINE__).stream() #define DLOG_WARN_IF(cond) \ if (cond) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kWarn).stream() + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kWarn).stream() #define DLOG_ERROR_IF(cond) \ if (cond) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kError).stream() + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kError).stream() #define DLOG_FATAL_IF(cond) \ if (cond) \ - Tensor::Logger(__FILE__, __LINE__, Tensor::Logger::kFatal).stream() + trantor::Logger(__FILE__, __LINE__, trantor::Logger::kFatal).stream() #endif const char *strerror_tl(int savedErrno); From 0c56e7339ea18a22a05b727c0271c95de3669c78 Mon Sep 17 00:00:00 2001 From: Neil Cook Date: Wed, 1 Sep 2021 14:56:45 +0100 Subject: [PATCH 035/182] Improve Error handling for certificates/private keys (#171) * Use OpenSSL functions to retrieve the actual error when reading certificates/keys instead of errno * Add more detail to the LOG_FATAL message in newSSLServerContext() --- trantor/net/inner/TcpConnectionImpl.cc | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index b7b168d0..68c24bd6 100755 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -322,13 +322,11 @@ std::shared_ptr newSSLServerContext( { auto ctx = newSSLContext(useOldTLS, false, sslConfCmds); auto r = SSL_CTX_use_certificate_chain_file(ctx->get(), certPath.c_str()); + char errbuf[BUFSIZ]; if (!r) { -#ifndef _MSC_VER - LOG_FATAL << strerror(errno); -#else - LOG_FATAL << strerror_tl(errno); -#endif + ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); + LOG_FATAL << "Reading certificate: " << errbuf; abort(); } r = SSL_CTX_use_PrivateKey_file(ctx->get(), @@ -336,21 +334,15 @@ std::shared_ptr newSSLServerContext( SSL_FILETYPE_PEM); if (!r) { -#ifndef _MSC_VER - LOG_FATAL << strerror(errno); -#else - LOG_FATAL << strerror_tl(errno); -#endif + ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); + LOG_FATAL << "Reading private key: " << errbuf; abort(); } r = SSL_CTX_check_private_key(ctx->get()); if (!r) { -#ifndef _MSC_VER - LOG_FATAL << strerror(errno); -#else - LOG_FATAL << strerror_tl(errno); -#endif + ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); + LOG_FATAL << "Checking private key matches certificate: " << errbuf; abort(); } return ctx; From 81983fa57d2a1d73cc2779261c064fe9f031ff2f Mon Sep 17 00:00:00 2001 From: An Tao Date: Thu, 2 Sep 2021 08:23:09 +0800 Subject: [PATCH 036/182] Fix a race condition when TcpClient destoryed before connected (#172) --- trantor/net/TcpClient.cc | 1 - trantor/net/inner/Connector.cc | 13 ++++++++++++- trantor/net/inner/Connector.h | 1 + 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/trantor/net/TcpClient.cc b/trantor/net/TcpClient.cc index 98048a07..059f3aa6 100644 --- a/trantor/net/TcpClient.cc +++ b/trantor/net/TcpClient.cc @@ -99,7 +99,6 @@ TcpClient::~TcpClient() } else { - /// TODO need test in this condition connector_->stop(); } } diff --git a/trantor/net/inner/Connector.cc b/trantor/net/inner/Connector.cc index 1b703974..11434c66 100644 --- a/trantor/net/inner/Connector.cc +++ b/trantor/net/inner/Connector.cc @@ -37,6 +37,17 @@ void Connector::restart() } void Connector::stop() { + status_ = Status::Disconnected; + if (loop_->isInLoopThread()) + { + removeAndResetChannel(); + } + else + { + loop_->queueInLoop([thisPtr = shared_from_this()]() { + thisPtr->removeAndResetChannel(); + }); + } } void Connector::startInLoop() @@ -202,7 +213,7 @@ void Connector::handleWrite() } else { - // what happened? + // has been stopped assert(status_ == Status::Disconnected); } } diff --git a/trantor/net/inner/Connector.h b/trantor/net/inner/Connector.h index 2c943ba7..eda33736 100644 --- a/trantor/net/inner/Connector.h +++ b/trantor/net/inner/Connector.h @@ -16,6 +16,7 @@ #include #include +#include #include #include From 3b0745a09479d9a6d0870e878face070e3aec981 Mon Sep 17 00:00:00 2001 From: An Tao Date: Tue, 7 Sep 2021 18:07:16 +0800 Subject: [PATCH 037/182] Fix the error when calling removeAndResetChannel twice (#174) --- trantor/net/inner/Connector.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/trantor/net/inner/Connector.cc b/trantor/net/inner/Connector.cc index 11434c66..ca17a130 100644 --- a/trantor/net/inner/Connector.cc +++ b/trantor/net/inner/Connector.cc @@ -139,6 +139,10 @@ void Connector::connecting(int sockfd) int Connector::removeAndResetChannel() { + if (!channelPtr_) + { + return -1; + } channelPtr_->disableAll(); channelPtr_->remove(); int sockfd = channelPtr_->fd(); From 414c076fb098fb8871dbd18b1322c44873e9a5eb Mon Sep 17 00:00:00 2001 From: Neil Cook Date: Thu, 9 Sep 2021 16:13:23 +0100 Subject: [PATCH 038/182] Make c-ares support optional when building (#176) Currently c-ares support is always compiled in when it is found, however it may not be required or needed. This commit makes c-ares support optional so that it can be disabled at build time, thus making it a decision for the builder. --- CMakeLists.txt | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 35bbbe80..b55d1fb3 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.5) project(trantor) option(BUILD_DOC "Build Doxygen documentation" OFF) +option(BUILD_C-ARES "Build C-ARES" ON) list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) @@ -130,24 +131,26 @@ if(OpenSSL_FOUND) target_compile_definitions(${PROJECT_NAME} PRIVATE USE_OPENSSL) endif() -find_package(c-ares) -if(c-ares_FOUND) - message(STATUS "c-ares found!") - target_link_libraries(${PROJECT_NAME} PRIVATE c-ares_lib) - set(TRANTOR_SOURCES - ${TRANTOR_SOURCES} - trantor/net/inner/AresResolver.cc) - set(private_headers - ${private_headers} - trantor/net/inner/AresResolver.h) -else(c-ares_FOUND) - set(TRANTOR_SOURCES - ${TRANTOR_SOURCES} - trantor/net/inner/NormalResolver.cc) - set(private_headers - ${private_headers} - trantor/net/inner/NormalResolver.h) -endif(c-ares_FOUND) +if (BUILD_C-ARES) + find_package(c-ares) + if(c-ares_FOUND) + message(STATUS "c-ares found!") + target_link_libraries(${PROJECT_NAME} PRIVATE c-ares_lib) + set(TRANTOR_SOURCES + ${TRANTOR_SOURCES} + trantor/net/inner/AresResolver.cc) + set(private_headers + ${private_headers} + trantor/net/inner/AresResolver.h) + else(c-ares_FOUND) + set(TRANTOR_SOURCES + ${TRANTOR_SOURCES} + trantor/net/inner/NormalResolver.cc) + set(private_headers + ${private_headers} + trantor/net/inner/NormalResolver.h) + endif(c-ares_FOUND) +endif (BUILD_C-ARES) find_package(Threads) target_link_libraries(${PROJECT_NAME} PUBLIC Threads::Threads) From 3b3df77b5969ca9301751973d7d94d9f852f4b65 Mon Sep 17 00:00:00 2001 From: An Tao Date: Mon, 13 Sep 2021 08:32:34 +0800 Subject: [PATCH 039/182] Use locale.h (#178) --- trantor/utils/Utilities.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trantor/utils/Utilities.cc b/trantor/utils/Utilities.cc index 469f6eb9..4d887dd6 100755 --- a/trantor/utils/Utilities.cc +++ b/trantor/utils/Utilities.cc @@ -19,7 +19,7 @@ #else // _WIN32 #if __cplusplus < 201103L || __cplusplus >= 201703L #include -#include +#include #else // __cplusplus #include #include From 6215b9a325fbb9f5cf07fdaf164e8fb31dd32443 Mon Sep 17 00:00:00 2001 From: NitroMelon Date: Thu, 23 Sep 2021 23:42:49 +0800 Subject: [PATCH 040/182] Assert fd >= 0 in updateChannel(). (#181) --- trantor/net/inner/poller/EpollPoller.cc | 2 ++ trantor/net/inner/poller/KQueue.cc | 2 ++ trantor/net/inner/poller/PollPoller.cc | 2 ++ 3 files changed, 6 insertions(+) diff --git a/trantor/net/inner/poller/EpollPoller.cc b/trantor/net/inner/poller/EpollPoller.cc index ac268757..8be13aa5 100644 --- a/trantor/net/inner/poller/EpollPoller.cc +++ b/trantor/net/inner/poller/EpollPoller.cc @@ -135,6 +135,8 @@ void EpollPoller::fillActiveChannels(int numEvents, void EpollPoller::updateChannel(Channel *channel) { assertInLoopThread(); + assert(channel->fd() >= 0); + const int index = channel->index(); // LOG_TRACE << "fd = " << channel->fd() // << " events = " << channel->events() << " index = " << index; diff --git a/trantor/net/inner/poller/KQueue.cc b/trantor/net/inner/poller/KQueue.cc index ee917d29..acf4ae78 100644 --- a/trantor/net/inner/poller/KQueue.cc +++ b/trantor/net/inner/poller/KQueue.cc @@ -111,6 +111,8 @@ void KQueue::fillActiveChannels(int numEvents, void KQueue::updateChannel(Channel *channel) { assertInLoopThread(); + assert(channel->fd() >= 0); + const int index = channel->index(); // LOG_TRACE << "fd = " << channel->fd() // << " events = " << channel->events() << " index = " << index; diff --git a/trantor/net/inner/poller/PollPoller.cc b/trantor/net/inner/poller/PollPoller.cc index 9291d11a..5223433a 100644 --- a/trantor/net/inner/poller/PollPoller.cc +++ b/trantor/net/inner/poller/PollPoller.cc @@ -89,6 +89,8 @@ void PollPoller::fillActiveChannels(int numEvents, void PollPoller::updateChannel(Channel* channel) { Poller::assertInLoopThread(); + assert(channel->fd() >= 0); + // LOG_TRACE << "fd = " << channel->fd() << " events = " << // channel->events(); if (channel->index() < 0) From 6dba1941f4e205c7c0e0578ede52cebc7da5088a Mon Sep 17 00:00:00 2001 From: An Tao Date: Sat, 25 Sep 2021 21:40:15 +0800 Subject: [PATCH 041/182] Fix a bug when EAGAIN on reading sockets (#182) * Fix a bug when EAGAIN on reading sockets * remove ubuntu16.04 from CI --- .github/workflows/cmake.yml | 8 ++++---- trantor/net/inner/TcpConnectionImpl.cc | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 7ad9753b..fa2f9920 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -62,10 +62,10 @@ jobs: buildname: 'ubuntu-20.04/gcc' triplet: x64-linux compiler: gcc_64 - - os: ubuntu-16.04 - buildname: 'ubuntu-16.04/gcc' - triplet: x64-linux - compiler: gcc_64 + # - os: ubuntu-16.04 + # buildname: 'ubuntu-16.04/gcc' + # triplet: x64-linux + # compiler: gcc_64 - os: macos-latest buildname: 'macos/clang' triplet: x64-osx diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index 68c24bd6..766d0fd5 100755 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -552,9 +552,11 @@ void TcpConnectionImpl::readCallback() } else if (n < 0) { - if (errno == EPIPE || errno == ECONNRESET) // TODO: any others? + if (errno == EPIPE || errno == ECONNRESET || + errno == EAGAIN) // TODO: any others? { - LOG_DEBUG << "EPIPE or ECONNRESET, errno=" << errno; + LOG_DEBUG << "EPIPE or ECONNRESET, errno=" << errno + << " fd=" << socketPtr_->fd(); return; } #ifdef _WIN32 From d42afac871592304a61de140c56095aca1df1a97 Mon Sep 17 00:00:00 2001 From: James Armes Date: Sat, 25 Sep 2021 07:40:57 -0600 Subject: [PATCH 042/182] Add Clang support for -Wall -Wextra -Werror; fix -Wunused-parameter warnings when not using OpenSSL (#175) --- CMakeLists.txt | 2 +- trantor/net/TcpClient.cc | 7 +++++++ trantor/net/TcpClient.h | 7 ++++++- trantor/net/TcpServer.cc | 7 +++++++ trantor/net/inner/TcpConnectionImpl.cc | 21 +++++++++++++++++---- 5 files changed, 38 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b55d1fb3..49872ef0 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,7 +46,7 @@ else(BUILD_TRANTOR_SHARED) add_library(${PROJECT_NAME} STATIC) endif(BUILD_TRANTOR_SHARED) -if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Windows" AND CMAKE_CXX_COMPILER_ID MATCHES GNU) +if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Windows" AND CMAKE_CXX_COMPILER_ID MATCHES Clang|GNU) target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wextra -Werror) endif() diff --git a/trantor/net/TcpClient.cc b/trantor/net/TcpClient.cc index 059f3aa6..95cc6254 100644 --- a/trantor/net/TcpClient.cc +++ b/trantor/net/TcpClient.cc @@ -221,6 +221,13 @@ void TcpClient::enableSSL( } #else + // When not using OpenSSL, using `void` here will + // work around the unused parameter warnings without overhead. + (void)useOldTLS; + (void)validateCert; + (void)hostname; + (void)sslConfCmds; + LOG_FATAL << "OpenSSL is not found in your system!"; abort(); #endif diff --git a/trantor/net/TcpClient.h b/trantor/net/TcpClient.h index 045ad063..6b2c49f6 100644 --- a/trantor/net/TcpClient.h +++ b/trantor/net/TcpClient.h @@ -228,8 +228,13 @@ class TRANTOR_EXPORT TcpClient : NonCopyable mutable std::mutex mutex_; TcpConnectionPtr connection_; // @GuardedBy mutex_ std::shared_ptr sslCtxPtr_; - bool validateCert_{false}; std::string SSLHostName_; + +// This member variable is only needed when using OpenSSL, so +// we can safely remove it when not using OpenSSL. +#ifdef USE_OPENSSL + bool validateCert_{false}; +#endif #ifndef _WIN32 class IgnoreSigPipe { diff --git a/trantor/net/TcpServer.cc b/trantor/net/TcpServer.cc index d67488bd..6bf65129 100644 --- a/trantor/net/TcpServer.cc +++ b/trantor/net/TcpServer.cc @@ -195,6 +195,13 @@ void TcpServer::enableSSL( /* Create a new OpenSSL context */ sslCtxPtr_ = newSSLServerContext(certPath, keyPath, useOldTLS, sslConfCmds); #else + // When not using OpenSSL, using `void` here will + // work around the unused parameter warnings without overhead. + (void)certPath; + (void)keyPath; + (void)useOldTLS; + (void)sslConfCmds; + LOG_FATAL << "OpenSSL is not found in your system!"; abort(); #endif diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index 766d0fd5..b7891457 100755 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -352,10 +352,10 @@ std::shared_ptr newSSLServerContext( namespace trantor { std::shared_ptr newSSLServerContext( - const std::string &certPath, - const std::string &keyPath, - bool useOldTLS, - const std::vector> &sslConfCmds) + const std::string &, + const std::string &, + bool, + const std::vector> &) { LOG_FATAL << "OpenSSL is not found in your system!"; abort(); @@ -470,6 +470,11 @@ void TcpConnectionImpl::startServerEncryption( std::function callback) { #ifndef USE_OPENSSL + // When not using OpenSSL, using `void` here will + // work around the unused parameter warnings without overhead. + (void)ctx; + (void)callback; + LOG_FATAL << "OpenSSL is not found in your system!"; abort(); #else @@ -496,6 +501,14 @@ void TcpConnectionImpl::startClientEncryption( const std::vector> &sslConfCmds) { #ifndef USE_OPENSSL + // When not using OpenSSL, using `void` here will + // work around the unused parameter warnings without overhead. + (void)callback; + (void)useOldTLS; + (void)validateCert; + (void)hostname; + (void)sslConfCmds; + LOG_FATAL << "OpenSSL is not found in your system!"; abort(); #else From ccac964d1afa9247517d27ba1ff62eda3e8d7998 Mon Sep 17 00:00:00 2001 From: an-tao Date: Sat, 25 Sep 2021 21:56:21 +0800 Subject: [PATCH 043/182] Remove a compile time macro from public API headers --- trantor/net/TcpClient.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/trantor/net/TcpClient.h b/trantor/net/TcpClient.h index 6b2c49f6..c776d997 100644 --- a/trantor/net/TcpClient.h +++ b/trantor/net/TcpClient.h @@ -229,12 +229,8 @@ class TRANTOR_EXPORT TcpClient : NonCopyable TcpConnectionPtr connection_; // @GuardedBy mutex_ std::shared_ptr sslCtxPtr_; std::string SSLHostName_; - -// This member variable is only needed when using OpenSSL, so -// we can safely remove it when not using OpenSSL. -#ifdef USE_OPENSSL bool validateCert_{false}; -#endif + #ifndef _WIN32 class IgnoreSigPipe { From d38624009fc59d5522f8d69f64e2b7c66e3ebd25 Mon Sep 17 00:00:00 2001 From: An Tao Date: Sat, 25 Sep 2021 22:24:25 +0800 Subject: [PATCH 044/182] fix compilation warnings (#183) --- trantor/net/TcpClient.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/trantor/net/TcpClient.cc b/trantor/net/TcpClient.cc index 95cc6254..7f92a71b 100644 --- a/trantor/net/TcpClient.cc +++ b/trantor/net/TcpClient.cc @@ -63,6 +63,7 @@ TcpClient::TcpClient(EventLoop *loop, retry_(false), connect_(true) { + (void)validateCert_; connector_->setNewConnectionCallback( std::bind(&TcpClient::newConnection, this, _1)); connector_->setErrorCallback([this]() { From 9f515fe76cfdbef8beff193a5d2c876dd0bdf2fa Mon Sep 17 00:00:00 2001 From: An Tao Date: Sun, 3 Oct 2021 00:18:05 +0800 Subject: [PATCH 045/182] Fix a potential race condition (#185) --- trantor/net/inner/Connector.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trantor/net/inner/Connector.cc b/trantor/net/inner/Connector.cc index ca17a130..919c6836 100644 --- a/trantor/net/inner/Connector.cc +++ b/trantor/net/inner/Connector.cc @@ -147,7 +147,8 @@ int Connector::removeAndResetChannel() channelPtr_->remove(); int sockfd = channelPtr_->fd(); // Can't reset channel_ here, because we are inside Channel::handleEvent - loop_->queueInLoop([this]() { channelPtr_.reset(); }); + loop_->queueInLoop([channelPtr = channelPtr_]() {}); + channelPtr_.reset(); return sockfd; } From 5457061be229db8098b42afb068078603194a4f9 Mon Sep 17 00:00:00 2001 From: an-tao Date: Sun, 17 Oct 2021 11:33:00 +0800 Subject: [PATCH 046/182] Bump version to 1.5.2 --- CMakeLists.txt | 2 +- ChangeLog.md | 44 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 49872ef0..71ba9128 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) set(TRANTOR_MAJOR_VERSION 1) set(TRANTOR_MINOR_VERSION 5) -set(TRANTOR_PATCH_VERSION 1) +set(TRANTOR_PATCH_VERSION 2) set(TRANTOR_VERSION ${TRANTOR_MAJOR_VERSION}.${TRANTOR_MINOR_VERSION}.${TRANTOR_PATCH_VERSION}) diff --git a/ChangeLog.md b/ChangeLog.md index 3d249f54..1a50b76b 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -3,6 +3,44 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [1.5.2] - 2021-10-17 + +### API changes list + +### Changed + +- Disable setting SSL Configs when using LibreSSL + +- cmake: Use GNUInstallDirs to figure out install dirs + +- support HaikuOS + +- Improve Error handling for certificates/private keys + +- Make c-ares support optional when building + +- Use locale.h + +- Assert fd >= 0 in updateChannel() + +- Add Clang support for -Wall -Wextra -Werror; fix -Wunused-parameter + +### Fixed + +- Fix a small memory leak + +- Fix errors in log macros + +- Fix a race condition when TcpClient destroyed before connected + +- Fix the error of calling removeAndResetChannel twice + +- Fix a bug when EAGAIN on reading sockets + +- Fix compilation warnings + +- Fix a potential race condition + ## [1.5.1] - 2021-08-08 ### API changes list @@ -13,7 +51,7 @@ All notable changes to this project will be documented in this file. - Disable strict compiler check on Windows with GCC -- Added support for paths containing unicode characters on Windows +- Add support for paths containing unicode characters on Windows - Add BUILD_DOC cmake option (doxygen) @@ -326,7 +364,9 @@ All notable changes to this project will be documented in this file. ## [1.0.0-rc1] - 2019-06-11 -[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.1...HEAD +[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.2...HEAD + +[1.5.2]: https://github.com/an-tao/trantor/compare/v1.5.1...v1.5.2 [1.5.1]: https://github.com/an-tao/trantor/compare/v1.5.0...v1.5.1 From 586aacd084e088bbc041350a657b80b143820276 Mon Sep 17 00:00:00 2001 From: Martin Chang Date: Tue, 26 Oct 2021 09:23:42 +0800 Subject: [PATCH 047/182] make sure resolvers are added when C-Ares is manually disabled (#187) --- CMakeLists.txt | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 71ba9128..243a1d49 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,26 +131,31 @@ if(OpenSSL_FOUND) target_compile_definitions(${PROJECT_NAME} PRIVATE USE_OPENSSL) endif() +set(HAVE_C-ARES NO) if (BUILD_C-ARES) find_package(c-ares) if(c-ares_FOUND) message(STATUS "c-ares found!") - target_link_libraries(${PROJECT_NAME} PRIVATE c-ares_lib) - set(TRANTOR_SOURCES - ${TRANTOR_SOURCES} - trantor/net/inner/AresResolver.cc) - set(private_headers - ${private_headers} - trantor/net/inner/AresResolver.h) - else(c-ares_FOUND) - set(TRANTOR_SOURCES - ${TRANTOR_SOURCES} - trantor/net/inner/NormalResolver.cc) - set(private_headers - ${private_headers} - trantor/net/inner/NormalResolver.h) - endif(c-ares_FOUND) -endif (BUILD_C-ARES) + set(HAVE_C-ARES TRUE) + endif() +endif () + +if(HAVE_C-ARES) + target_link_libraries(${PROJECT_NAME} PRIVATE c-ares_lib) + set(TRANTOR_SOURCES + ${TRANTOR_SOURCES} + trantor/net/inner/AresResolver.cc) + set(private_headers + ${private_headers} + trantor/net/inner/AresResolver.h) +else() + set(TRANTOR_SOURCES + ${TRANTOR_SOURCES} + trantor/net/inner/NormalResolver.cc) + set(private_headers + ${private_headers} + trantor/net/inner/NormalResolver.h) +endif() find_package(Threads) target_link_libraries(${PROJECT_NAME} PUBLIC Threads::Threads) From 14e1554ca5c12b9a0d9758bcb9b8aca4a666321c Mon Sep 17 00:00:00 2001 From: Martin Chang Date: Fri, 19 Nov 2021 20:54:17 +0800 Subject: [PATCH 048/182] allow RVO in fromDbStringLocal (#189) --- trantor/utils/Date.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/trantor/utils/Date.cc b/trantor/utils/Date.cc index 14bc83ee..7525bf2a 100644 --- a/trantor/utils/Date.cc +++ b/trantor/utils/Date.cc @@ -310,8 +310,7 @@ Date Date::fromDbStringLocal(const std::string &datetime) } } } - return std::move( - trantor::Date(year, month, day, hour, minute, second, microSecond)); + return trantor::Date(year, month, day, hour, minute, second, microSecond); } std::string Date::toCustomedFormattedStringLocal(const std::string &fmtStr, bool showMicroseconds) const From 63ff8d9bad639c5fa3e6bd7072669813fae4c6af Mon Sep 17 00:00:00 2001 From: Martin Chang Date: Fri, 26 Nov 2021 19:41:30 +0800 Subject: [PATCH 049/182] TcpClientImpl support SSL client certificate (#190) --- trantor/net/TcpClient.cc | 9 +++++-- trantor/net/TcpClient.h | 6 ++++- trantor/net/inner/TcpConnectionImpl.cc | 37 ++++++++++++++++++++++++++ trantor/net/inner/TcpConnectionImpl.h | 7 +++++ 4 files changed, 56 insertions(+), 3 deletions(-) mode change 100755 => 100644 trantor/net/inner/TcpConnectionImpl.cc diff --git a/trantor/net/TcpClient.cc b/trantor/net/TcpClient.cc index 7f92a71b..cc409e68 100644 --- a/trantor/net/TcpClient.cc +++ b/trantor/net/TcpClient.cc @@ -206,11 +206,14 @@ void TcpClient::enableSSL( bool useOldTLS, bool validateCert, std::string hostname, - const std::vector> &sslConfCmds) + const std::vector> &sslConfCmds, + const std::string &certPath, + const std::string &keyPath) { #ifdef USE_OPENSSL /* Create a new OpenSSL context */ - sslCtxPtr_ = newSSLContext(useOldTLS, validateCert, sslConfCmds); + sslCtxPtr_ = newSSLClientContext( + useOldTLS, validateCert, certPath, keyPath, sslConfCmds); validateCert_ = validateCert; if (!hostname.empty()) { @@ -228,6 +231,8 @@ void TcpClient::enableSSL( (void)validateCert; (void)hostname; (void)sslConfCmds; + (void)certPath; + (void)keyPath; LOG_FATAL << "OpenSSL is not found in your system!"; abort(); diff --git a/trantor/net/TcpClient.h b/trantor/net/TcpClient.h index c776d997..a2fba764 100644 --- a/trantor/net/TcpClient.h +++ b/trantor/net/TcpClient.h @@ -199,6 +199,8 @@ class TRANTOR_EXPORT TcpClient : NonCopyable * not used. * @param sslConfCmds The commands used to call the SSL_CONF_cmd function in * OpenSSL. + * @param certPath The path of the certificate file. + * @param keyPath The path of the private key file. * @note It's well known that TLS 1.0 and 1.1 are not considered secure in * 2020. And it's a good practice to only use TLS 1.2 and above. */ @@ -206,7 +208,9 @@ class TRANTOR_EXPORT TcpClient : NonCopyable bool validateCert = true, std::string hostname = "", const std::vector> - &sslConfCmds = {}); + &sslConfCmds = {}, + const std::string &certPath = "", + const std::string &keyPath = ""); private: /// Not thread safe, but in loop diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc old mode 100755 new mode 100644 index b7891457..1d2db31a --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -347,6 +347,43 @@ std::shared_ptr newSSLServerContext( } return ctx; } +std::shared_ptr newSSLClientContext( + bool useOldTLS, + bool validateCert, + const std::string &certPath, + const std::string &keyPath, + const std::vector> &sslConfCmds) +{ + auto ctx = newSSLContext(useOldTLS, validateCert, sslConfCmds); + if (certPath.empty() || keyPath.empty()) + return ctx; + + auto r = SSL_CTX_use_certificate_chain_file(ctx->get(), certPath.c_str()); + char errbuf[BUFSIZ]; + if (!r) + { + ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); + LOG_FATAL << "Reading certificate: " << errbuf; + abort(); + } + r = SSL_CTX_use_PrivateKey_file(ctx->get(), + keyPath.c_str(), + SSL_FILETYPE_PEM); + if (!r) + { + ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); + LOG_FATAL << "Reading private key: " << errbuf; + abort(); + } + r = SSL_CTX_check_private_key(ctx->get()); + if (!r) + { + ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); + LOG_FATAL << "Checking private key matches certificate: " << errbuf; + abort(); + } + return ctx; +} } // namespace trantor #else namespace trantor diff --git a/trantor/net/inner/TcpConnectionImpl.h b/trantor/net/inner/TcpConnectionImpl.h index baf8dc90..a004a8ba 100755 --- a/trantor/net/inner/TcpConnectionImpl.h +++ b/trantor/net/inner/TcpConnectionImpl.h @@ -47,6 +47,13 @@ std::shared_ptr newSSLServerContext( const std::string &keyPath, bool useOldTLS, const std::vector> &sslConfCmds); +std::shared_ptr newSSLClientContext( + bool useOldTLS, + bool validateCert, + const std::string &certPath = "", + const std::string &keyPath = "", + const std::vector> &sslConfCmds = {}); + // void initServerSSLContext(const std::shared_ptr &ctx, // const std::string &certPath, // const std::string &keyPath); From e72a63518a9c43c95ccf57b7a193d186c6fe5bf9 Mon Sep 17 00:00:00 2001 From: an-tao Date: Sun, 28 Nov 2021 16:55:55 +0800 Subject: [PATCH 050/182] Bump version to 1.5.3 --- CMakeLists.txt | 2 +- ChangeLog.md | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 243a1d49..33b8e2cb 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) set(TRANTOR_MAJOR_VERSION 1) set(TRANTOR_MINOR_VERSION 5) -set(TRANTOR_PATCH_VERSION 2) +set(TRANTOR_PATCH_VERSION 3) set(TRANTOR_VERSION ${TRANTOR_MAJOR_VERSION}.${TRANTOR_MINOR_VERSION}.${TRANTOR_PATCH_VERSION}) diff --git a/ChangeLog.md b/ChangeLog.md index 1a50b76b..634b7f5a 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -3,6 +3,20 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [1.5.3] - 2021-11-28 + +### API changes list + +- TcpClientImpl support SSL client certificate + +### Changed + +- Allow RVO in fromDbStringLocal + +### Fixed + +- Make sure resolvers are added when C-Ares is manually disabled + ## [1.5.2] - 2021-10-17 ### API changes list @@ -364,7 +378,9 @@ All notable changes to this project will be documented in this file. ## [1.0.0-rc1] - 2019-06-11 -[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.2...HEAD +[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.3...HEAD + +[1.5.3]: https://github.com/an-tao/trantor/compare/v1.5.2...v1.5.3 [1.5.2]: https://github.com/an-tao/trantor/compare/v1.5.1...v1.5.2 From 4b66320d34eb4c11010eddb75b281ada4618a5f9 Mon Sep 17 00:00:00 2001 From: An Tao Date: Fri, 10 Dec 2021 11:56:15 +0800 Subject: [PATCH 051/182] Fix the error when sending partial files (#192) --- trantor/net/inner/TcpConnectionImpl.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index 1d2db31a..2f11e451 100644 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -1513,7 +1513,9 @@ void TcpConnectionImpl::sendFileInLoop(const BufferNodePtr &filePtr) { auto n = read(filePtr->sendFd_, &(*fileBufferPtr_)[0], - fileBufferPtr_->size()); + std::min(fileBufferPtr_->size(), + static_castsize())>( + filePtr->fileBytesToSend_))); #else _fseeki64(filePtr->sendFp_, filePtr->offset_, SEEK_SET); if (!fileBufferPtr_) @@ -1522,9 +1524,12 @@ void TcpConnectionImpl::sendFileInLoop(const BufferNodePtr &filePtr) } while (filePtr->fileBytesToSend_ > 0) { + auto bytes = static_castsize())>( + filePtr->fileBytesToSend_); auto n = fread(&(*fileBufferPtr_)[0], 1, - fileBufferPtr_->size(), + (fileBufferPtr_->size() < bytes ? fileBufferPtr_->size() + : bytes), filePtr->sendFp_); #endif if (n > 0) From cf5fa83c795b4214ca60578608639bdd62c1ef8e Mon Sep 17 00:00:00 2001 From: An Tao Date: Fri, 10 Dec 2021 11:56:47 +0800 Subject: [PATCH 052/182] Correctly handle the error of the getaddrinfo function (#191) --- trantor/net/inner/NormalResolver.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trantor/net/inner/NormalResolver.cc b/trantor/net/inner/NormalResolver.cc index 3cfd40d0..50831998 100644 --- a/trantor/net/inner/NormalResolver.cc +++ b/trantor/net/inner/NormalResolver.cc @@ -56,13 +56,13 @@ void NormalResolver::resolve(const std::string &hostname, } } } - struct addrinfo hints, *res; + struct addrinfo hints, *res = nullptr; memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE; auto error = getaddrinfo(hostname.data(), nullptr, &hints, &res); - if (error == -1 || res == nullptr) + if (error != 0 || res == nullptr) { LOG_SYSERR << "InetAddress::resolve"; if (res != nullptr) From b85e2793fc24c6e5b9b5e9708954f2ab587e35bb Mon Sep 17 00:00:00 2001 From: antao Date: Fri, 10 Dec 2021 12:27:42 +0800 Subject: [PATCH 053/182] Bump version to 1.5.4 --- CMakeLists.txt | 2 +- ChangeLog.md | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 33b8e2cb..99687442 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) set(TRANTOR_MAJOR_VERSION 1) set(TRANTOR_MINOR_VERSION 5) -set(TRANTOR_PATCH_VERSION 3) +set(TRANTOR_PATCH_VERSION 4) set(TRANTOR_VERSION ${TRANTOR_MAJOR_VERSION}.${TRANTOR_MINOR_VERSION}.${TRANTOR_PATCH_VERSION}) diff --git a/ChangeLog.md b/ChangeLog.md index 634b7f5a..026d2f58 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -3,6 +3,18 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [1.5.4] - 2021-12-10 + +### API changes list + +### Changed + +- Correctly handle the error of the getaddrinfo function + +### Fixed + +- Fix the error when sending partial files + ## [1.5.3] - 2021-11-28 ### API changes list @@ -378,7 +390,9 @@ All notable changes to this project will be documented in this file. ## [1.0.0-rc1] - 2019-06-11 -[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.3...HEAD +[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.4...HEAD + +[1.5.4]: https://github.com/an-tao/trantor/compare/v1.5.3...v1.5.4 [1.5.3]: https://github.com/an-tao/trantor/compare/v1.5.2...v1.5.3 From 1af2c68ca216918073c336d5691c2316a56e0459 Mon Sep 17 00:00:00 2001 From: Martin Chang Date: Sun, 19 Dec 2021 11:07:36 +0800 Subject: [PATCH 054/182] Move EventLoop::runAfter to a template (#193) --- trantor/net/EventLoop.cc | 22 ---------------------- trantor/net/EventLoop.h | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/trantor/net/EventLoop.cc b/trantor/net/EventLoop.cc index 8a60a03d..05cfac62 100644 --- a/trantor/net/EventLoop.cc +++ b/trantor/net/EventLoop.cc @@ -202,28 +202,6 @@ void EventLoop::abortNotInLoopThread() "thread"; exit(1); } -void EventLoop::runInLoop(const Func &cb) -{ - if (isInLoopThread()) - { - cb(); - } - else - { - queueInLoop(cb); - } -} -void EventLoop::runInLoop(Func &&cb) -{ - if (isInLoopThread()) - { - cb(); - } - else - { - queueInLoop(std::move(cb)); - } -} void EventLoop::queueInLoop(const Func &cb) { funcs_.enqueue(cb); diff --git a/trantor/net/EventLoop.h b/trantor/net/EventLoop.h index d28f6b8b..a3e12049 100644 --- a/trantor/net/EventLoop.h +++ b/trantor/net/EventLoop.h @@ -121,8 +121,18 @@ class TRANTOR_EXPORT EventLoop : NonCopyable * @note If the current thread is the thread of the event loop, the function * f is executed directly before the method exiting. */ - void runInLoop(const Func &f); - void runInLoop(Func &&f); + template + inline void runInLoop(Functor &&f) + { + if (isInLoopThread()) + { + f(); + } + else + { + queueInLoop(std::forward(f)); + } + } /** * @brief Run the function f in the thread of the event loop. From a01b3b40614417b72904be41718c55877d1188e6 Mon Sep 17 00:00:00 2001 From: An Tao Date: Fri, 21 Jan 2022 15:45:27 +0800 Subject: [PATCH 055/182] Remove an assertion when removing channels (#195) --- trantor/net/Channel.cc | 2 ++ trantor/net/EventLoop.cc | 7 ------- trantor/net/TcpServer.cc | 22 +++++++++++++++++----- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/trantor/net/Channel.cc b/trantor/net/Channel.cc index eaceea17..56e8e933 100644 --- a/trantor/net/Channel.cc +++ b/trantor/net/Channel.cc @@ -53,6 +53,8 @@ void Channel::update() void Channel::handleEvent() { // LOG_TRACE<<"revents_="< guard = tie_.lock(); diff --git a/trantor/net/EventLoop.cc b/trantor/net/EventLoop.cc index 05cfac62..76abd864 100644 --- a/trantor/net/EventLoop.cc +++ b/trantor/net/EventLoop.cc @@ -136,13 +136,6 @@ void EventLoop::removeChannel(Channel *channel) { assert(channel->ownerLoop() == this); assertInLoopThread(); - if (eventHandling_) - { - assert(currentActiveChannel_ == channel || - std::find(activeChannels_.begin(), - activeChannels_.end(), - channel) == activeChannels_.end()); - } poller_->removeChannel(channel); } void EventLoop::quit() diff --git a/trantor/net/TcpServer.cc b/trantor/net/TcpServer.cc index 6bf65129..a97ddfa4 100644 --- a/trantor/net/TcpServer.cc +++ b/trantor/net/TcpServer.cc @@ -165,14 +165,26 @@ void TcpServer::stop() void TcpServer::connectionClosed(const TcpConnectionPtr &connectionPtr) { LOG_TRACE << "connectionClosed"; - // loop_->assertInLoopThread(); - loop_->runInLoop([this, connectionPtr]() { + if (loop_->isInLoopThread()) + { size_t n = connSet_.erase(connectionPtr); (void)n; assert(n == 1); - }); - - static_cast(connectionPtr.get())->connectDestroyed(); + loop_->queueInLoop([connectionPtr]() { + static_cast(connectionPtr.get()) + ->connectDestroyed(); + }); + } + else + { + loop_->queueInLoop([this, connectionPtr]() { + size_t n = connSet_.erase(connectionPtr); + (void)n; + assert(n == 1); + static_cast(connectionPtr.get()) + ->connectDestroyed(); + }); + } } const std::string TcpServer::ipPort() const From 3ddca84c56e0c5866e6e39bc3138b59a67af9793 Mon Sep 17 00:00:00 2001 From: Glenn Schmottlach Date: Wed, 26 Jan 2022 10:25:59 -0500 Subject: [PATCH 056/182] Prevent TcpClient::removeConnection call on deleted TcpClient instance. (#197) --- trantor/net/TcpClient.cc | 54 ++++++++++++++++++++++++++-------------- trantor/net/TcpClient.h | 3 ++- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/trantor/net/TcpClient.cc b/trantor/net/TcpClient.cc index cc409e68..6de3555b 100644 --- a/trantor/net/TcpClient.cc +++ b/trantor/net/TcpClient.cc @@ -82,25 +82,25 @@ TcpClient::~TcpClient() { std::lock_guard lock(mutex_); conn = std::dynamic_pointer_cast(connection_); - } - if (conn) - { - assert(loop_ == conn->getLoop()); - // TODO: not 100% safe, if we are in different thread - auto loop = loop_; - loop_->runInLoop([conn, loop]() { - conn->setCloseCallback([loop](const TcpConnectionPtr &connPtr) { - loop->queueInLoop([connPtr]() { - static_cast(connPtr.get()) - ->connectDestroyed(); + if (conn) + { + assert(loop_ == conn->getLoop()); + // TODO: not 100% safe, if we are in different thread + auto loop = loop_; + loop_->runInLoop([conn, loop]() { + conn->setCloseCallback([loop](const TcpConnectionPtr &connPtr) { + loop->queueInLoop([connPtr]() { + static_cast(connPtr.get()) + ->connectDestroyed(); + }); }); }); - }); - conn->forceClose(); - } - else - { - connector_->stop(); + conn->forceClose(); + } + else + { + connector_->stop(); + } } } @@ -166,7 +166,25 @@ void TcpClient::newConnection(int sockfd) conn->setConnectionCallback(connectionCallback_); conn->setRecvMsgCallback(messageCallback_); conn->setWriteCompleteCallback(writeCompleteCallback_); - conn->setCloseCallback(std::bind(&TcpClient::removeConnection, this, _1)); + + std::weak_ptr weakSelf(shared_from_this()); + auto closeCb = std::function( + [weakSelf](const TcpConnectionPtr &c) { + if (auto self = weakSelf.lock()) + { + self->removeConnection(c); + } + // Else the TcpClient instance has already been destroyed + else + { + LOG_DEBUG << "TcpClient::removeConnection was skipped because " + "TcpClient instanced already freed"; + c->getLoop()->queueInLoop( + std::bind(&TcpConnectionImpl::connectDestroyed, + std::dynamic_pointer_cast(c))); + } + }); + conn->setCloseCallback(std::move(closeCb)); { std::lock_guard lock(mutex_); connection_ = conn; diff --git a/trantor/net/TcpClient.h b/trantor/net/TcpClient.h index a2fba764..1ac915e3 100644 --- a/trantor/net/TcpClient.h +++ b/trantor/net/TcpClient.h @@ -34,7 +34,8 @@ class SSLContext; * @brief This class represents a TCP client. * */ -class TRANTOR_EXPORT TcpClient : NonCopyable +class TRANTOR_EXPORT TcpClient : NonCopyable, + public std::enable_shared_from_this { public: /** From 4d0f72e3d98e8a2eebe9598e9eb3ebc3acc27886 Mon Sep 17 00:00:00 2001 From: Glenn Schmottlach Date: Wed, 26 Jan 2022 23:16:06 -0500 Subject: [PATCH 057/182] Wait for loop to exit in EventLoop destructor. (#198) --- trantor/net/EventLoop.cc | 41 +++++++++++++++++++++++++++++++--------- trantor/net/EventLoop.h | 8 +++++--- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/trantor/net/EventLoop.cc b/trantor/net/EventLoop.cc index 76abd864..e6b979a6 100644 --- a/trantor/net/EventLoop.cc +++ b/trantor/net/EventLoop.cc @@ -25,7 +25,9 @@ #include #include #ifdef _WIN32 +#include #include +#include using ssize_t = long long; #else #include @@ -101,7 +103,7 @@ EventLoop::EventLoop() void EventLoop::resetTimerQueue() { assertInLoopThread(); - assert(!looping_); + assert(!looping_.load(std::memory_order_acquire)); timerQueue_->reset(); } #endif @@ -111,8 +113,29 @@ void EventLoop::resetAfterFork() } EventLoop::~EventLoop() { +#ifdef _WIN32 + DWORD delay = 1; /* 1 msec */ +#else + struct timespec delay = {0, 1000000}; /* 1 msec */ +#endif + quit(); - assert(!looping_); + + // Spin waiting for the loop to exit because + // this may take some time to complete. We + // assume the loop thread will *always* exit. + // If this cannot be guaranteed then one option + // might be to abort waiting and + // assert(!looping_) after some delay; + while (looping_.load(std::memory_order_acquire)) + { +#ifdef _WIN32 + Sleep(delay); +#else + nanosleep(&delay, nullptr); +#endif + } + t_loopInThisThread = nullptr; #ifdef __linux__ close(wakeupFd_); @@ -140,7 +163,7 @@ void EventLoop::removeChannel(Channel *channel) } void EventLoop::quit() { - quit_ = true; + quit_.store(true, std::memory_order_release); Func f; while (funcsOnQuit_.dequeue(f)) @@ -160,10 +183,10 @@ void EventLoop::loop() { assert(!looping_); assertInLoopThread(); - looping_ = true; - quit_ = false; + looping_.store(true, std::memory_order_release); + quit_.store(false, std::memory_order_release); - while (!quit_) + while (!quit_.load(std::memory_order_acquire)) { activeChannels_.clear(); #ifdef __linux__ @@ -187,7 +210,7 @@ void EventLoop::loop() // std::cout << "looping" << endl; doRunInLoopFuncs(); } - looping_ = false; + looping_.store(false, std::memory_order_release); } void EventLoop::abortNotInLoopThread() { @@ -198,7 +221,7 @@ void EventLoop::abortNotInLoopThread() void EventLoop::queueInLoop(const Func &cb) { funcs_.enqueue(cb); - if (!isInLoopThread() || !looping_) + if (!isInLoopThread() || !looping_.load(std::memory_order_acquire)) { wakeup(); } @@ -206,7 +229,7 @@ void EventLoop::queueInLoop(const Func &cb) void EventLoop::queueInLoop(Func &&cb) { funcs_.enqueue(std::move(cb)); - if (!isInLoopThread() || !looping_) + if (!isInLoopThread() || !looping_.load(std::memory_order_acquire)) { wakeup(); } diff --git a/trantor/net/EventLoop.h b/trantor/net/EventLoop.h index a3e12049..fd0253d2 100644 --- a/trantor/net/EventLoop.h +++ b/trantor/net/EventLoop.h @@ -28,6 +28,7 @@ #include #include #include +#include namespace trantor { @@ -270,7 +271,8 @@ class TRANTOR_EXPORT EventLoop : NonCopyable */ bool isRunning() { - return looping_ && (!quit_); + return looping_.load(std::memory_order_acquire) && + (!quit_.load(std::memory_order_acquire)); } /** @@ -297,9 +299,9 @@ class TRANTOR_EXPORT EventLoop : NonCopyable void abortNotInLoopThread(); void wakeup(); void wakeupRead(); - bool looping_; + std::atomic looping_; std::thread::id threadId_; - bool quit_; + std::atomic quit_; std::unique_ptr poller_; ChannelList activeChannels_; From d489ccec5ba5c8568dfb98eb0092a77b94d3eda7 Mon Sep 17 00:00:00 2001 From: An Tao Date: Thu, 27 Jan 2022 22:50:55 +0800 Subject: [PATCH 058/182] Add r-reference version of set-callback methods to TcpConnectionImpl class (#200) --- trantor/net/inner/TcpConnectionImpl.h | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/trantor/net/inner/TcpConnectionImpl.h b/trantor/net/inner/TcpConnectionImpl.h index a004a8ba..6c4fb4bd 100755 --- a/trantor/net/inner/TcpConnectionImpl.h +++ b/trantor/net/inner/TcpConnectionImpl.h @@ -230,23 +230,42 @@ class TcpConnectionImpl : public TcpConnection, { recvMsgCallback_ = cb; } + void setRecvMsgCallback(RecvMessageCallback &&cb) + { + recvMsgCallback_ = std::move(cb); + } void setConnectionCallback(const ConnectionCallback &cb) { connectionCallback_ = cb; } + void setConnectionCallback(ConnectionCallback &&cb) + { + connectionCallback_ = std::move(cb); + } void setWriteCompleteCallback(const WriteCompleteCallback &cb) { writeCompleteCallback_ = cb; } + void setWriteCompleteCallback(WriteCompleteCallback &&cb) + { + writeCompleteCallback_ = std::move(cb); + } void setCloseCallback(const CloseCallback &cb) { closeCallback_ = cb; } + void setCloseCallback(CloseCallback &&cb) + { + closeCallback_ = std::move(cb); + } void setSSLErrorCallback(const SSLErrorCallback &cb) { sslErrorCallback_ = cb; } - + void setSSLErrorCallback(SSLErrorCallback &&cb) + { + sslErrorCallback_ = std::move(cb); + } void connectDestroyed(); virtual void connectEstablished(); From 59d0ba7366761b57577de6945f8e7df09d7bb4da Mon Sep 17 00:00:00 2001 From: An Tao Date: Fri, 28 Jan 2022 16:16:24 +0800 Subject: [PATCH 059/182] Fix a bug when closing connections on Windows/MacOS (#202) --- trantor/net/TcpServer.cc | 36 ++++++++++++++++++++++-------------- trantor/net/TcpServer.h | 1 + 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/trantor/net/TcpServer.cc b/trantor/net/TcpServer.cc index a97ddfa4..0eefa987 100644 --- a/trantor/net/TcpServer.cc +++ b/trantor/net/TcpServer.cc @@ -162,30 +162,38 @@ void TcpServer::stop() f.get(); } } -void TcpServer::connectionClosed(const TcpConnectionPtr &connectionPtr) +void TcpServer::handleCloseInLoop(const TcpConnectionPtr &connectionPtr) { - LOG_TRACE << "connectionClosed"; - if (loop_->isInLoopThread()) + size_t n = connSet_.erase(connectionPtr); + (void)n; + assert(n == 1); + auto connLoop = connectionPtr->getLoop(); + if (connLoop == loop_) { - size_t n = connSet_.erase(connectionPtr); - (void)n; - assert(n == 1); - loop_->queueInLoop([connectionPtr]() { - static_cast(connectionPtr.get()) - ->connectDestroyed(); - }); + static_cast(connectionPtr.get()) + ->connectDestroyed(); } else { - loop_->queueInLoop([this, connectionPtr]() { - size_t n = connSet_.erase(connectionPtr); - (void)n; - assert(n == 1); + connLoop->queueInLoop([connectionPtr]() { static_cast(connectionPtr.get()) ->connectDestroyed(); }); } } +void TcpServer::connectionClosed(const TcpConnectionPtr &connectionPtr) +{ + LOG_TRACE << "connectionClosed"; + if (loop_->isInLoopThread()) + { + handleCloseInLoop(connectionPtr); + } + else + { + loop_->queueInLoop( + [this, connectionPtr]() { handleCloseInLoop(connectionPtr); }); + } +} const std::string TcpServer::ipPort() const { diff --git a/trantor/net/TcpServer.h b/trantor/net/TcpServer.h index 194fbb24..f141cfb5 100644 --- a/trantor/net/TcpServer.h +++ b/trantor/net/TcpServer.h @@ -215,6 +215,7 @@ class TRANTOR_EXPORT TcpServer : NonCopyable private: EventLoop *loop_; + void handleCloseInLoop(const TcpConnectionPtr &connectionPtr); std::unique_ptr acceptorPtr_; void newConnection(int fd, const InetAddress &peer); std::string serverName_; From 9adb3652c4c983f0f430b03e18a65f1cd0dccae4 Mon Sep 17 00:00:00 2001 From: Martin Chang Date: Sat, 12 Feb 2022 21:54:36 +0800 Subject: [PATCH 060/182] Fix logger causes if statement mismatch (#203) --- trantor/tests/CMakeLists.txt | 4 +- trantor/tests/LoggerMacroTest.cc | 12 +++++ trantor/utils/Logger.h | 86 +++++++++++++++++--------------- 3 files changed, 62 insertions(+), 40 deletions(-) create mode 100644 trantor/tests/LoggerMacroTest.cc diff --git a/trantor/tests/CMakeLists.txt b/trantor/tests/CMakeLists.txt index 052dd49e..82d7468f 100644 --- a/trantor/tests/CMakeLists.txt +++ b/trantor/tests/CMakeLists.txt @@ -20,6 +20,7 @@ add_executable(delayed_ssl_server_test DelayedSSLServerTest.cc) add_executable(delayed_ssl_client_test DelayedSSLClientTest.cc) add_executable(run_on_quit_test RunOnQuitTest.cc) add_executable(path_conversion_test PathConversionTest.cc) +add_executable(logger_macro_test LoggerMacroTest.cc) set(targets_list ssl_server_test ssl_client_test @@ -42,7 +43,8 @@ set(targets_list delayed_ssl_server_test delayed_ssl_client_test run_on_quit_test - path_conversion_test) + path_conversion_test + logger_macro_test) set_property(TARGET ${targets_list} PROPERTY CXX_STANDARD 14) set_property(TARGET ${targets_list} PROPERTY CXX_STANDARD_REQUIRED ON) diff --git a/trantor/tests/LoggerMacroTest.cc b/trantor/tests/LoggerMacroTest.cc new file mode 100644 index 00000000..f510aac7 --- /dev/null +++ b/trantor/tests/LoggerMacroTest.cc @@ -0,0 +1,12 @@ +#include + +using namespace trantor; + +int main() +{ + trantor::Logger::setLogLevel(trantor::Logger::kInfo); + if (0) + LOG_INFO << "dummy"; + else + LOG_WARN << "it works"; +} \ No newline at end of file diff --git a/trantor/utils/Logger.h b/trantor/utils/Logger.h index a1874cc1..862186cd 100644 --- a/trantor/utils/Logger.h +++ b/trantor/utils/Logger.h @@ -23,6 +23,8 @@ #include #include +#define TRANTOR_IF_(cond) for (int _r = 0; _r == 0 && (cond); _r = 1) + namespace trantor { /** @@ -219,16 +221,16 @@ class TRANTOR_EXPORT RawLogger : public NonCopyable }; #ifdef NDEBUG #define LOG_TRACE \ - if (0) \ + TRANTOR_IF_(0) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kTrace, __func__) \ .stream() #else #define LOG_TRACE \ - if (trantor::Logger::logLevel() <= trantor::Logger::kTrace) \ + TRANTOR_IF_(trantor::Logger::logLevel() <= trantor::Logger::kTrace) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kTrace, __func__) \ .stream() #define LOG_TRACE_TO(index) \ - if (trantor::Logger::logLevel() <= trantor::Logger::kTrace) \ + TRANTOR_IF_(trantor::Logger::logLevel() <= trantor::Logger::kTrace) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kTrace, __func__) \ .setIndex(index) \ .stream() @@ -236,19 +238,19 @@ class TRANTOR_EXPORT RawLogger : public NonCopyable #endif #define LOG_DEBUG \ - if (trantor::Logger::logLevel() <= trantor::Logger::kDebug) \ + TRANTOR_IF_(trantor::Logger::logLevel() <= trantor::Logger::kDebug) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kDebug, __func__) \ .stream() #define LOG_DEBUG_TO(index) \ - if (trantor::Logger::logLevel() <= trantor::Logger::kDebug) \ + TRANTOR_IF_(trantor::Logger::logLevel() <= trantor::Logger::kDebug) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kDebug, __func__) \ .setIndex(index) \ .stream() -#define LOG_INFO \ - if (trantor::Logger::logLevel() <= trantor::Logger::kInfo) \ +#define LOG_INFO \ + TRANTOR_IF_(trantor::Logger::logLevel() <= trantor::Logger::kInfo) \ trantor::Logger(__FILE__, __LINE__).stream() -#define LOG_INFO_TO(index) \ - if (trantor::Logger::logLevel() <= trantor::Logger::kInfo) \ +#define LOG_INFO_TO(index) \ + TRANTOR_IF_(trantor::Logger::logLevel() <= trantor::Logger::kInfo) \ trantor::Logger(__FILE__, __LINE__).setIndex(index).stream() #define LOG_WARN \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kWarn).stream() @@ -276,79 +278,82 @@ class TRANTOR_EXPORT RawLogger : public NonCopyable #define LOG_RAW_TO(index) trantor::RawLogger().setIndex(index).stream() #define LOG_TRACE_IF(cond) \ - if ((trantor::Logger::logLevel() <= trantor::Logger::kTrace) && (cond)) \ + TRANTOR_IF_((trantor::Logger::logLevel() <= trantor::Logger::kTrace) && \ + (cond)) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kTrace, __func__) \ .stream() #define LOG_DEBUG_IF(cond) \ - if ((trantor::Logger::logLevel() <= trantor::Logger::kDebug) && (cond)) \ + TRANTOR_IF_((trantor::Logger::logLevel() <= trantor::Logger::kDebug) && \ + (cond)) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kDebug, __func__) \ .stream() #define LOG_INFO_IF(cond) \ - if ((trantor::Logger::logLevel() <= trantor::Logger::kInfo) && (cond)) \ + TRANTOR_IF_((trantor::Logger::logLevel() <= trantor::Logger::kInfo) && \ + (cond)) \ trantor::Logger(__FILE__, __LINE__).stream() #define LOG_WARN_IF(cond) \ - if (cond) \ + TRANTOR_IF_(cond) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kWarn).stream() #define LOG_ERROR_IF(cond) \ - if (cond) \ + TRANTOR_IF_(cond) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kError).stream() #define LOG_FATAL_IF(cond) \ - if (cond) \ + TRANTOR_IF_(cond) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kFatal).stream() #ifdef NDEBUG #define DLOG_TRACE \ - if (0) \ + TRANTOR_IF_(0) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kTrace, __func__) \ .stream() #define DLOG_DEBUG \ - if (0) \ + TRANTOR_IF_(0) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kDebug, __func__) \ .stream() -#define DLOG_INFO \ - if (0) \ +#define DLOG_INFO \ + TRANTOR_IF_(0) \ trantor::Logger(__FILE__, __LINE__).stream() -#define DLOG_WARN \ - if (0) \ +#define DLOG_WARN \ + TRANTOR_IF_(0) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kWarn).stream() #define DLOG_ERROR \ - if (0) \ + TRANTOR_IF_(0) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kError).stream() #define DLOG_FATAL \ - if (0) \ + TRANTOR_IF_(0) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kFatal).stream() #define DLOG_TRACE_IF(cond) \ - if (0) \ + TRANTOR_IF_(0) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kTrace, __func__) \ .stream() #define DLOG_DEBUG_IF(cond) \ - if (0) \ + TRANTOR_IF_(0) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kDebug, __func__) \ .stream() #define DLOG_INFO_IF(cond) \ - if (0) \ + TRANTOR_IF_(0) \ trantor::Logger(__FILE__, __LINE__).stream() #define DLOG_WARN_IF(cond) \ - if (0) \ + TRANTOR_IF_(0) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kWarn).stream() #define DLOG_ERROR_IF(cond) \ - if (0) \ + TRANTOR_IF_(0) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kError).stream() #define DLOG_FATAL_IF(cond) \ - if (0) \ + TRANTOR_IF_(0) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kFatal).stream() #else #define DLOG_TRACE \ - if (trantor::Logger::logLevel() <= trantor::Logger::kTrace) \ + TRANTOR_IF_(trantor::Logger::logLevel() <= trantor::Logger::kTrace) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kTrace, __func__) \ .stream() #define DLOG_DEBUG \ - if (trantor::Logger::logLevel() <= trantor::Logger::kDebug) \ + TRANTOR_IF_(trantor::Logger::logLevel() <= trantor::Logger::kDebug) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kDebug, __func__) \ .stream() -#define DLOG_INFO \ - if (trantor::Logger::logLevel() <= trantor::Logger::kInfo) \ +#define DLOG_INFO \ + TRANTOR_IF_(trantor::Logger::logLevel() <= trantor::Logger::kInfo) \ trantor::Logger(__FILE__, __LINE__).stream() #define DLOG_WARN \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kWarn).stream() @@ -358,24 +363,27 @@ class TRANTOR_EXPORT RawLogger : public NonCopyable trantor::Logger(__FILE__, __LINE__, trantor::Logger::kFatal).stream() #define DLOG_TRACE_IF(cond) \ - if ((trantor::Logger::logLevel() <= trantor::Logger::kTrace) && (cond)) \ + TRANTOR_IF_((trantor::Logger::logLevel() <= trantor::Logger::kTrace) && \ + (cond)) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kTrace, __func__) \ .stream() #define DLOG_DEBUG_IF(cond) \ - if ((trantor::Logger::logLevel() <= trantor::Logger::kDebug) && (cond)) \ + TRANTOR_IF_((trantor::Logger::logLevel() <= trantor::Logger::kDebug) && \ + (cond)) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kDebug, __func__) \ .stream() #define DLOG_INFO_IF(cond) \ - if ((trantor::Logger::logLevel() <= trantor::Logger::kInfo) && (cond)) \ + TRANTOR_IF_((trantor::Logger::logLevel() <= trantor::Logger::kInfo) && \ + (cond)) \ trantor::Logger(__FILE__, __LINE__).stream() #define DLOG_WARN_IF(cond) \ - if (cond) \ + TRANTOR_IF_(cond) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kWarn).stream() #define DLOG_ERROR_IF(cond) \ - if (cond) \ + TRANTOR_IF_(cond) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kError).stream() #define DLOG_FATAL_IF(cond) \ - if (cond) \ + TRANTOR_IF_(cond) \ trantor::Logger(__FILE__, __LINE__, trantor::Logger::kFatal).stream() #endif From 0a7c2002496083c7a19ec92ab9fce6b4f8430d80 Mon Sep 17 00:00:00 2001 From: an-tao Date: Sat, 19 Feb 2022 18:20:52 +0800 Subject: [PATCH 061/182] Bump version to 1.5.5 --- CMakeLists.txt | 2 +- ChangeLog.md | 26 +++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 99687442..f4454aae 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) set(TRANTOR_MAJOR_VERSION 1) set(TRANTOR_MINOR_VERSION 5) -set(TRANTOR_PATCH_VERSION 4) +set(TRANTOR_PATCH_VERSION 5) set(TRANTOR_VERSION ${TRANTOR_MAJOR_VERSION}.${TRANTOR_MINOR_VERSION}.${TRANTOR_PATCH_VERSION}) diff --git a/ChangeLog.md b/ChangeLog.md index 026d2f58..3c2c05c4 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -3,6 +3,28 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [1.5.5] - 2022-02-19 + +### API changes list + +### Changed + +- Move EventLoop::runAfter to a template. + +- Remove an assertion when removing channels. + +- Prevent TcpClient::removeConnection call on deleted TcpClient instance. + +- Wait for loop to exit in EventLoop destructor. + +- Add r-reference version of set-callback methods to TcpConnectionImpl. + +### Fixed + +- Fix a bug when closing connections on Windows/MacOS. + +- Fix logger causes if statement mismatch. + ## [1.5.4] - 2021-12-10 ### API changes list @@ -390,7 +412,9 @@ All notable changes to this project will be documented in this file. ## [1.0.0-rc1] - 2019-06-11 -[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.4...HEAD +[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.5...HEAD + +[1.5.5]: https://github.com/an-tao/trantor/compare/v1.5.4...v1.5.5 [1.5.4]: https://github.com/an-tao/trantor/compare/v1.5.3...v1.5.4 From 56dc2617943acbcc996477d47305a5e91307f087 Mon Sep 17 00:00:00 2001 From: LE GARREC Vincent Date: Fri, 8 Apr 2022 10:34:33 +0200 Subject: [PATCH 062/182] Fix tolower with sanitizer cfi (#208) --- trantor/net/TcpClient.cc | 2 +- trantor/net/inner/TcpConnectionImpl.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/trantor/net/TcpClient.cc b/trantor/net/TcpClient.cc index 6de3555b..a4a3995b 100644 --- a/trantor/net/TcpClient.cc +++ b/trantor/net/TcpClient.cc @@ -238,7 +238,7 @@ void TcpClient::enableSSL( std::transform(hostname.begin(), hostname.end(), hostname.begin(), - tolower); + [](unsigned char c) { return tolower(c); }); SSLHostName_ = std::move(hostname); } diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index 2f11e451..5be981d6 100644 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -554,7 +554,7 @@ void TcpConnectionImpl::startClientEncryption( std::transform(hostname.begin(), hostname.end(), hostname.begin(), - tolower); + [](unsigned char c) { return tolower(c); }); assert(sslEncryptionPtr_ != nullptr); sslEncryptionPtr_->hostname_ = hostname; } From 16fb3f956b78c1c9019f183b5b27ff38ca8eaaf5 Mon Sep 17 00:00:00 2001 From: "Joakim L. Gilje" Date: Fri, 8 Apr 2022 10:35:21 +0200 Subject: [PATCH 063/182] include unconditionally on freebsd (#207) --- trantor/utils/Logger.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/trantor/utils/Logger.cc b/trantor/utils/Logger.cc index a8c26024..d159dcc3 100644 --- a/trantor/utils/Logger.cc +++ b/trantor/utils/Logger.cc @@ -24,7 +24,9 @@ #include #elif defined _WIN32 #include -#elif defined __FreeBSD__ +#endif + +#if defined __FreeBSD__ #include #endif From 55868747c82608989e07664b625c226d023fc50f Mon Sep 17 00:00:00 2001 From: Nitromelon Date: Sat, 9 Apr 2022 13:30:00 +0800 Subject: [PATCH 064/182] Make MsgBuffer constructor explicit. (#209) --- trantor/utils/MsgBuffer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trantor/utils/MsgBuffer.h b/trantor/utils/MsgBuffer.h index 33c8633f..29817f6a 100644 --- a/trantor/utils/MsgBuffer.h +++ b/trantor/utils/MsgBuffer.h @@ -43,7 +43,7 @@ class TRANTOR_EXPORT MsgBuffer * * @param len The initial size of the buffer. */ - MsgBuffer(size_t len = kBufferDefaultLength); + explicit MsgBuffer(size_t len = kBufferDefaultLength); /** * @brief Get the beginning of the buffer. From e43e8ca882fe0eb7fa76887cc576e304da359b96 Mon Sep 17 00:00:00 2001 From: Nitromelon Date: Sat, 9 Apr 2022 13:30:15 +0800 Subject: [PATCH 065/182] Always queue connectDestroyed() in loop. (#206) --- trantor/net/TcpServer.cc | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/trantor/net/TcpServer.cc b/trantor/net/TcpServer.cc index 0eefa987..1990e87c 100644 --- a/trantor/net/TcpServer.cc +++ b/trantor/net/TcpServer.cc @@ -168,18 +168,15 @@ void TcpServer::handleCloseInLoop(const TcpConnectionPtr &connectionPtr) (void)n; assert(n == 1); auto connLoop = connectionPtr->getLoop(); - if (connLoop == loop_) - { + + // NOTE: always queue this operation in connLoop, because this connection + // may be in loop_'s current active channels, waiting to be processed. + // If `connectDestroyed()` is called here, we will be using an wild pointer + // later. + connLoop->queueInLoop([connectionPtr]() { static_cast(connectionPtr.get()) ->connectDestroyed(); - } - else - { - connLoop->queueInLoop([connectionPtr]() { - static_cast(connectionPtr.get()) - ->connectDestroyed(); - }); - } + }); } void TcpServer::connectionClosed(const TcpConnectionPtr &connectionPtr) { From f308457328dc5b59f290cb3b03258c40dc859b03 Mon Sep 17 00:00:00 2001 From: Nitromelon Date: Sat, 9 Apr 2022 13:30:47 +0800 Subject: [PATCH 066/182] Stop calling abort() in runtime. (#204) --- trantor/net/TcpClient.cc | 4 ++-- trantor/net/TcpServer.cc | 4 ++-- trantor/net/inner/TcpConnectionImpl.cc | 18 +++++++++--------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/trantor/net/TcpClient.cc b/trantor/net/TcpClient.cc index a4a3995b..2058dfc8 100644 --- a/trantor/net/TcpClient.cc +++ b/trantor/net/TcpClient.cc @@ -153,7 +153,7 @@ void TcpClient::newConnection(int sockfd) SSLHostName_); #else LOG_FATAL << "OpenSSL is not found in your system!"; - abort(); + throw std::runtime_error("OpenSSL is not found in your system!"); #endif } else @@ -253,6 +253,6 @@ void TcpClient::enableSSL( (void)keyPath; LOG_FATAL << "OpenSSL is not found in your system!"; - abort(); + throw std::runtime_error("OpenSSL is not found in your system!"); #endif } diff --git a/trantor/net/TcpServer.cc b/trantor/net/TcpServer.cc index 1990e87c..7fc7180b 100644 --- a/trantor/net/TcpServer.cc +++ b/trantor/net/TcpServer.cc @@ -76,7 +76,7 @@ void TcpServer::newConnection(int sockfd, const InetAddress &peer) sslCtxPtr_); #else LOG_FATAL << "OpenSSL is not found in your system!"; - abort(); + throw std::runtime_error("OpenSSL is not found in your system!"); #endif } else @@ -220,6 +220,6 @@ void TcpServer::enableSSL( (void)sslConfCmds; LOG_FATAL << "OpenSSL is not found in your system!"; - abort(); + throw std::runtime_error("OpenSSL is not found in your system!"); #endif } diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index 5be981d6..2a140235 100644 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -327,7 +327,7 @@ std::shared_ptr newSSLServerContext( { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); LOG_FATAL << "Reading certificate: " << errbuf; - abort(); + throw std::runtime_error("SSL_CTX_use_certificate_chain_file error."); } r = SSL_CTX_use_PrivateKey_file(ctx->get(), keyPath.c_str(), @@ -336,14 +336,14 @@ std::shared_ptr newSSLServerContext( { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); LOG_FATAL << "Reading private key: " << errbuf; - abort(); + throw std::runtime_error("SSL_CTX_use_PrivateKey_file error"); } r = SSL_CTX_check_private_key(ctx->get()); if (!r) { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); LOG_FATAL << "Checking private key matches certificate: " << errbuf; - abort(); + throw std::runtime_error("SSL_CTX_check_private_key error"); } return ctx; } @@ -364,7 +364,7 @@ std::shared_ptr newSSLClientContext( { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); LOG_FATAL << "Reading certificate: " << errbuf; - abort(); + throw std::runtime_error("SSL_CTX_use_certificate_chain_file error."); } r = SSL_CTX_use_PrivateKey_file(ctx->get(), keyPath.c_str(), @@ -373,14 +373,14 @@ std::shared_ptr newSSLClientContext( { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); LOG_FATAL << "Reading private key: " << errbuf; - abort(); + throw std::runtime_error("SSL_CTX_use_PrivateKey_file error."); } r = SSL_CTX_check_private_key(ctx->get()); if (!r) { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); LOG_FATAL << "Checking private key matches certificate: " << errbuf; - abort(); + throw std::runtime_error("SSL_CTX_check_private_key error."); } return ctx; } @@ -395,7 +395,7 @@ std::shared_ptr newSSLServerContext( const std::vector> &) { LOG_FATAL << "OpenSSL is not found in your system!"; - abort(); + throw std::runtime_error("OpenSSL is not found in your system!"); } } // namespace trantor #endif @@ -513,7 +513,7 @@ void TcpConnectionImpl::startServerEncryption( (void)callback; LOG_FATAL << "OpenSSL is not found in your system!"; - abort(); + throw std::runtime_error("OpenSSL is not found in your system!"); #else if (loop_->isInLoopThread()) { @@ -547,7 +547,7 @@ void TcpConnectionImpl::startClientEncryption( (void)sslConfCmds; LOG_FATAL << "OpenSSL is not found in your system!"; - abort(); + throw std::runtime_error("OpenSSL is not found in your system!"); #else if (!hostname.empty()) { From c1e57a06e2e825c937e12a25efe847f800547aeb Mon Sep 17 00:00:00 2001 From: Greisberger Christophe Date: Wed, 20 Apr 2022 03:39:34 +0200 Subject: [PATCH 067/182] Add support for sending data streams via callback (TcpConnection::sendStream()) (#196) --- CMakeLists.txt | 3 + trantor/net/TcpConnection.h | 13 + trantor/net/inner/Socket.cc | 4 + trantor/net/inner/TcpConnectionImpl.cc | 317 ++++++++++++++++++++----- trantor/net/inner/TcpConnectionImpl.h | 30 ++- trantor/tests/CMakeLists.txt | 2 + trantor/tests/SendstreamTest.cc | 146 ++++++++++++ 7 files changed, 457 insertions(+), 58 deletions(-) mode change 100755 => 100644 trantor/net/TcpConnection.h mode change 100644 => 100755 trantor/net/inner/Socket.cc mode change 100755 => 100644 trantor/net/inner/TcpConnectionImpl.h create mode 100644 trantor/tests/SendstreamTest.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index f4454aae..4e553e8a 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -148,6 +148,9 @@ if(HAVE_C-ARES) set(private_headers ${private_headers} trantor/net/inner/AresResolver.h) + if(APPLE) + target_link_libraries(${PROJECT_NAME} PRIVATE resolv) + endif() else() set(TRANTOR_SOURCES ${TRANTOR_SOURCES} diff --git a/trantor/net/TcpConnection.h b/trantor/net/TcpConnection.h old mode 100755 new mode 100644 index 88151d80..49e781c6 --- a/trantor/net/TcpConnection.h +++ b/trantor/net/TcpConnection.h @@ -76,6 +76,19 @@ class TRANTOR_EXPORT TcpConnection virtual void sendFile(const wchar_t *fileName, size_t offset = 0, size_t length = 0) = 0; + /** + * @brief Send a stream to the peer. + * + * @param callback function to retrieve the stream data (stream ends when a + * zero size is returned) the callback will be called with nullptr when the + * send is finished/interrupted, so that it cleans up any internal data (ex: + * close file). + * @warning The buffer size should be >= 10 to allow http chunked-encoding + * data stream + */ + virtual void sendStream(std::function + callback) = 0; // (buffer, buffer size) -> size + // of data put in buffer /** * @brief Get the local address of the connection. diff --git a/trantor/net/inner/Socket.cc b/trantor/net/inner/Socket.cc old mode 100644 new mode 100755 index c9eb58db..2aff5a03 --- a/trantor/net/inner/Socket.cc +++ b/trantor/net/inner/Socket.cc @@ -228,7 +228,11 @@ int Socket::getSocketError() if (::getsockopt(sockFd_, SOL_SOCKET, SO_ERROR, &optval, &optlen) < 0) { +#ifdef _WIN32 + return ::WSAGetLastError(); +#else return errno; +#endif } else { diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index 2a140235..cd331e42 100644 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -38,6 +38,17 @@ using namespace trantor; +#ifdef _WIN32 +// Winsock does not set errno, and WSAGetLastError() has different values than +// errno socket errors +#undef EWOULDBLOCK +#define EWOULDBLOCK WSAEWOULDBLOCK +#undef EPIPE +#define EPIPE WSAENOTCONN +#undef ECONNRESET +#define ECONNRESET WSAECONNRESET +#endif + #ifdef USE_OPENSSL namespace trantor { @@ -602,11 +613,15 @@ void TcpConnectionImpl::readCallback() } else if (n < 0) { - if (errno == EPIPE || errno == ECONNRESET || - errno == EAGAIN) // TODO: any others? + if (errno == EPIPE || errno == ECONNRESET) { - LOG_DEBUG << "EPIPE or ECONNRESET, errno=" << errno +#ifdef _WIN32 + LOG_DEBUG << "WSAENOTCONN or WSAECONNRESET, errno=" << errno << " fd=" << socketPtr_->fd(); +#else + LOG_DEBUG << "EPIPE or ECONNRESET, errno=" << errno + << " fd=" << socketPtr_->fd(); +#endif return; } #ifdef _WIN32 @@ -616,6 +631,13 @@ void TcpConnectionImpl::readCallback() handleClose(); return; } +#else + if (errno == EAGAIN) // TODO: any others? + { + LOG_DEBUG << "EAGAIN, errno=" << errno + << " fd=" << socketPtr_->fd(); + return; + } #endif LOG_SYSERR << "read socket error"; handleClose(); @@ -714,19 +736,17 @@ void TcpConnectionImpl::writeCallback() { assert(!writeBufferList_.empty()); auto writeBuffer_ = writeBufferList_.front(); -#ifndef _WIN32 - if (writeBuffer_->sendFd_ < 0) -#else - if (writeBuffer_->sendFp_ == nullptr) -#endif + if (!writeBuffer_->isFile()) { + // not a file if (writeBuffer_->msgBuffer_->readableBytes() <= 0) { + // finished sending writeBufferList_.pop_front(); if (writeBufferList_.empty()) { + // stop writing ioChannelPtr_->disableWriting(); - // if (writeCompleteCallback_) writeCompleteCallback_(shared_from_this()); if (status_ == ConnStatus::Disconnecting) @@ -736,17 +756,16 @@ void TcpConnectionImpl::writeCallback() } else { + // send next + // what if the next is not a file??? auto fileNode = writeBufferList_.front(); -#ifndef _WIN32 - assert(fileNode->sendFd_ >= 0); -#else - assert(fileNode->sendFp_); -#endif + assert(fileNode->isFile()); sendFileInLoop(fileNode); } } else { + // continue sending auto n = writeInLoop(writeBuffer_->msgBuffer_->peek(), writeBuffer_->msgBuffer_->readableBytes()); @@ -765,8 +784,13 @@ void TcpConnectionImpl::writeCallback() // TODO: any others? if (errno == EPIPE || errno == ECONNRESET) { - LOG_DEBUG << "EPIPE or ECONNRESET, erron=" - << errno; +#ifdef _WIN32 + LOG_DEBUG + << "WSAENOTCONN or WSAECONNRESET, errno=" + << errno; +#else + LOG_DEBUG << "EPIPE or ECONNRESET, errno=" << errno; +#endif return; } LOG_SYSERR << "Unexpected error(" << errno << ")"; @@ -777,12 +801,14 @@ void TcpConnectionImpl::writeCallback() } else { - // file + // is a file if (writeBuffer_->fileBytesToSend_ <= 0) { + // finished sending writeBufferList_.pop_front(); if (writeBufferList_.empty()) { + // stop writing ioChannelPtr_->disableWriting(); if (writeCompleteCallback_) writeCompleteCallback_(shared_from_this()); @@ -793,11 +819,8 @@ void TcpConnectionImpl::writeCallback() } else { -#ifndef _WIN32 - if (writeBufferList_.front()->sendFd_ < 0) -#else - if (writeBufferList_.front()->sendFp_ == nullptr) -#endif + // next is not a file + if (!writeBufferList_.front()->isFile()) { // There is data to be sent in the buffer. auto n = writeInLoop( @@ -820,9 +843,15 @@ void TcpConnectionImpl::writeCallback() // TODO: any others? if (errno == EPIPE || errno == ECONNRESET) { - LOG_DEBUG - << "EPIPE or ECONNRESET, erron=" - << errno; +#ifdef _WIN32 + LOG_DEBUG << "WSAENOTCONN or " + "WSAECONNRESET, errno=" + << errno; +#else + LOG_DEBUG << "EPIPE or " + "ECONNRESET, erron=" + << errno; +#endif return; } LOG_SYSERR << "Unexpected error(" << errno @@ -833,7 +862,7 @@ void TcpConnectionImpl::writeCallback() } else { - // more file + // next is a file sendFileInLoop(writeBufferList_.front()); } } @@ -846,7 +875,7 @@ void TcpConnectionImpl::writeCallback() } else { - LOG_SYSERR << "no writing but call write callback"; + LOG_SYSERR << "no writing but write callback called"; } #ifdef USE_OPENSSL } @@ -925,7 +954,11 @@ void TcpConnectionImpl::handleError() int err = socketPtr_->getSocketError(); if (err == 0) return; - if (err == EPIPE || err == ECONNRESET || err == 104) + if (err == EPIPE || +#ifndef _WIN32 + err == EBADMSG || // ??? 104=EBADMSG +#endif + err == ECONNRESET) { LOG_DEBUG << "[" << name_ << "] - SO_ERROR = " << err << " " << strerror_tl(err); @@ -1009,7 +1042,12 @@ void TcpConnectionImpl::sendInLoop(const char *buffer, size_t length) { if (errno == EPIPE || errno == ECONNRESET) // TODO: any others? { - LOG_DEBUG << "EPIPE or ECONNRESET, erron=" << errno; +#ifdef _WIN32 + LOG_DEBUG << "WSAENOTCONN or WSAECONNRESET, errno=" + << errno; +#else + LOG_DEBUG << "EPIPE or ECONNRESET, errno=" << errno; +#endif return; } LOG_SYSERR << "Unexpected error(" << errno << ")"; @@ -1027,11 +1065,7 @@ void TcpConnectionImpl::sendInLoop(const char *buffer, size_t length) node->msgBuffer_ = std::make_shared(); writeBufferList_.push_back(std::move(node)); } -#ifndef _WIN32 - else if (writeBufferList_.back()->sendFd_ >= 0) -#else - else if (writeBufferList_.back()->sendFp_) -#endif + else if (writeBufferList_.back()->isFile()) { BufferNodePtr node = std::make_shared(); node->msgBuffer_ = std::make_shared(); @@ -1461,17 +1495,74 @@ void TcpConnectionImpl::sendFile(FILE *fp, size_t offset, size_t length) } } +void TcpConnectionImpl::sendStream( + std::function callback) +{ + BufferNodePtr node = std::make_shared(); + node->offset_ = + 0; // not used, the offset should be handled by the callback + node->fileBytesToSend_ = 1; // force to > 0 until stream sent + node->streamCallback_ = std::move(callback); + if (loop_->isInLoopThread()) + { + std::lock_guard guard(sendNumMutex_); + if (sendNum_ == 0) + { + writeBufferList_.push_back(node); + if (writeBufferList_.size() == 1) + { + sendFileInLoop(writeBufferList_.front()); + return; + } + } + else + { + ++sendNum_; + auto thisPtr = shared_from_this(); + loop_->queueInLoop([thisPtr, node]() { + thisPtr->writeBufferList_.push_back(node); + { + std::lock_guard guard1(thisPtr->sendNumMutex_); + --thisPtr->sendNum_; + } + + if (thisPtr->writeBufferList_.size() == 1) + { + thisPtr->sendFileInLoop(thisPtr->writeBufferList_.front()); + } + }); + } + } + else + { + auto thisPtr = shared_from_this(); + std::lock_guard guard(sendNumMutex_); + ++sendNum_; + loop_->queueInLoop([thisPtr, node]() { + LOG_TRACE << "Push sendstream to list"; + thisPtr->writeBufferList_.push_back(node); + + { + std::lock_guard guard1(thisPtr->sendNumMutex_); + --thisPtr->sendNum_; + } + + if (thisPtr->writeBufferList_.size() == 1) + { + thisPtr->sendFileInLoop(thisPtr->writeBufferList_.front()); + } + }); + } +} + void TcpConnectionImpl::sendFileInLoop(const BufferNodePtr &filePtr) { loop_->assertInLoopThread(); -#ifndef _WIN32 - assert(filePtr->sendFd_ >= 0); -#else - assert(filePtr->sendFp_); -#endif + assert(filePtr->isFile()); #ifdef __linux__ - if (!isEncrypted_) + if (!isEncrypted_ && !filePtr->streamCallback_) { + LOG_TRACE << "send file in loop using linux kernel sendfile()"; auto bytesSent = sendfile(socketPtr_->fd(), filePtr->sendFd_, &filePtr->offset_, @@ -1503,12 +1594,108 @@ void TcpConnectionImpl::sendFileInLoop(const BufferNodePtr &filePtr) return; } #endif -#ifndef _WIN32 - lseek(filePtr->sendFd_, filePtr->offset_, SEEK_SET); + // Send stream + if (filePtr->streamCallback_) + { + LOG_TRACE << "send stream in loop"; + if (!fileBufferPtr_) + { + fileBufferPtr_ = std::make_unique>(); + fileBufferPtr_->reserve(16 * 1024); + } + while ((filePtr->fileBytesToSend_ > 0) || !fileBufferPtr_->empty()) + { + // get next chunk + if (fileBufferPtr_->empty()) + { + // LOG_TRACE << "send stream in loop: fetch data + // on buffer empty"; + fileBufferPtr_->resize(16 * 1024); + std::size_t nData; + nData = filePtr->streamCallback_(fileBufferPtr_->data(), + fileBufferPtr_->size()); + fileBufferPtr_->resize(nData); + if (nData == 0) // no more data! + { + LOG_TRACE << "send stream in loop: no more data"; + filePtr->fileBytesToSend_ = 0; + } + } + if (fileBufferPtr_->empty()) + { + LOG_TRACE << "send stream in loop: break on buffer empty"; + break; + } + auto nToWrite = fileBufferPtr_->size(); + auto nWritten = writeInLoop(fileBufferPtr_->data(), nToWrite); + if (nWritten >= 0) + { +#ifndef NDEBUG // defined by CMake for release build + filePtr->nDataWritten_ += nWritten; + LOG_TRACE << "send stream in loop: bytes written: " << nWritten + << " / total bytes written: " + << filePtr->nDataWritten_; +#endif + if (static_cast(nWritten) < nToWrite) + { + // Partial write - return and wait for next call to continue + fileBufferPtr_->erase(fileBufferPtr_->begin(), + fileBufferPtr_->begin() + nWritten); + if (!ioChannelPtr_->isWriting()) + ioChannelPtr_->enableWriting(); + LOG_TRACE << "send stream in loop: return on partial write " + "(socket buffer full?)"; + return; + } + // LOG_TRACE << "send stream in loop: continue on + // data written"; + fileBufferPtr_->resize(0); + continue; + } + // nWritten < 0 +#ifdef _WIN32 + if (errno != 0 && errno != EWOULDBLOCK) +#else + if (errno != EWOULDBLOCK) +#endif + { + if (errno == EPIPE || errno == ECONNRESET) + { +#ifdef _WIN32 + LOG_DEBUG << "WSAENOTCONN or WSAECONNRESET, errno=" + << errno; +#else + LOG_DEBUG << "EPIPE or ECONNRESET, errno=" << errno; +#endif + // abort + LOG_DEBUG + << "send stream in loop: return on connection closed"; + filePtr->fileBytesToSend_ = 0; + return; + } + // TODO: any others? + LOG_SYSERR << "send stream in loop: return on unexpected error(" + << errno << ")"; + filePtr->fileBytesToSend_ = 0; + return; + } + // Socket buffer full - return and wait for next call + LOG_TRACE << "send stream in loop: break on socket buffer full (?)"; + break; + } + if (!ioChannelPtr_->isWriting()) + ioChannelPtr_->enableWriting(); + LOG_TRACE << "send stream in loop: return on loop exit"; + return; + } + // Send file + LOG_TRACE << "send file in loop"; if (!fileBufferPtr_) { fileBufferPtr_ = std::make_unique>(16 * 1024); } +#ifndef _WIN32 + lseek(filePtr->sendFd_, filePtr->offset_, SEEK_SET); while (filePtr->fileBytesToSend_ > 0) { auto n = read(filePtr->sendFd_, @@ -1518,12 +1705,9 @@ void TcpConnectionImpl::sendFileInLoop(const BufferNodePtr &filePtr) filePtr->fileBytesToSend_))); #else _fseeki64(filePtr->sendFp_, filePtr->offset_, SEEK_SET); - if (!fileBufferPtr_) - { - fileBufferPtr_ = std::make_unique>(16 * 1024); - } while (filePtr->fileBytesToSend_ > 0) { + // LOG_TRACE << "send file in loop: fetch more remaining data"; auto bytes = static_castsize())>( filePtr->fileBytesToSend_); auto n = fread(&(*fileBufferPtr_)[0], @@ -1545,10 +1729,16 @@ void TcpConnectionImpl::sendFileInLoop(const BufferNodePtr &filePtr) { ioChannelPtr_->enableWriting(); } + LOG_TRACE << "send file in loop: return on partial write " + "(socket buffer full?)"; return; } else if (nSend == n) + { + // LOG_TRACE << "send file in loop: + // continue on data written"; continue; + } } if (nSend < 0) { @@ -1561,28 +1751,41 @@ void TcpConnectionImpl::sendFileInLoop(const BufferNodePtr &filePtr) // TODO: any others? if (errno == EPIPE || errno == ECONNRESET) { - LOG_DEBUG << "EPIPE or ECONNRESET, erron=" << errno; +#ifdef _WIN32 + LOG_DEBUG << "WSAENOTCONN or WSAECONNRESET, errno=" + << errno; +#else + LOG_DEBUG << "EPIPE or ECONNRESET, errno=" << errno; +#endif + LOG_DEBUG + << "send file in loop: return on connection closed"; return; } - LOG_SYSERR << "Unexpected error(" << errno << ")"; + LOG_SYSERR + << "send file in loop: return on unexpected error(" + << errno << ")"; return; } + LOG_TRACE + << "send file in loop: break on socket buffer full (?)"; break; } } if (n < 0) { - LOG_SYSERR << "read error"; + LOG_SYSERR << "send file in loop: return on read error"; if (ioChannelPtr_->isWriting()) ioChannelPtr_->disableWriting(); return; } if (n == 0) { - LOG_SYSERR << "read"; + LOG_SYSERR + << "send file in loop: return on read 0 (file truncated)"; return; } } + LOG_TRACE << "send file in loop: return on loop exit"; if (!ioChannelPtr_->isWriting()) { ioChannelPtr_->enableWriting(); @@ -1597,19 +1800,23 @@ ssize_t TcpConnectionImpl::writeInLoop(const char *buffer, size_t length) #ifdef USE_OPENSSL if (!isEncrypted_) { +// LOG_TRACE << "write in loop"; #endif - bytesSent_ += length; #ifndef _WIN32 - return write(socketPtr_->fd(), buffer, length); + int nWritten = write(socketPtr_->fd(), buffer, length); #else - errno = 0; - return ::send(socketPtr_->fd(), buffer, static_cast(length), 0); + int nWritten = + ::send(socketPtr_->fd(), buffer, static_cast(length), 0); + errno = (nWritten < 0) ? ::WSAGetLastError() : 0; #endif + if (nWritten > 0) + bytesSent_ += nWritten; + return nWritten; #ifdef USE_OPENSSL } else { - LOG_TRACE << "send in loop"; + // LOG_TRACE << "write encrypted in loop"; loop_->assertInLoopThread(); if (status_ != ConnStatus::Connected && status_ != ConnStatus::Disconnecting) diff --git a/trantor/net/inner/TcpConnectionImpl.h b/trantor/net/inner/TcpConnectionImpl.h old mode 100755 new mode 100644 index 6c4fb4bd..d872a09b --- a/trantor/net/inner/TcpConnectionImpl.h +++ b/trantor/net/inner/TcpConnectionImpl.h @@ -125,6 +125,8 @@ class TcpConnectionImpl : public TcpConnection, virtual void sendFile(const wchar_t *fileName, size_t offset = 0, size_t length = 0) override; + virtual void sendStream( + std::function callback) override; virtual const InetAddress &localAddr() const override { @@ -272,15 +274,35 @@ class TcpConnectionImpl : public TcpConnection, protected: struct BufferNode { + // sendFile() specific #ifndef _WIN32 int sendFd_{-1}; - off_t offset_; + off_t offset_{0}; #else FILE *sendFp_{nullptr}; - long long offset_; + long long offset_{0}; #endif - ssize_t fileBytesToSend_; + ssize_t fileBytesToSend_{0}; + // sendStream() specific + std::function streamCallback_; +#ifndef NDEBUG // defined by CMake for release build + std::size_t nDataWritten_{0}; +#endif + // generic std::shared_ptr msgBuffer_; + bool isFile() const + { + if (streamCallback_) + return true; +#ifndef _WIN32 + if (sendFd_ >= 0) + return true; +#else + if (sendFp_) + return true; +#endif + return false; + } ~BufferNode() { #ifndef _WIN32 @@ -290,6 +312,8 @@ class TcpConnectionImpl : public TcpConnection, if (sendFp_) fclose(sendFp_); #endif + if (streamCallback_) + streamCallback_(nullptr, 0); // cleanup callback internals } }; using BufferNodePtr = std::shared_ptr; diff --git a/trantor/tests/CMakeLists.txt b/trantor/tests/CMakeLists.txt index 82d7468f..146f62b2 100644 --- a/trantor/tests/CMakeLists.txt +++ b/trantor/tests/CMakeLists.txt @@ -13,6 +13,7 @@ add_executable(concurrent_task_queue_test ConcurrentTaskQueueTest.cc) add_executable(tcp_client_test TcpClientTest.cc) add_executable(async_file_logger_test1 AsyncFileLoggerTest1.cc) add_executable(sendfile_test SendfileTest.cc) +add_executable(sendstream_test SendstreamTest.cc) add_executable(timing_wheel_test TimingWheelTest.cc) add_executable(kickoff_test KickoffTest.cc) add_executable(dns_test DnsTest.cc) @@ -37,6 +38,7 @@ set(targets_list tcp_client_test async_file_logger_test1 sendfile_test + sendstream_test timing_wheel_test kickoff_test dns_test diff --git a/trantor/tests/SendstreamTest.cc b/trantor/tests/SendstreamTest.cc new file mode 100644 index 00000000..2a0c65a5 --- /dev/null +++ b/trantor/tests/SendstreamTest.cc @@ -0,0 +1,146 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef _WIN32 +#include +#else +#include +#endif + +std::size_t fileCallback(const std::string &, int, char *, std::size_t); + +using namespace trantor; +#define USE_IPV6 0 +int main(int argc, char *argv[]) +{ + if (argc < 2) + { + std::cout << "usage:" << argv[0] << " filename" << std::endl; + return 1; + } + std::cout << "filename:" << argv[1] << std::endl; + struct stat filestat; + if (stat(argv[1], &filestat) < 0) + { + perror(""); + exit(1); + } + std::cout << "file len=" << filestat.st_size << std::endl; + + auto fp = fopen(argv[1], "rb"); + + if (fp == nullptr) + { + perror(""); + exit(1); + } + fclose(fp); + + LOG_DEBUG << "test start"; + + Logger::setLogLevel(Logger::kTrace); + EventLoopThread loopThread; + loopThread.run(); + +#if USE_IPV6 + InetAddress addr(1207, true, true); +#else + InetAddress addr(1207); +#endif + TcpServer server(loopThread.getLoop(), addr, "test"); + server.setRecvMessageCallback( + [](const TcpConnectionPtr &connectionPtr, MsgBuffer *buffer) { + // LOG_DEBUG<<"recv callback!"; + }); + int counter = 0; + server.setConnectionCallback([argv, + &counter](const TcpConnectionPtr &connPtr) { + if (connPtr->connected()) + { + LOG_DEBUG << "New connection"; + std::thread t([connPtr, argv, &counter]() { + for (int i = 0; i < 5; ++i) + { + int fd; +#ifdef _WIN32 + _sopen_s( + &fd, argv[1], _O_BINARY | _O_RDONLY, _SH_DENYNO, 0); +#else + fd = open(argv[1], O_RDONLY); +#endif + auto callback = std::bind(fileCallback, + argv[1], + fd, + std::placeholders::_1, + std::placeholders::_2); + connPtr->sendStream(callback); + char str[64]; + ++counter; + sprintf(str, "\n%d streams sent!\n", counter); + connPtr->send(str, strlen(str)); + } + }); + t.detach(); + + for (int i = 0; i < 3; ++i) + { + int fd; +#ifdef _WIN32 + _sopen_s(&fd, argv[1], _O_BINARY | _O_RDONLY, _SH_DENYNO, 0); +#else + fd = open(argv[1], O_RDONLY); +#endif + auto callback = std::bind(fileCallback, + argv[1], + fd, + std::placeholders::_1, + std::placeholders::_2); + connPtr->sendStream(callback); + char str[64]; + ++counter; + sprintf(str, "\n%d streams sent!\n", counter); + connPtr->send(str, strlen(str)); + } + } + else if (connPtr->disconnected()) + { + LOG_DEBUG << "connection disconnected"; + } + }); + server.setIoLoopNum(3); + server.start(); + loopThread.wait(); + return 0; +} + +std::size_t fileCallback(const std::string &strFile, + int nFd, + char *pBuffer, + std::size_t nBuffSize) +{ + if (nFd < 0) + return 0; + if (pBuffer == nullptr) + { + LOG_DEBUG << strFile.c_str() << " closed."; +#ifdef _WIN32 + _close(nFd); +#else + close(nFd); +#endif + return 0; + } +#ifdef _WIN32 + int nRead = _read(nFd, pBuffer, (unsigned int)nBuffSize); +#else + ssize_t nRead = read(nFd, pBuffer, nBuffSize); +#endif + if (nRead < 0) + return 0; + return std::size_t(nRead); +}; From 28694d8fe9b32bde355627d8f57d1f4dbeebc523 Mon Sep 17 00:00:00 2001 From: Nitromelon Date: Thu, 19 May 2022 09:46:05 +0800 Subject: [PATCH 068/182] Give EventLoopThread::loop_ static lifetime (#212) --- trantor/net/EventLoop.cc | 17 +++++++---------- trantor/net/EventLoopThread.cc | 12 ++++++------ 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/trantor/net/EventLoop.cc b/trantor/net/EventLoop.cc index e6b979a6..cd25a2ca 100644 --- a/trantor/net/EventLoop.cc +++ b/trantor/net/EventLoop.cc @@ -165,15 +165,6 @@ void EventLoop::quit() { quit_.store(true, std::memory_order_release); - Func f; - while (funcsOnQuit_.dequeue(f)) - { - f(); - } - - // There is a chance that loop() just executes while(!quit_) and exits, - // then EventLoop destructs, then we are accessing an invalid object. - // Can be fixed using mutex_ in both places. if (!isInLoopThread()) { wakeup(); @@ -205,12 +196,18 @@ void EventLoop::loop() currentActiveChannel_ = *it; currentActiveChannel_->handleEvent(); } - currentActiveChannel_ = NULL; + currentActiveChannel_ = nullptr; eventHandling_ = false; // std::cout << "looping" << endl; doRunInLoopFuncs(); } looping_.store(false, std::memory_order_release); + + Func f; + while (funcsOnQuit_.dequeue(f)) + { + f(); + } } void EventLoop::abortNotInLoopThread() { diff --git a/trantor/net/EventLoopThread.cc b/trantor/net/EventLoopThread.cc index 39ee02e4..4fa786dc 100644 --- a/trantor/net/EventLoopThread.cc +++ b/trantor/net/EventLoopThread.cc @@ -27,6 +27,7 @@ EventLoopThread::EventLoopThread(const std::string &threadName) auto f = promiseForLoopPointer_.get_future(); loop_ = f.get(); } + EventLoopThread::~EventLoopThread() { run(); @@ -39,28 +40,27 @@ EventLoopThread::~EventLoopThread() thread_.join(); } } -// void EventLoopThread::stop() { -// if(loop_) -// loop_->quit(); -//} + void EventLoopThread::wait() { thread_.join(); } + void EventLoopThread::loopFuncs() { #ifdef __linux__ ::prctl(PR_SET_NAME, loopThreadName_.c_str()); #endif - EventLoop loop; + thread_local static EventLoop loop; loop.queueInLoop([this]() { promiseForLoop_.set_value(1); }); promiseForLoopPointer_.set_value(&loop); auto f = promiseForRun_.get_future(); (void)f.get(); loop.loop(); // LOG_DEBUG << "loop out"; - loop_ = NULL; + loop_ = nullptr; } + void EventLoopThread::run() { std::call_once(once_, [this]() { From 46d9b443036cea5cb202881ea066e7d920bc7d95 Mon Sep 17 00:00:00 2001 From: Irineu Antunes Date: Wed, 8 Jun 2022 12:51:42 -0300 Subject: [PATCH 069/182] Added mTLS support (#213) --- CMakeLists.txt | 4 + trantor/net/TcpClient.cc | 6 +- trantor/net/TcpClient.h | 3 +- trantor/net/TcpConnection.h | 3 +- trantor/net/TcpServer.cc | 7 +- trantor/net/TcpServer.h | 3 +- trantor/net/inner/TcpConnectionImpl.cc | 120 +++++++++++++++++++++---- trantor/net/inner/TcpConnectionImpl.h | 6 +- 8 files changed, 127 insertions(+), 25 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e553e8a..148b9d13 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -128,6 +128,10 @@ endif(WIN32) find_package(OpenSSL) if(OpenSSL_FOUND) target_link_libraries(${PROJECT_NAME} PRIVATE OpenSSL::SSL OpenSSL::Crypto) + + # To enable the acceptance of local issued certificates + # Add the flag: ALLOW_SELF_SIGNED_CERTS + # ONLY FOR TESTING PURPOSES target_compile_definitions(${PROJECT_NAME} PRIVATE USE_OPENSSL) endif() diff --git a/trantor/net/TcpClient.cc b/trantor/net/TcpClient.cc index 2058dfc8..c440816c 100644 --- a/trantor/net/TcpClient.cc +++ b/trantor/net/TcpClient.cc @@ -226,12 +226,13 @@ void TcpClient::enableSSL( std::string hostname, const std::vector> &sslConfCmds, const std::string &certPath, - const std::string &keyPath) + const std::string &keyPath, + const std::string &caPath) { #ifdef USE_OPENSSL /* Create a new OpenSSL context */ sslCtxPtr_ = newSSLClientContext( - useOldTLS, validateCert, certPath, keyPath, sslConfCmds); + useOldTLS, validateCert, certPath, keyPath, sslConfCmds, caPath); validateCert_ = validateCert; if (!hostname.empty()) { @@ -251,6 +252,7 @@ void TcpClient::enableSSL( (void)sslConfCmds; (void)certPath; (void)keyPath; + (void)caPath; LOG_FATAL << "OpenSSL is not found in your system!"; throw std::runtime_error("OpenSSL is not found in your system!"); diff --git a/trantor/net/TcpClient.h b/trantor/net/TcpClient.h index 1ac915e3..9349063c 100644 --- a/trantor/net/TcpClient.h +++ b/trantor/net/TcpClient.h @@ -211,7 +211,8 @@ class TRANTOR_EXPORT TcpClient : NonCopyable, const std::vector> &sslConfCmds = {}, const std::string &certPath = "", - const std::string &keyPath = ""); + const std::string &keyPath = "", + const std::string &caPath = ""); private: /// Not thread safe, but in loop diff --git a/trantor/net/TcpConnection.h b/trantor/net/TcpConnection.h index 49e781c6..479874c5 100644 --- a/trantor/net/TcpConnection.h +++ b/trantor/net/TcpConnection.h @@ -30,7 +30,8 @@ TRANTOR_EXPORT std::shared_ptr newSSLServerContext( const std::string &certPath, const std::string &keyPath, bool useOldTLS = false, - const std::vector> &sslConfCmds = {}); + const std::vector> &sslConfCmds = {}, + const std::string &caPath = ""); /** * @brief This class represents a TCP connection. * diff --git a/trantor/net/TcpServer.cc b/trantor/net/TcpServer.cc index 7fc7180b..1d675e33 100644 --- a/trantor/net/TcpServer.cc +++ b/trantor/net/TcpServer.cc @@ -206,11 +206,13 @@ void TcpServer::enableSSL( const std::string &certPath, const std::string &keyPath, bool useOldTLS, - const std::vector> &sslConfCmds) + const std::vector> &sslConfCmds, + const std::string &caPath) { #ifdef USE_OPENSSL /* Create a new OpenSSL context */ - sslCtxPtr_ = newSSLServerContext(certPath, keyPath, useOldTLS, sslConfCmds); + sslCtxPtr_ = + newSSLServerContext(certPath, keyPath, useOldTLS, sslConfCmds, caPath); #else // When not using OpenSSL, using `void` here will // work around the unused parameter warnings without overhead. @@ -218,6 +220,7 @@ void TcpServer::enableSSL( (void)keyPath; (void)useOldTLS; (void)sslConfCmds; + (void)caPath; LOG_FATAL << "OpenSSL is not found in your system!"; throw std::runtime_error("OpenSSL is not found in your system!"); diff --git a/trantor/net/TcpServer.h b/trantor/net/TcpServer.h index f141cfb5..d5b77062 100644 --- a/trantor/net/TcpServer.h +++ b/trantor/net/TcpServer.h @@ -211,7 +211,8 @@ class TRANTOR_EXPORT TcpServer : NonCopyable const std::string &keyPath, bool useOldTLS = false, const std::vector> - &sslConfCmds = {}); + &sslConfCmds = {}, + const std::string &caPath = ""); private: EventLoop *loop_; diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index cd331e42..42f8ae38 100644 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -290,6 +290,7 @@ class SSLContext { return ctxPtr_; } + bool mtlsEnabled = false; private: SSL_CTX *ctxPtr_; @@ -297,9 +298,10 @@ class SSLContext class SSLConn { public: - explicit SSLConn(SSL_CTX *ctx) + explicit SSLConn(SSL_CTX *ctx, bool mtlsEnabled_) { SSL_ = SSL_new(ctx); + mtlsEnabled = mtlsEnabled_; } ~SSLConn() { @@ -312,6 +314,7 @@ class SSLConn { return SSL_; } + bool mtlsEnabled = false; private: SSL *SSL_; @@ -329,7 +332,8 @@ std::shared_ptr newSSLServerContext( const std::string &certPath, const std::string &keyPath, bool useOldTLS, - const std::vector> &sslConfCmds) + const std::vector> &sslConfCmds, + const std::string &caPath) { auto ctx = newSSLContext(useOldTLS, false, sslConfCmds); auto r = SSL_CTX_use_certificate_chain_file(ctx->get(), certPath.c_str()); @@ -356,6 +360,29 @@ std::shared_ptr newSSLServerContext( LOG_FATAL << "Checking private key matches certificate: " << errbuf; throw std::runtime_error("SSL_CTX_check_private_key error"); } + + if (!caPath.empty()) + { + auto checkCA = + SSL_CTX_load_verify_locations(ctx->get(), caPath.c_str(), NULL); + LOG_DEBUG << "CA CHECK LOC: " << checkCA; + if (checkCA) + { + STACK_OF(X509_NAME) *cert_names = + SSL_load_client_CA_file(caPath.c_str()); + if (cert_names != NULL) + { + SSL_CTX_set_client_CA_list(ctx->get(), cert_names); + } + ctx->mtlsEnabled = true; + } + else + { + LOG_FATAL << "caPath location error "; + throw std::runtime_error("SSL_CTX_load_verify_locations error"); + } + } + return ctx; } std::shared_ptr newSSLClientContext( @@ -363,7 +390,8 @@ std::shared_ptr newSSLClientContext( bool validateCert, const std::string &certPath, const std::string &keyPath, - const std::vector> &sslConfCmds) + const std::vector> &sslConfCmds, + const std::string &caPath) { auto ctx = newSSLContext(useOldTLS, validateCert, sslConfCmds); if (certPath.empty() || keyPath.empty()) @@ -393,6 +421,29 @@ std::shared_ptr newSSLClientContext( LOG_FATAL << "Checking private key matches certificate: " << errbuf; throw std::runtime_error("SSL_CTX_check_private_key error."); } + + if (!caPath.empty()) + { + auto checkCA = + SSL_CTX_load_verify_locations(ctx->get(), caPath.c_str(), NULL); + LOG_DEBUG << "CA CHECK LOC: " << checkCA; + if (checkCA) + { + STACK_OF(X509_NAME) *cert_names = + SSL_load_client_CA_file(caPath.c_str()); + if (cert_names != NULL) + { + SSL_CTX_set_client_CA_list(ctx->get(), cert_names); + } + ctx->mtlsEnabled = true; + } + else + { + LOG_FATAL << "caPath location error "; + throw std::runtime_error("SSL_CTX_load_verify_locations error"); + } + } + return ctx; } } // namespace trantor @@ -403,7 +454,8 @@ std::shared_ptr newSSLServerContext( const std::string &, const std::string &, bool, - const std::vector> &) + const std::vector> &, + const std::string &) { LOG_FATAL << "OpenSSL is not found in your system!"; throw std::runtime_error("OpenSSL is not found in your system!"); @@ -457,11 +509,15 @@ void TcpConnectionImpl::startClientEncryptionInLoop( sslEncryptionPtr_->sslCtxPtr_ = newSSLContext(useOldTLS, validateCert_, sslConfCmds); sslEncryptionPtr_->sslPtr_ = - std::make_unique(sslEncryptionPtr_->sslCtxPtr_->get()); - if (validateCert) + std::make_unique(sslEncryptionPtr_->sslCtxPtr_->get(), + sslEncryptionPtr_->sslCtxPtr_->mtlsEnabled); + if (validateCert || sslEncryptionPtr_->sslPtr_->mtlsEnabled) { + LOG_DEBUG << "MTLS: " << sslEncryptionPtr_->sslPtr_->mtlsEnabled; SSL_set_verify(sslEncryptionPtr_->sslPtr_->get(), - SSL_VERIFY_NONE, + sslEncryptionPtr_->sslPtr_->mtlsEnabled + ? SSL_VERIFY_PEER + : SSL_VERIFY_NONE, nullptr); validateCert_ = validateCert; } @@ -497,13 +553,21 @@ void TcpConnectionImpl::startServerEncryptionInLoop( sslEncryptionPtr_->sslCtxPtr_ = ctx; sslEncryptionPtr_->isServer_ = true; sslEncryptionPtr_->sslPtr_ = - std::make_unique(sslEncryptionPtr_->sslCtxPtr_->get()); + std::make_unique(sslEncryptionPtr_->sslCtxPtr_->get(), + sslEncryptionPtr_->sslCtxPtr_->mtlsEnabled); isEncrypted_ = true; sslEncryptionPtr_->isUpgrade_ = true; - if (sslEncryptionPtr_->isServer_ == false) + if (sslEncryptionPtr_->isServer_ == false || + sslEncryptionPtr_->sslPtr_->mtlsEnabled) + { + LOG_DEBUG << "MTLS: " << sslEncryptionPtr_->sslPtr_->mtlsEnabled; SSL_set_verify(sslEncryptionPtr_->sslPtr_->get(), - SSL_VERIFY_NONE, + sslEncryptionPtr_->sslPtr_->mtlsEnabled + ? SSL_VERIFY_PEER + : SSL_VERIFY_NONE, nullptr); + } + auto r = SSL_set_fd(sslEncryptionPtr_->sslPtr_->get(), socketPtr_->fd()); (void)r; assert(r); @@ -1895,13 +1959,20 @@ TcpConnectionImpl::TcpConnectionImpl(EventLoop *loop, socketPtr_->setKeepAlive(true); name_ = localAddr.toIpPort() + "--" + peerAddr.toIpPort(); sslEncryptionPtr_ = std::make_unique(); - sslEncryptionPtr_->sslPtr_ = std::make_unique(ctxPtr->get()); + sslEncryptionPtr_->sslPtr_ = + std::make_unique(ctxPtr->get(), ctxPtr->mtlsEnabled); sslEncryptionPtr_->isServer_ = isServer; validateCert_ = validateCert; - if (isServer == false) + if (isServer == false || sslEncryptionPtr_->sslPtr_->mtlsEnabled) + { + LOG_DEBUG << "MTLS: " << sslEncryptionPtr_->sslPtr_->mtlsEnabled; SSL_set_verify(sslEncryptionPtr_->sslPtr_->get(), - SSL_VERIFY_NONE, + sslEncryptionPtr_->sslPtr_->mtlsEnabled + ? SSL_VERIFY_PEER + : SSL_VERIFY_NONE, nullptr); + } + if (!isServer && !hostname.empty()) { SSL_set_tlsext_host_name(sslEncryptionPtr_->sslPtr_->get(), @@ -1925,12 +1996,25 @@ bool TcpConnectionImpl::validatePeerCertificate() SSL *ssl = sslEncryptionPtr_->sslPtr_->get(); auto result = SSL_get_verify_result(ssl); - if (result != X509_V_OK) + +#ifdef ALLOW_SELF_SIGNED_CERTS + if (result != X509_V_OK && + result != X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT && + result != X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN && + result != X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY) + { + LOG_DEBUG << "cert error code: " << result; + LOG_ERROR << "Server certificate is not valid"; + return false; + } +#else + if (result != X509_V_OK && result) { LOG_DEBUG << "cert error code: " << result; LOG_ERROR << "Server certificate is not valid"; return false; } +#endif X509 *cert = SSL_get_peer_certificate(ssl); if (cert == nullptr) @@ -1944,7 +2028,10 @@ bool TcpConnectionImpl::validatePeerCertificate() internal::verifyAltName(cert, sslEncryptionPtr_->hostname_); X509_free(cert); - if (domainIsValid) + LOG_DEBUG << "domainIsValid: " << domainIsValid; + + // if mtlsEnabled, ignore domain validation + if (sslEncryptionPtr_->sslPtr_->mtlsEnabled || domainIsValid) { return true; } @@ -1965,7 +2052,8 @@ void TcpConnectionImpl::doHandshaking() { // Clients don't commonly have certificates. Let's not validate // that - if (validateCert_ && sslEncryptionPtr_->isServer_ == false) + if (validateCert_ && (!sslEncryptionPtr_->isServer_ || + sslEncryptionPtr_->sslPtr_->mtlsEnabled)) { if (validatePeerCertificate() == false) { diff --git a/trantor/net/inner/TcpConnectionImpl.h b/trantor/net/inner/TcpConnectionImpl.h index d872a09b..76d64567 100644 --- a/trantor/net/inner/TcpConnectionImpl.h +++ b/trantor/net/inner/TcpConnectionImpl.h @@ -46,13 +46,15 @@ std::shared_ptr newSSLServerContext( const std::string &certPath, const std::string &keyPath, bool useOldTLS, - const std::vector> &sslConfCmds); + const std::vector> &sslConfCmds, + const std::string &caPath); std::shared_ptr newSSLClientContext( bool useOldTLS, bool validateCert, const std::string &certPath = "", const std::string &keyPath = "", - const std::vector> &sslConfCmds = {}); + const std::vector> &sslConfCmds = {}, + const std::string &caPath = ""); // void initServerSSLContext(const std::shared_ptr &ctx, // const std::string &certPath, From bbcc0acd312c789d98df13dd0a065ae6dc3806f7 Mon Sep 17 00:00:00 2001 From: ChunPing Chung Date: Sun, 19 Jun 2022 23:18:49 +0800 Subject: [PATCH 070/182] Optimization SSL name matching (#215) Co-authored-by: marty1885 --- .gitignore | 1 + trantor/net/inner/TcpConnectionImpl.cc | 53 +--------- trantor/unittests/CMakeLists.txt | 4 +- trantor/unittests/sslNameVerifyUnittest.cc | 50 +++++++++ trantor/utils/Utilities.cc | 113 +++++++++++++++++++++ trantor/utils/Utilities.h | 9 ++ 6 files changed, 181 insertions(+), 49 deletions(-) create mode 100644 trantor/unittests/sslNameVerifyUnittest.cc mode change 100755 => 100644 trantor/utils/Utilities.cc diff --git a/.gitignore b/.gitignore index f3f2f6b4..a237028b 100755 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ cmake-build-debug .vscode .vs CMakeSettings.json +.cache diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index 42f8ae38..4064f9ad 100644 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -34,8 +34,6 @@ #include #include #endif -#include - using namespace trantor; #ifdef _WIN32 @@ -88,49 +86,6 @@ inline bool loadWindowsSystemCert(X509_STORE *store) } #endif -inline std::string certNameToRegex(const std::string &certName) -{ - std::string result; - result.reserve(certName.size() + 11); - - bool isStar = false; - bool isLeadingStar = true; - for (char ch : certName) - { - if (isStar == false) - { - if (ch == '*') - isStar = true; - else if (ch == '.') - { - result += "\\."; - isLeadingStar = false; - } - else - { - result.push_back(ch); - isLeadingStar = false; - } - } - else - { - if (ch == '.' && isLeadingStar) - result += "([^.]*\\.|)?"; - else - result += std::string("[^.]*") + ch; - isStar = false; - } - } - assert(isStar == false); - return result; -} - -inline bool verifyName(const std::string &certName, const std::string &hostname) -{ - std::regex re(certNameToRegex(certName)); - return std::regex_match(hostname, re); -} - inline bool verifyCommonName(X509 *cert, const std::string &hostname) { X509_NAME *subjectName = X509_get_subject_name(cert); @@ -145,8 +100,9 @@ inline bool verifyCommonName(X509 *cert, const std::string &hostname) if (length == -1) return false; - return verifyName(std::string(name.begin(), name.begin() + length), - hostname); + return utils::verifySslName(std::string(name.begin(), + name.begin() + length), + hostname); } return false; @@ -177,7 +133,8 @@ inline bool verifyAltName(X509 *cert, const std::string &hostname) auto name = (const char *)ASN1_STRING_data(val->d.ia5); #endif auto name_len = (size_t)ASN1_STRING_length(val->d.ia5); - good = verifyName(std::string(name, name + name_len), hostname); + good = utils::verifySslName(std::string(name, name + name_len), + hostname); } } diff --git a/trantor/unittests/CMakeLists.txt b/trantor/unittests/CMakeLists.txt index ef643679..4814b566 100644 --- a/trantor/unittests/CMakeLists.txt +++ b/trantor/unittests/CMakeLists.txt @@ -4,12 +4,14 @@ add_executable(inetaddress_unittest InetAddressUnittest.cc) add_executable(date_unittest DateUnittest.cc) add_executable(split_string_unittest splitStringUnittest.cc) add_executable(string_encoding_unittest stringEncodingUnittest.cc) +add_executable(ssl_name_verify_unittest sslNameVerifyUnittest.cc) set(UNITTEST_TARGETS msgbuffer_unittest inetaddress_unittest date_unittest split_string_unittest - string_encoding_unittest) + string_encoding_unittest + ssl_name_verify_unittest) set_property(TARGET ${UNITTEST_TARGETS} PROPERTY CXX_STANDARD 14) set_property(TARGET ${UNITTEST_TARGETS} PROPERTY CXX_STANDARD_REQUIRED ON) set_property(TARGET ${UNITTEST_TARGETS} PROPERTY CXX_EXTENSIONS OFF) diff --git a/trantor/unittests/sslNameVerifyUnittest.cc b/trantor/unittests/sslNameVerifyUnittest.cc new file mode 100644 index 00000000..c5f0e1f3 --- /dev/null +++ b/trantor/unittests/sslNameVerifyUnittest.cc @@ -0,0 +1,50 @@ +#include +#include +#include +using namespace trantor; +using namespace trantor::utils; + +TEST(sslNameCheck, baseCases) +{ + EXPECT_EQ(verifySslName("example.com", "example.com"), true); + EXPECT_EQ(verifySslName("example.com", "example.org"), false); + EXPECT_EQ(verifySslName("example.com", "www.example.com"), false); +} + +TEST(sslNameCheck, rfc6125Examples) +{ + EXPECT_EQ(verifySslName("*.example.com", "foo.example.com"), true); + EXPECT_EQ(verifySslName("*.example.com", "foo.bar.example.com"), false); + EXPECT_EQ(verifySslName("*.example.com", "example.com"), false); + EXPECT_EQ(verifySslName("*bar.example.com", "foobar.example.com"), true); + EXPECT_EQ(verifySslName("baz*.example.com", "baz1.example.com"), true); + EXPECT_EQ(verifySslName("b*z.example.com", "buzz.example.com"), true); +} + +TEST(sslNameCheck, rfcCounterExamples) +{ + EXPECT_EQ(verifySslName("buz*.example.com", "buaz.example.com"), false); + EXPECT_EQ(verifySslName("*bar.example.com", "aaasdasbaz.example.com"), + false); + EXPECT_EQ(verifySslName("b*z.example.com", "baaaaaa.example.com"), false); +} + +TEST(sslNameCheck, wildExamples) +{ + EXPECT_EQ(verifySslName("datatracker.ietf.org", "datatracker.ietf.org"), + true); + EXPECT_EQ(verifySslName("*.nsysu.edu.tw", "nsysu.edu.tw"), false); + EXPECT_EQ(verifySslName("nsysu.edu.tw", "nsysu.edu.tw"), true); +} + +TEST(sslNameCheck, edgeCase) +{ + EXPECT_EQ(verifySslName(".example.com", "example.com"), false); + EXPECT_EQ(verifySslName("example.com.", "example.com."), true); +} + +int main(int argc, char **argv) +{ + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/trantor/utils/Utilities.cc b/trantor/utils/Utilities.cc old mode 100755 new mode 100644 index 4d887dd6..978dac8c --- a/trantor/utils/Utilities.cc +++ b/trantor/utils/Utilities.cc @@ -132,5 +132,118 @@ std::string fromWidePath(const std::wstring &wstrPath) return toUtf8(srcPath); } +bool verifySslName(const std::string &certName, const std::string &hostname) +{ + if (certName.find('*') == std::string::npos) + { + return certName == hostname; + } + + size_t firstDot = certName.find('.'); + size_t hostFirstDot = hostname.find('.'); + size_t pos, len, hostPos, hostLen; + + if (firstDot != std::string::npos) + { + pos = firstDot + 1; + } + else + { + firstDot = pos = certName.size(); + } + + len = certName.size() - pos; + + if (hostFirstDot != std::string::npos) + { + hostPos = hostFirstDot + 1; + } + else + { + hostFirstDot = hostPos = hostname.size(); + } + + hostLen = hostname.size() - hostPos; + + // *. in the beginning of the cert name + if (certName.compare(0, firstDot, "*") == 0) + { + return certName.compare(pos, len, hostname, hostPos, hostLen) == 0; + } + // * in the left most. but other chars in the right + else if (certName[0] == '*') + { + // compare if `hostname` ends with `certName` but without the leftmost + // should be fine as domain names can't be that long + intmax_t hostnameIdx = hostname.size() - 1; + intmax_t certNameIdx = certName.size() - 1; + while (hostnameIdx >= 0 && certNameIdx != 0) + { + if (hostname[hostnameIdx] != certName[certNameIdx]) + { + return false; + } + hostnameIdx--; + certNameIdx--; + } + if (certNameIdx != 0) + { + return false; + } + return true; + } + // * in the right of the first dot + else if (firstDot != 0 && certName[firstDot - 1] == '*') + { + if (certName.compare(pos, len, hostname, hostPos, hostLen) != 0) + { + return false; + } + for (size_t i = 0; + i < hostFirstDot && i < firstDot && certName[i] != '*'; + i++) + { + if (hostname[i] != certName[i]) + { + return false; + } + } + return true; + } + // else there's a * in the middle + else + { + if (certName.compare(pos, len, hostname, hostPos, hostLen) != 0) + { + return false; + } + for (size_t i = 0; + i < hostFirstDot && i < firstDot && certName[i] != '*'; + i++) + { + if (hostname[i] != certName[i]) + { + return false; + } + } + intmax_t hostnameIdx = hostFirstDot - 1; + intmax_t certNameIdx = firstDot - 1; + while (hostnameIdx >= 0 && certNameIdx >= 0 && + certName[certNameIdx] != '*') + { + if (hostname[hostnameIdx] != certName[certNameIdx]) + { + return false; + } + hostnameIdx--; + certNameIdx--; + } + return true; + } + + // should not reach + return certName == hostname; +} + } // namespace utils } // namespace trantor diff --git a/trantor/utils/Utilities.h b/trantor/utils/Utilities.h index 0182b1ef..3e477c06 100644 --- a/trantor/utils/Utilities.h +++ b/trantor/utils/Utilities.h @@ -171,6 +171,15 @@ inline std::string fromNativePath(const std::wstring &strPath) return fromWidePath(strPath); } +/** + * @brief Check if the name supplied by the SSL Cert matchs a FQDN + * @param certName The name supplied by the SSL Cert + * @param hostName The FQDN to match + * + * @return true if matches. false otherwise + */ +bool verifySslName(const std::string &certName, const std::string &hostname); + } // namespace utils } // namespace trantor From fb817286eb82ebce95acfad8903e5a4cd0b93fdb Mon Sep 17 00:00:00 2001 From: kn71026 Date: Mon, 20 Jun 2022 11:23:08 +0800 Subject: [PATCH 071/182] Clarify SSL error message (#214) Co-authored-by: Martin Chang --- trantor/net/inner/TcpConnectionImpl.cc | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index 4064f9ad..f342d5d2 100644 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -298,7 +298,8 @@ std::shared_ptr newSSLServerContext( if (!r) { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); - LOG_FATAL << "Reading certificate: " << errbuf; + LOG_FATAL << "Reading certificate: " << certPath + << " failed. Error: " << errbuf; throw std::runtime_error("SSL_CTX_use_certificate_chain_file error."); } r = SSL_CTX_use_PrivateKey_file(ctx->get(), @@ -307,14 +308,16 @@ std::shared_ptr newSSLServerContext( if (!r) { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); - LOG_FATAL << "Reading private key: " << errbuf; + LOG_FATAL << "Reading private key: " << keyPath + << " failed. Error: " << errbuf; throw std::runtime_error("SSL_CTX_use_PrivateKey_file error"); } r = SSL_CTX_check_private_key(ctx->get()); if (!r) { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); - LOG_FATAL << "Checking private key matches certificate: " << errbuf; + LOG_FATAL << "Checking private key matches certificate: " << certPath + << " and " << keyPath << " mismatches. Error: " << errbuf; throw std::runtime_error("SSL_CTX_check_private_key error"); } @@ -359,7 +362,8 @@ std::shared_ptr newSSLClientContext( if (!r) { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); - LOG_FATAL << "Reading certificate: " << errbuf; + LOG_FATAL << "Reading certificate: " << certPath + << " failed. Error: " << errbuf; throw std::runtime_error("SSL_CTX_use_certificate_chain_file error."); } r = SSL_CTX_use_PrivateKey_file(ctx->get(), @@ -368,15 +372,17 @@ std::shared_ptr newSSLClientContext( if (!r) { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); - LOG_FATAL << "Reading private key: " << errbuf; - throw std::runtime_error("SSL_CTX_use_PrivateKey_file error."); + LOG_FATAL << "Reading private key: " << keyPath + << " failed. Error: " << errbuf; + throw std::runtime_error("SSL_CTX_use_PrivateKey_file error"); } r = SSL_CTX_check_private_key(ctx->get()); if (!r) { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); - LOG_FATAL << "Checking private key matches certificate: " << errbuf; - throw std::runtime_error("SSL_CTX_check_private_key error."); + LOG_FATAL << "Checking private key matches certificate: " << certPath + << " and " << keyPath << " mismatches. Error: " << errbuf; + throw std::runtime_error("SSL_CTX_check_private_key error"); } if (!caPath.empty()) From 295f31367f704a5abca9f949894e39178f9d18c1 Mon Sep 17 00:00:00 2001 From: LE GARREC Vincent Date: Wed, 29 Jun 2022 03:44:41 +0200 Subject: [PATCH 072/182] Rename BUILD_TRANTOR_SHARED to BUILD_SHARED_LIBS (#216) --- .github/workflows/cmake.yml | 2 +- CMakeLists.txt | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index fa2f9920..cf937dee 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -43,7 +43,7 @@ jobs: working-directory: ./build run: | [[ ${{ matrix.link }} == "SHARED" ]] && shared="ON" || shared="OFF" - cmake .. -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=on -DBUILD_TRANTOR_SHARED=$shared -DCMAKE_TOOLCHAIN_FILE="conan_paths.cmake" -DCMAKE_INSTALL_PREFIX=../install + cmake .. -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=on -DBUILD_SHARED_LIBS=$shared -DCMAKE_TOOLCHAIN_FILE="conan_paths.cmake" -DCMAKE_INSTALL_PREFIX=../install - name: Build working-directory: ${{env.GITHUB_WORKSPACE}} diff --git a/CMakeLists.txt b/CMakeLists.txt index 148b9d13..16c8301b 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,7 @@ project(trantor) option(BUILD_DOC "Build Doxygen documentation" OFF) option(BUILD_C-ARES "Build C-ARES" ON) +option(BUILD_SHARED_LIBS "Build trantor as a shared lib" OFF) list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) @@ -24,14 +25,13 @@ set(INSTALL_TRANTOR_CMAKE_DIR ${DEF_INSTALL_TRANTOR_CMAKE_DIR} CACHE PATH "Installation directory for cmake files") -if(BUILD_TRANTOR_SHARED) - set(CMAKE_POSITION_INDEPENDENT_CODE TRUE) +add_library(${PROJECT_NAME}) +if(BUILD_SHARED_LIBS) list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIB_DIR}" isSystemDir) if("${isSystemDir}" STREQUAL "-1") set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIB_DIR}") endif("${isSystemDir}" STREQUAL "-1") - add_library(${PROJECT_NAME} SHARED) set_target_properties(${PROJECT_NAME} PROPERTIES VERSION ${TRANTOR_VERSION} SOVERSION ${TRANTOR_MAJOR_VERSION}) @@ -42,9 +42,7 @@ if(BUILD_TRANTOR_SHARED) # exact same compiler for drogon and your app. target_compile_options(${PROJECT_NAME} PUBLIC /wd4251 /wd4275) endif() -else(BUILD_TRANTOR_SHARED) - add_library(${PROJECT_NAME} STATIC) -endif(BUILD_TRANTOR_SHARED) +endif(BUILD_SHARED_LIBS) if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Windows" AND CMAKE_CXX_COMPILER_ID MATCHES Clang|GNU) target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wextra -Werror) From e117197512b11ecbb331259223da1efe8f3f9f08 Mon Sep 17 00:00:00 2001 From: LE GARREC Vincent Date: Wed, 6 Jul 2022 04:05:17 +0200 Subject: [PATCH 073/182] Fix thread sanitizer (#217) --- trantor/net/EventLoopThread.cc | 23 ++++++++++++++++------- trantor/net/EventLoopThread.h | 9 ++++++--- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/trantor/net/EventLoopThread.cc b/trantor/net/EventLoopThread.cc index 4fa786dc..2764664c 100644 --- a/trantor/net/EventLoopThread.cc +++ b/trantor/net/EventLoopThread.cc @@ -31,9 +31,14 @@ EventLoopThread::EventLoopThread(const std::string &threadName) EventLoopThread::~EventLoopThread() { run(); - if (loop_) + std::shared_ptr loop; { - loop_->quit(); + std::unique_lock lk(loopMutex_); + loop = loop_; + } + if (loop) + { + loop->quit(); } if (thread_.joinable()) { @@ -51,14 +56,18 @@ void EventLoopThread::loopFuncs() #ifdef __linux__ ::prctl(PR_SET_NAME, loopThreadName_.c_str()); #endif - thread_local static EventLoop loop; - loop.queueInLoop([this]() { promiseForLoop_.set_value(1); }); - promiseForLoopPointer_.set_value(&loop); + thread_local static std::shared_ptr loop = + std::make_shared(); + loop->queueInLoop([this]() { promiseForLoop_.set_value(1); }); + promiseForLoopPointer_.set_value(loop); auto f = promiseForRun_.get_future(); (void)f.get(); - loop.loop(); + loop->loop(); // LOG_DEBUG << "loop out"; - loop_ = nullptr; + { + std::unique_lock lk(loopMutex_); + loop_ = nullptr; + } } void EventLoopThread::run() diff --git a/trantor/net/EventLoopThread.h b/trantor/net/EventLoopThread.h index 4d9caa18..7a36383b 100644 --- a/trantor/net/EventLoopThread.h +++ b/trantor/net/EventLoopThread.h @@ -48,7 +48,7 @@ class TRANTOR_EXPORT EventLoopThread : NonCopyable */ EventLoop *getLoop() const { - return loop_; + return loop_.get(); } /** @@ -59,10 +59,13 @@ class TRANTOR_EXPORT EventLoopThread : NonCopyable void run(); private: - EventLoop *loop_; + // With C++20, use std::atomic> + std::shared_ptr loop_; + std::mutex loopMutex_; + std::string loopThreadName_; void loopFuncs(); - std::promise promiseForLoopPointer_; + std::promise> promiseForLoopPointer_; std::promise promiseForRun_; std::promise promiseForLoop_; std::once_flag once_; From 582a7191f2412b870d422d4b23b7b602bf027078 Mon Sep 17 00:00:00 2001 From: an-tao Date: Sat, 9 Jul 2022 23:45:08 +0800 Subject: [PATCH 074/182] Bump version to 1.5.6 --- CMakeLists.txt | 2 +- ChangeLog.md | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 16c8301b..bdb08885 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) set(TRANTOR_MAJOR_VERSION 1) set(TRANTOR_MINOR_VERSION 5) -set(TRANTOR_PATCH_VERSION 5) +set(TRANTOR_PATCH_VERSION 6) set(TRANTOR_VERSION ${TRANTOR_MAJOR_VERSION}.${TRANTOR_MINOR_VERSION}.${TRANTOR_PATCH_VERSION}) diff --git a/ChangeLog.md b/ChangeLog.md index 3c2c05c4..c09de8a3 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -3,6 +3,38 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [1.5.6] - 2022-07-09 + +### API changes list + +- Add support for sending data streams via callback. + +- Added mTLS support. + +### Changed + +- Make MsgBuffer constructor explicit. + +- Always queue connectDestroyed() in loop. + +- Stop calling abort() in runtime. + +- Give EventLoopThread::loop_ static lifetime. + +- Optimization SSL name matching. + +- Clarify SSL error message. + +- Rename BUILD_TRANTOR_SHARED to BUILD_SHARED_LIBS. + +### Fixed + +- Fix tolower with sanitizer cfi + +- include unconditionally on freebsd + +- Fix thread sanitizer. + ## [1.5.5] - 2022-02-19 ### API changes list @@ -412,7 +444,9 @@ All notable changes to this project will be documented in this file. ## [1.0.0-rc1] - 2019-06-11 -[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.5...HEAD +[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.6...HEAD + +[1.5.6]: https://github.com/an-tao/trantor/compare/v1.5.5...v1.5.6 [1.5.5]: https://github.com/an-tao/trantor/compare/v1.5.4...v1.5.5 From 5345dad5a53c5264dc060481cb0a12ef1db08949 Mon Sep 17 00:00:00 2001 From: Nitromelon Date: Fri, 22 Jul 2022 10:35:30 +0800 Subject: [PATCH 075/182] Add utc methods for trantor::Date. (#218) --- trantor/utils/Date.cc | 11 +++++++++++ trantor/utils/Date.h | 17 +++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/trantor/utils/Date.cc b/trantor/utils/Date.cc index 7525bf2a..918e8ab0 100644 --- a/trantor/utils/Date.cc +++ b/trantor/utils/Date.cc @@ -274,6 +274,11 @@ std::string Date::toDbStringLocal() const } return buf; } +std::string Date::toDbString() const +{ + return after(static_cast(-timezoneOffset())).toDbStringLocal(); +} + Date Date::fromDbStringLocal(const std::string &datetime) { unsigned int year = {0}, month = {0}, day = {0}, hour = {0}, minute = {0}, @@ -312,6 +317,12 @@ Date Date::fromDbStringLocal(const std::string &datetime) } return trantor::Date(year, month, day, hour, minute, second, microSecond); } +Date Date::fromDbString(const std::string &datetime) +{ + return fromDbStringLocal(datetime).after( + static_cast(timezoneOffset())); +} + std::string Date::toCustomedFormattedStringLocal(const std::string &fmtStr, bool showMicroseconds) const { diff --git a/trantor/utils/Date.h b/trantor/utils/Date.h index 8543a2de..01e2ab5e 100644 --- a/trantor/utils/Date.h +++ b/trantor/utils/Date.h @@ -74,6 +74,13 @@ class TRANTOR_EXPORT Date return Date::date(); } + static int64_t timezoneOffset() + { + static int64_t offset = + 0 - Date::fromDbStringLocal("1970-01-01").secondsSinceEpoch(); + return offset; + } + /** * @brief Return a new Date instance that represents the time after some * seconds from *this. @@ -233,6 +240,10 @@ class TRANTOR_EXPORT Date * is zero 2018-01-01 10:10:25:102414 //If the microsecond is not zero */ std::string toDbStringLocal() const; + /** + * @brief Generate a UTC time string for database. + */ + std::string toDbString() const; /** * @brief From DB string to trantor local time zone. @@ -240,6 +251,12 @@ class TRANTOR_EXPORT Date * Inverse of toDbStringLocal() */ static Date fromDbStringLocal(const std::string &datetime); + /** + * @brief From DB string to trantor UTC time. + * + * Inverse of toDbString() + */ + static Date fromDbString(const std::string &datetime); /** * @brief Generate a UTC time string. From 9230b61a13c567322c3835931ab4dcb2eae70126 Mon Sep 17 00:00:00 2001 From: Ken Matsui <26405363+ken-matsui@users.noreply.github.com> Date: Tue, 26 Jul 2022 13:21:06 +0900 Subject: [PATCH 076/182] Remove an unnecessary semi-colon (#219) --- trantor/tests/SendstreamTest.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trantor/tests/SendstreamTest.cc b/trantor/tests/SendstreamTest.cc index 2a0c65a5..144adc71 100644 --- a/trantor/tests/SendstreamTest.cc +++ b/trantor/tests/SendstreamTest.cc @@ -143,4 +143,4 @@ std::size_t fileCallback(const std::string &strFile, if (nRead < 0) return 0; return std::size_t(nRead); -}; +} From 7d9f232c412ad697a4ace35c8b5f7fe22f46124d Mon Sep 17 00:00:00 2001 From: An Tao Date: Mon, 1 Aug 2022 13:22:43 +0800 Subject: [PATCH 077/182] Fix a race condition (#220) --- trantor/net/TcpServer.cc | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/trantor/net/TcpServer.cc b/trantor/net/TcpServer.cc index 1d675e33..28aa7853 100644 --- a/trantor/net/TcpServer.cc +++ b/trantor/net/TcpServer.cc @@ -145,19 +145,36 @@ void TcpServer::start() } void TcpServer::stop() { - loop_->runInLoop([this]() { acceptorPtr_.reset(); }); - for (auto connection : connSet_) + if (loop_->isInLoopThread()) + { + acceptorPtr_.reset(); + for (auto connection : connSet_) + { + connection->forceClose(); + } + } + else { - connection->forceClose(); + std::promise pro; + auto f = pro.get_future(); + loop_->queueInLoop([this, &pro]() { + acceptorPtr_.reset(); + for (auto connection : connSet_) + { + connection->forceClose(); + } + pro.set_value(); + }); + f.get(); } loopPoolPtr_.reset(); for (auto &iter : timingWheelMap_) { - std::promise pro; + std::promise pro; auto f = pro.get_future(); iter.second->getLoop()->runInLoop([&iter, &pro]() mutable { iter.second.reset(); - pro.set_value(1); + pro.set_value(); }); f.get(); } From bf9eae841366758810c8ff121ede0eef592c6420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vedran=20Mileti=C4=87?= Date: Sat, 27 Aug 2022 15:28:29 +0000 Subject: [PATCH 078/182] Added support for Solaris (#94) Tested on illumos distribution OmniOS Community Edition. --- CMakeLists.txt | 2 ++ trantor/net/InetAddress.cc | 2 ++ trantor/utils/Date.cc | 3 ++- trantor/utils/Date.h | 2 +- trantor/utils/Logger.cc | 5 +++++ 5 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bdb08885..3ec011e2 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,6 +169,8 @@ if(WIN32) if(OpenSSL_FOUND) target_link_libraries(${PROJECT_NAME} PRIVATE Crypt32 Secur32) endif(OpenSSL_FOUND) +else(WIN32) + target_link_libraries(${PROJECT_NAME} PRIVATE pthread $<$:socket>) endif(WIN32) file(WRITE ${CMAKE_BINARY_DIR}/test_atomic.cpp diff --git a/trantor/net/InetAddress.cc b/trantor/net/InetAddress.cc index 0f05ccc1..dc1aa2fb 100644 --- a/trantor/net/InetAddress.cc +++ b/trantor/net/InetAddress.cc @@ -217,6 +217,8 @@ const uint32_t *InetAddress::ip6NetEndian() const // assert(family() == AF_INET6); #if defined __linux__ || defined __HAIKU__ return addr6_.sin6_addr.s6_addr32; +#elif defined __sun + return addr6_.sin6_addr._S6_un._S6_u32; #elif defined _WIN32 // TODO is this OK ? const struct in6_addr_uint *addr_temp = diff --git a/trantor/utils/Date.cc b/trantor/utils/Date.cc index 918e8ab0..dd9c284f 100644 --- a/trantor/utils/Date.cc +++ b/trantor/utils/Date.cc @@ -363,7 +363,8 @@ Date::Date(unsigned int year, tm.tm_min = minute; tm.tm_sec = second; epoch = mktime(&tm); - microSecondsSinceEpoch_ = epoch * MICRO_SECONDS_PRE_SEC + microSecond; + microSecondsSinceEpoch_ = + static_cast(epoch) * MICRO_SECONDS_PRE_SEC + microSecond; } } // namespace trantor diff --git a/trantor/utils/Date.h b/trantor/utils/Date.h index 01e2ab5e..a3677979 100644 --- a/trantor/utils/Date.h +++ b/trantor/utils/Date.h @@ -18,7 +18,7 @@ #include #include -#define MICRO_SECONDS_PRE_SEC 1000000 +#define MICRO_SECONDS_PRE_SEC 1000000LL namespace trantor { diff --git a/trantor/utils/Logger.cc b/trantor/utils/Logger.cc index d159dcc3..b86d8ccc 100644 --- a/trantor/utils/Logger.cc +++ b/trantor/utils/Logger.cc @@ -119,6 +119,11 @@ void Logger::formatTime() { threadId_ = getthrid(); } +#elif defined __sun + if (threadId_ == 0) + { + threadId_ = static_cast(pthread_self()); + } #elif defined _WIN32 || defined __HAIKU__ if (threadId_ == 0) { From fdffef9af0fbf27743f2c63e00c56d511a3a038c Mon Sep 17 00:00:00 2001 From: An Tao Date: Sat, 3 Sep 2022 21:02:48 +0800 Subject: [PATCH 079/182] Define ssize_t as std::intptr_t on Windows (#221) --- trantor/utils/MsgBuffer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trantor/utils/MsgBuffer.h b/trantor/utils/MsgBuffer.h index 29817f6a..19d42e27 100644 --- a/trantor/utils/MsgBuffer.h +++ b/trantor/utils/MsgBuffer.h @@ -22,7 +22,7 @@ #include #include #ifdef _WIN32 -using ssize_t = long long; +using ssize_t = std::intptr_t; #endif namespace trantor From 28a78a8c01a1dc274b2e2129be100a2ff9e93919 Mon Sep 17 00:00:00 2001 From: An Tao Date: Tue, 6 Sep 2022 11:55:13 +0800 Subject: [PATCH 080/182] Add an environment without openssl to github actions (#223) --- .github/workflows/cmake.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index cf937dee..f847d2ba 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -62,10 +62,10 @@ jobs: buildname: 'ubuntu-20.04/gcc' triplet: x64-linux compiler: gcc_64 - # - os: ubuntu-16.04 - # buildname: 'ubuntu-16.04/gcc' - # triplet: x64-linux - # compiler: gcc_64 + - os: ubuntu-20.04 + buildname: 'ubuntu-20.04/gcc without openssl' + triplet: x64-linux + compiler: gcc_64 - os: macos-latest buildname: 'macos/clang' triplet: x64-osx @@ -84,14 +84,20 @@ jobs: brew install c-ares openssl - name: (Linux) Install dependencies - if: runner.os == 'Linux' + if: matrix.buildname == 'ubuntu-20.04/gcc' run: | # Installing packages might fail as the github image becomes outdated sudo apt update # These aren't available or don't work well in vcpkg sudo apt install openssl libssl-dev sudo apt install dos2unix - + - name: (Linux) Install dependencies + if: matrix.buildname == 'ubuntu-20.04/gcc without openssl' + run: | + # Installing packages might fail as the github image becomes outdated + sudo apt update + # These aren't available or don't work well in vcpkg + sudo apt install dos2unix - name: install gtest run: | wget https://github.com/google/googletest/archive/release-1.10.0.tar.gz From 09f8a20769f87d18212e085f849ee28a5a730dac Mon Sep 17 00:00:00 2001 From: An Tao Date: Fri, 9 Sep 2022 00:56:48 +0800 Subject: [PATCH 081/182] Fix iterator invalidation bug when stopping TCP server (#225) --- trantor/net/TcpServer.cc | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/trantor/net/TcpServer.cc b/trantor/net/TcpServer.cc index 28aa7853..23b57d48 100644 --- a/trantor/net/TcpServer.cc +++ b/trantor/net/TcpServer.cc @@ -148,7 +148,15 @@ void TcpServer::stop() if (loop_->isInLoopThread()) { acceptorPtr_.reset(); - for (auto connection : connSet_) + // copy the connSet_ to a vector, use the vector to close the + // connections to avoid the iterator invalidation. + std::vector connPtrs; + connPtrs.reserve(connSet_.size()); + for (auto &conn : connSet_) + { + connPtrs.push_back(conn); + } + for (auto connection : connPtrs) { connection->forceClose(); } @@ -159,7 +167,13 @@ void TcpServer::stop() auto f = pro.get_future(); loop_->queueInLoop([this, &pro]() { acceptorPtr_.reset(); - for (auto connection : connSet_) + std::vector connPtrs; + connPtrs.reserve(connSet_.size()); + for (auto &conn : connSet_) + { + connPtrs.push_back(conn); + } + for (auto connection : connPtrs) { connection->forceClose(); } From 473059b7a2f7fda97c4f612b9b68abfe193d87d8 Mon Sep 17 00:00:00 2001 From: Irineu Antunes Date: Tue, 13 Sep 2022 22:38:50 -0300 Subject: [PATCH 082/182] Added SSL Error Trace Log and mTLS Samples (#222) --- trantor/net/inner/TcpConnectionImpl.cc | 29 ++++++- trantor/net/inner/TcpConnectionImpl.h | 1 + trantor/tests/MTLSClient.cc | 115 +++++++++++++++++++++++++ trantor/tests/MTLSServer.cc | 95 ++++++++++++++++++++ 4 files changed, 236 insertions(+), 4 deletions(-) create mode 100644 trantor/tests/MTLSClient.cc create mode 100644 trantor/tests/MTLSServer.cc diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index f342d5d2..d0e91ed8 100644 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -321,11 +321,19 @@ std::shared_ptr newSSLServerContext( throw std::runtime_error("SSL_CTX_check_private_key error"); } + if (!SSL_CTX_set_ecdh_auto(ctx->get(), 1)) + { + LOG_TRACE << "Failed to set_ecdh_auto, set_ecdh_auto DISABLED"; + } + else + { + LOG_TRACE << "set_ecdh_auto ENABLED"; + } + if (!caPath.empty()) { auto checkCA = SSL_CTX_load_verify_locations(ctx->get(), caPath.c_str(), NULL); - LOG_DEBUG << "CA CHECK LOC: " << checkCA; if (checkCA) { STACK_OF(X509_NAME) *cert_names = @@ -335,6 +343,7 @@ std::shared_ptr newSSLServerContext( SSL_CTX_set_client_CA_list(ctx->get(), cert_names); } ctx->mtlsEnabled = true; + LOG_TRACE << "mTLS session ENABLED"; } else { @@ -2005,6 +2014,17 @@ bool TcpConnectionImpl::validatePeerCertificate() } } +std::string TcpConnectionImpl::getOpenSSLErrorStack() +{ + BIO *bio = BIO_new(BIO_s_mem()); + ERR_print_errors(bio); + char *buf; + size_t len = BIO_get_mem_data(bio, &buf); + std::string ret(buf, len); + BIO_free(bio); + return ret; +} + void TcpConnectionImpl::doHandshaking() { assert(sslEncryptionPtr_->statusOfSSL_ == SSLStatus::Handshaking); @@ -2013,8 +2033,9 @@ void TcpConnectionImpl::doHandshaking() LOG_TRACE << "hand shaking: " << r; if (r == 1) { - // Clients don't commonly have certificates. Let's not validate - // that + // Clients don't commonly have certificates (except on mTLS). + // So if the SSL session is on server-side and without mTLS enabled, + // let's not validate the client certificate. if (validateCert_ && (!sslEncryptionPtr_->isServer_ || sslEncryptionPtr_->sslPtr_->mtlsEnabled)) { @@ -2059,8 +2080,8 @@ void TcpConnectionImpl::doHandshaking() } else { - // ERR_print_errors(err); LOG_TRACE << "SSL handshake err: " << err; + LOG_TRACE << "SSL error stack: " << getOpenSSLErrorStack(); ioChannelPtr_->disableReading(); sslEncryptionPtr_->statusOfSSL_ = SSLStatus::DisConnected; if (sslErrorCallback_) diff --git a/trantor/net/inner/TcpConnectionImpl.h b/trantor/net/inner/TcpConnectionImpl.h index 76d64567..72818e52 100644 --- a/trantor/net/inner/TcpConnectionImpl.h +++ b/trantor/net/inner/TcpConnectionImpl.h @@ -370,6 +370,7 @@ class TcpConnectionImpl : public TcpConnection, private: void doHandshaking(); bool validatePeerCertificate(); + std::string getOpenSSLErrorStack(); struct SSLEncryption { SSLStatus statusOfSSL_ = SSLStatus::Handshaking; diff --git a/trantor/tests/MTLSClient.cc b/trantor/tests/MTLSClient.cc new file mode 100644 index 00000000..488813fe --- /dev/null +++ b/trantor/tests/MTLSClient.cc @@ -0,0 +1,115 @@ +/** + * + * # Generate CA file + * openssl req -new -x509 -days 365 -keyout ca-key.pem -out ca-crt.pem + * + * # Generate Key File (ie: same for client and server, but you can create one + * for each one) openssl genrsa -out server-key.pem 4096 + * + * # Generate Server certificate: + * openssl req -new -sha256 -key server-key.pem -out ca-csr.pem + * openssl x509 -req -days 365 -in ca-csr.pem -CA ca-crt.pem -CAkey ca-key.pem + * -CAcreateserial -out server-crt.pem openssl verify -CAfile ca-crt.pem + * server-crt.pem + * + * + * # For client (to specify a certificate client mode only - no domain): + * # Create file client_cert_ext.cnf: + * cat client_cert_ext.cnf + * + * keyUsage = critical, digitalSignature, keyEncipherment + * extendedKeyUsage = clientAuth + * basicConstraints = critical, CA:FALSE + * authorityKeyIdentifier = keyid,issuer + * subjectAltName = DNS:Client + * + * Create client cert (using the same serve key and CA) + * openssl x509 -req -in ca-csr.pem -days 1000 -CA ca-crt.pem -CAkey ca-key.pem + * -set_serial 01 -extfile client_cert_ext.cnf > client-crt.pem + * + * openssl verify -CAfile ca-crt.pem client-crt.pem + * openssl x509 -in client-crt.pem -text -noout -purpose + * + * # Compile sample: + * + * g++ -o MTLSClient MTLSClient.cc -ltrantor -lssl -lcrypto -lpthread + * + * # Tests + * + * # Listen generic SSL server + * openssl s_server -accept 8888 -CAfile ./ca-crt.pem -cert ./server-crt.pem + * -key ./server-key.pem -state + * + * # Listen generic SSL server with mTLS verification + * openssl s_server -accept 8888 -CAfile ./ca-crt.pem -cert ./server-crt.pem + * -key ./server-key.pem -state -verify_return_error -Verify 1 + * + * # Test the mTLS client bin + * ./MTLSClient + * + * **/ + +#include +#include +#include +#include +#include +#include +using namespace trantor; +#define USE_IPV6 0 +int main() +{ + trantor::Logger::setLogLevel(trantor::Logger::kTrace); + LOG_DEBUG << "TcpClient class test!"; + EventLoop loop; +#if USE_IPV6 + InetAddress serverAddr("::1", 8888, true); +#else + InetAddress serverAddr("127.0.0.1", 8888); +#endif + std::shared_ptr client[10]; + std::atomic_int connCount; + connCount = 1; + for (int i = 0; i < connCount; ++i) + { + client[i] = std::make_shared(&loop, + serverAddr, + "tcpclienttest"); + std::vector> sslcmd = {}; + // That key is common for client and server + // The CA file must be the client CA, for this sample the CA is common + // for both + client[i]->enableSSL(false, + false, + "localhost", + sslcmd, + "./client-crt.pem", + "./server-key.pem", + "./ca-crt.pem"); + client[i]->setConnectionCallback( + [i, &loop, &connCount](const TcpConnectionPtr &conn) { + if (conn->connected()) + { + LOG_DEBUG << i << " connected!"; + char tmp[20]; + sprintf(tmp, "%d client!!", i); + conn->send(tmp); + } + else + { + LOG_DEBUG << i << " disconnected"; + --connCount; + if (connCount == 0) + loop.quit(); + } + }); + client[i]->setMessageCallback( + [](const TcpConnectionPtr &conn, MsgBuffer *buf) { + LOG_DEBUG << std::string(buf->peek(), buf->readableBytes()); + buf->retrieveAll(); + conn->shutdown(); + }); + client[i]->connect(); + } + loop.loop(); +} diff --git a/trantor/tests/MTLSServer.cc b/trantor/tests/MTLSServer.cc new file mode 100644 index 00000000..54a853e2 --- /dev/null +++ b/trantor/tests/MTLSServer.cc @@ -0,0 +1,95 @@ +/** + * + * # Generate CA file + * openssl req -new -x509 -days 365 -keyout ca-key.pem -out ca-crt.pem + * + * # Generate Key File (ie: same for client and server, but you can create one + * for each one) openssl genrsa -out server-key.pem 4096 + * + * # Generate Server certificate: + * openssl req -new -sha256 -key server-key.pem -out ca-csr.pem + * openssl x509 -req -days 365 -in ca-csr.pem -CA ca-crt.pem -CAkey ca-key.pem + * -CAcreateserial -out server-crt.pem openssl verify -CAfile ca-crt.pem + * server-crt.pem + * + * + * # For client (to specify a certificate client mode only - no domain): + * # Create file client_cert_ext.cnf: + * cat client_cert_ext.cnf + * + * keyUsage = critical, digitalSignature, keyEncipherment + * extendedKeyUsage = clientAuth + * basicConstraints = critical, CA:FALSE + * authorityKeyIdentifier = keyid,issuer + * subjectAltName = DNS:Client + * + * Create client cert (using the same serve key and CA) + * openssl x509 -req -in ca-csr.pem -days 1000 -CA ca-crt.pem -CAkey ca-key.pem + * -set_serial 01 -extfile client_cert_ext.cnf > client-crt.pem + * + * openssl verify -CAfile ca-crt.pem client-crt.pem + * openssl x509 -in client-crt.pem -text -noout -purpose + * + * # Compile sample: + * + * g++ -o MTLSServer MTLSServer.cc -ltrantor -lssl -lcrypto -lpthread + * + * # Tests + * + * # Should Fail + * openssl s_client -connect 0.0.0.0:8888 -state + * + * # Should Connect (the CA file must be the server CA), for this sample the CA + * is common for both openssl s_client -connect 0.0.0.0:8888 -key + * ./server-key.pem -cert ./server-crt.pem -CAfile ./ca-crt.pem -state + * + * **/ + +#include +#include +#include +#include +#include +using namespace trantor; +#define USE_IPV6 0 +int main() +{ + LOG_DEBUG << "test start"; + Logger::setLogLevel(Logger::kTrace); + EventLoopThread loopThread; + loopThread.run(); +#if USE_IPV6 + InetAddress addr(8888, true, true); +#else + InetAddress addr(8888); +#endif + TcpServer server(loopThread.getLoop(), addr, "test"); + std::vector> sslcmd = {}; + + // the CA file must be the client CA, for this sample the CA is common for + // both + server.enableSSL( + "server-crt.pem", "server-key.pem", false, sslcmd, "ca-crt.pem"); + server.setRecvMessageCallback( + [](const TcpConnectionPtr &connectionPtr, MsgBuffer *buffer) { + // LOG_DEBUG<<"recv callback!"; + std::cout << std::string(buffer->peek(), buffer->readableBytes()); + connectionPtr->send(buffer->peek(), buffer->readableBytes()); + buffer->retrieveAll(); + connectionPtr->forceClose(); + }); + server.setConnectionCallback([](const TcpConnectionPtr &connPtr) { + if (connPtr->connected()) + { + LOG_DEBUG << "New connection"; + connPtr->send("Hello world\r\n"); + } + else if (connPtr->disconnected()) + { + LOG_DEBUG << "connection disconnected"; + } + }); + server.setIoLoopNum(3); + server.start(); + loopThread.wait(); +} From 442ad902757483974c9dcd61c5a02aa05bc23a94 Mon Sep 17 00:00:00 2001 From: timomrs <92291334+timomrs@users.noreply.github.com> Date: Wed, 14 Sep 2022 04:44:29 +0300 Subject: [PATCH 083/182] Partial fix of exception safety in the event loop (#224) Previously, if an exception was thrown from the event loop, the whole app would hang in `EventLoop::~EventLoop()`. Fix that by adding a simple scopeguard for the loop flag in `EventLoop::loop()`. Also superficially fix another similar bug in `doRunInLoopFuncs()`, although as the comments point out, it's exception safety is still under question. Overall, this exposes another bug whose source is still unclear: if an exception is thrown in a loop function, the program ultimately exits with `FATAL It is forbidden to run loop on threads other than event-loop thread - EventLoop.cc:258. However, this is still better than the previous hang. --- trantor/net/EventLoop.cc | 99 +++++++++++++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 21 deletions(-) diff --git a/trantor/net/EventLoop.cc b/trantor/net/EventLoop.cc index cd25a2ca..a2231192 100644 --- a/trantor/net/EventLoop.cc +++ b/trantor/net/EventLoop.cc @@ -170,6 +170,35 @@ void EventLoop::quit() wakeup(); } } + +// The event loop needs a scope exit, so here's the simplest most limited +// C++14 scope exit available (from +// https://stackoverflow.com/a/42506763/3173540) +// +// TODO: If this is needed anywhere else, introduce a proper on_exit from, for +// example, the GSL library +namespace +{ +template +struct ScopeExit +{ + ScopeExit(F &&f) : f_(std::forward(f)) + { + } + ~ScopeExit() + { + f_(); + } + F f_; +}; + +template +ScopeExit makeScopeExit(F &&f) +{ + return ScopeExit(std::forward(f)); +}; +} // namespace + void EventLoop::loop() { assert(!looping_); @@ -177,37 +206,60 @@ void EventLoop::loop() looping_.store(true, std::memory_order_release); quit_.store(false, std::memory_order_release); - while (!quit_.load(std::memory_order_acquire)) - { - activeChannels_.clear(); + std::exception_ptr loopException; + try + { // Scope where the loop flag is set + + auto loopFlagCleaner = makeScopeExit( + [this]() { looping_.store(false, std::memory_order_release); }); + while (!quit_.load(std::memory_order_acquire)) + { + activeChannels_.clear(); #ifdef __linux__ - poller_->poll(kPollTimeMs, &activeChannels_); + poller_->poll(kPollTimeMs, &activeChannels_); #else - poller_->poll(static_cast(timerQueue_->getTimeout()), - &activeChannels_); - timerQueue_->processTimers(); + poller_->poll(static_cast(timerQueue_->getTimeout()), + &activeChannels_); + timerQueue_->processTimers(); #endif - // TODO sort channel by priority - // std::cout<<"after ->poll()"<handleEvent(); + // TODO sort channel by priority + // std::cout<<"after ->poll()"<handleEvent(); + } + currentActiveChannel_ = nullptr; + eventHandling_ = false; + // std::cout << "looping" << endl; + doRunInLoopFuncs(); } - currentActiveChannel_ = nullptr; - eventHandling_ = false; - // std::cout << "looping" << endl; - doRunInLoopFuncs(); + // loopFlagCleaner clears the loop flag here + } + catch (std::exception &e) + { + LOG_WARN << "Exception thrown from event loop, rethrowing after " + "running functions on quit"; + loopException = std::current_exception(); } - looping_.store(false, std::memory_order_release); + // Run the quit functions even if exceptions were thrown + // TODO: if more exceptions are thrown in the quit functions, some are left + // un-run. Can this be made exception safe? Func f; while (funcsOnQuit_.dequeue(f)) { f(); } + + // Throw the exception from the end + if (loopException) + { + LOG_WARN << "Rethrowing exception from event loop"; + std::rethrow_exception(loopException); + } } void EventLoop::abortNotInLoopThread() { @@ -283,8 +335,14 @@ void EventLoop::doRunInLoopFuncs() { callingFuncs_ = true; { + // Assure the flag is cleared even if func throws + auto callingFlagCleaner = + makeScopeExit([this]() { callingFuncs_ = false; }); // the destructor for the Func may itself insert a new entry into the // queue + // TODO: The following is exception-unsafe. If one of the funcs throws, + // the remaining ones will not get run. The simplest fix is to catch any + // exceptions and rethrow them later, but somehow that seems fishy... while (!funcs_.empty()) { Func func; @@ -294,7 +352,6 @@ void EventLoop::doRunInLoopFuncs() } } } - callingFuncs_ = false; } void EventLoop::wakeup() { From 6578d27f14423f3733c977a3aaec2a543483a810 Mon Sep 17 00:00:00 2001 From: An Tao Date: Wed, 14 Sep 2022 10:31:05 +0800 Subject: [PATCH 084/182] Use LOG_TRACE instead of LOG_DEBUG (#226) --- trantor/net/EventLoopThread.cc | 1 - trantor/net/TcpClient.cc | 2 +- trantor/net/inner/AresResolver.cc | 1 - trantor/net/inner/Connector.cc | 4 +-- trantor/net/inner/TcpConnectionImpl.cc | 48 +++++++++++++------------- 5 files changed, 27 insertions(+), 29 deletions(-) diff --git a/trantor/net/EventLoopThread.cc b/trantor/net/EventLoopThread.cc index 2764664c..c4170a64 100644 --- a/trantor/net/EventLoopThread.cc +++ b/trantor/net/EventLoopThread.cc @@ -63,7 +63,6 @@ void EventLoopThread::loopFuncs() auto f = promiseForRun_.get_future(); (void)f.get(); loop->loop(); - // LOG_DEBUG << "loop out"; { std::unique_lock lk(loopMutex_); loop_ = nullptr; diff --git a/trantor/net/TcpClient.cc b/trantor/net/TcpClient.cc index c440816c..7b28fb03 100644 --- a/trantor/net/TcpClient.cc +++ b/trantor/net/TcpClient.cc @@ -177,7 +177,7 @@ void TcpClient::newConnection(int sockfd) // Else the TcpClient instance has already been destroyed else { - LOG_DEBUG << "TcpClient::removeConnection was skipped because " + LOG_TRACE << "TcpClient::removeConnection was skipped because " "TcpClient instanced already freed"; c->getLoop()->queueInLoop( std::bind(&TcpConnectionImpl::connectDestroyed, diff --git a/trantor/net/inner/AresResolver.cc b/trantor/net/inner/AresResolver.cc index 029e0859..ec844614 100644 --- a/trantor/net/inner/AresResolver.cc +++ b/trantor/net/inner/AresResolver.cc @@ -131,7 +131,6 @@ void AresResolver::resolveInLoop(const std::string& hostname, struct timeval tv; struct timeval* tvp = ares_timeout(ctx_, NULL, &tv); double timeout = getSeconds(tvp); - // LOG_DEBUG << "timeout " << timeout << " active " << timerActive_; if (!timerActive_ && timeout >= 0.0) { loop_->runAfter(timeout, diff --git a/trantor/net/inner/Connector.cc b/trantor/net/inner/Connector.cc index 919c6836..9d4d3ed6 100644 --- a/trantor/net/inner/Connector.cc +++ b/trantor/net/inner/Connector.cc @@ -60,7 +60,7 @@ void Connector::startInLoop() } else { - LOG_DEBUG << "do not connect"; + LOG_TRACE << "do not connect"; } } void Connector::connect() @@ -272,6 +272,6 @@ void Connector::retry(int sockfd) } else { - LOG_DEBUG << "do not connect"; + LOG_TRACE << "do not connect"; } } diff --git a/trantor/net/inner/TcpConnectionImpl.cc b/trantor/net/inner/TcpConnectionImpl.cc index d0e91ed8..22f90c93 100644 --- a/trantor/net/inner/TcpConnectionImpl.cc +++ b/trantor/net/inner/TcpConnectionImpl.cc @@ -398,7 +398,7 @@ std::shared_ptr newSSLClientContext( { auto checkCA = SSL_CTX_load_verify_locations(ctx->get(), caPath.c_str(), NULL); - LOG_DEBUG << "CA CHECK LOC: " << checkCA; + LOG_TRACE << "CA CHECK LOC: " << checkCA; if (checkCA) { STACK_OF(X509_NAME) *cert_names = @@ -485,7 +485,7 @@ void TcpConnectionImpl::startClientEncryptionInLoop( sslEncryptionPtr_->sslCtxPtr_->mtlsEnabled); if (validateCert || sslEncryptionPtr_->sslPtr_->mtlsEnabled) { - LOG_DEBUG << "MTLS: " << sslEncryptionPtr_->sslPtr_->mtlsEnabled; + LOG_TRACE << "MTLS: " << sslEncryptionPtr_->sslPtr_->mtlsEnabled; SSL_set_verify(sslEncryptionPtr_->sslPtr_->get(), sslEncryptionPtr_->sslPtr_->mtlsEnabled ? SSL_VERIFY_PEER @@ -532,7 +532,7 @@ void TcpConnectionImpl::startServerEncryptionInLoop( if (sslEncryptionPtr_->isServer_ == false || sslEncryptionPtr_->sslPtr_->mtlsEnabled) { - LOG_DEBUG << "MTLS: " << sslEncryptionPtr_->sslPtr_->mtlsEnabled; + LOG_TRACE << "MTLS: " << sslEncryptionPtr_->sslPtr_->mtlsEnabled; SSL_set_verify(sslEncryptionPtr_->sslPtr_->get(), sslEncryptionPtr_->sslPtr_->mtlsEnabled ? SSL_VERIFY_PEER @@ -652,10 +652,10 @@ void TcpConnectionImpl::readCallback() if (errno == EPIPE || errno == ECONNRESET) { #ifdef _WIN32 - LOG_DEBUG << "WSAENOTCONN or WSAECONNRESET, errno=" << errno + LOG_TRACE << "WSAENOTCONN or WSAECONNRESET, errno=" << errno << " fd=" << socketPtr_->fd(); #else - LOG_DEBUG << "EPIPE or ECONNRESET, errno=" << errno + LOG_TRACE << "EPIPE or ECONNRESET, errno=" << errno << " fd=" << socketPtr_->fd(); #endif return; @@ -663,14 +663,14 @@ void TcpConnectionImpl::readCallback() #ifdef _WIN32 if (errno == WSAECONNABORTED) { - LOG_DEBUG << "WSAECONNABORTED, errno=" << errno; + LOG_TRACE << "WSAECONNABORTED, errno=" << errno; handleClose(); return; } #else if (errno == EAGAIN) // TODO: any others? { - LOG_DEBUG << "EAGAIN, errno=" << errno + LOG_TRACE << "EAGAIN, errno=" << errno << " fd=" << socketPtr_->fd(); return; } @@ -821,11 +821,11 @@ void TcpConnectionImpl::writeCallback() if (errno == EPIPE || errno == ECONNRESET) { #ifdef _WIN32 - LOG_DEBUG + LOG_TRACE << "WSAENOTCONN or WSAECONNRESET, errno=" << errno; #else - LOG_DEBUG << "EPIPE or ECONNRESET, errno=" << errno; + LOG_TRACE << "EPIPE or ECONNRESET, errno=" << errno; #endif return; } @@ -880,11 +880,11 @@ void TcpConnectionImpl::writeCallback() if (errno == EPIPE || errno == ECONNRESET) { #ifdef _WIN32 - LOG_DEBUG << "WSAENOTCONN or " + LOG_TRACE << "WSAENOTCONN or " "WSAECONNRESET, errno=" << errno; #else - LOG_DEBUG << "EPIPE or " + LOG_TRACE << "EPIPE or " "ECONNRESET, erron=" << errno; #endif @@ -996,7 +996,7 @@ void TcpConnectionImpl::handleError() #endif err == ECONNRESET) { - LOG_DEBUG << "[" << name_ << "] - SO_ERROR = " << err << " " + LOG_TRACE << "[" << name_ << "] - SO_ERROR = " << err << " " << strerror_tl(err); } else @@ -1079,10 +1079,10 @@ void TcpConnectionImpl::sendInLoop(const char *buffer, size_t length) if (errno == EPIPE || errno == ECONNRESET) // TODO: any others? { #ifdef _WIN32 - LOG_DEBUG << "WSAENOTCONN or WSAECONNRESET, errno=" + LOG_TRACE << "WSAENOTCONN or WSAECONNRESET, errno=" << errno; #else - LOG_DEBUG << "EPIPE or ECONNRESET, errno=" << errno; + LOG_TRACE << "EPIPE or ECONNRESET, errno=" << errno; #endif return; } @@ -1698,13 +1698,13 @@ void TcpConnectionImpl::sendFileInLoop(const BufferNodePtr &filePtr) if (errno == EPIPE || errno == ECONNRESET) { #ifdef _WIN32 - LOG_DEBUG << "WSAENOTCONN or WSAECONNRESET, errno=" + LOG_TRACE << "WSAENOTCONN or WSAECONNRESET, errno=" << errno; #else - LOG_DEBUG << "EPIPE or ECONNRESET, errno=" << errno; + LOG_TRACE << "EPIPE or ECONNRESET, errno=" << errno; #endif // abort - LOG_DEBUG + LOG_TRACE << "send stream in loop: return on connection closed"; filePtr->fileBytesToSend_ = 0; return; @@ -1788,12 +1788,12 @@ void TcpConnectionImpl::sendFileInLoop(const BufferNodePtr &filePtr) if (errno == EPIPE || errno == ECONNRESET) { #ifdef _WIN32 - LOG_DEBUG << "WSAENOTCONN or WSAECONNRESET, errno=" + LOG_TRACE << "WSAENOTCONN or WSAECONNRESET, errno=" << errno; #else - LOG_DEBUG << "EPIPE or ECONNRESET, errno=" << errno; + LOG_TRACE << "EPIPE or ECONNRESET, errno=" << errno; #endif - LOG_DEBUG + LOG_TRACE << "send file in loop: return on connection closed"; return; } @@ -1937,7 +1937,7 @@ TcpConnectionImpl::TcpConnectionImpl(EventLoop *loop, validateCert_ = validateCert; if (isServer == false || sslEncryptionPtr_->sslPtr_->mtlsEnabled) { - LOG_DEBUG << "MTLS: " << sslEncryptionPtr_->sslPtr_->mtlsEnabled; + LOG_TRACE << "MTLS: " << sslEncryptionPtr_->sslPtr_->mtlsEnabled; SSL_set_verify(sslEncryptionPtr_->sslPtr_->get(), sslEncryptionPtr_->sslPtr_->mtlsEnabled ? SSL_VERIFY_PEER @@ -1975,14 +1975,14 @@ bool TcpConnectionImpl::validatePeerCertificate() result != X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN && result != X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY) { - LOG_DEBUG << "cert error code: " << result; + LOG_TRACE << "cert error code: " << result; LOG_ERROR << "Server certificate is not valid"; return false; } #else if (result != X509_V_OK && result) { - LOG_DEBUG << "cert error code: " << result; + LOG_TRACE << "cert error code: " << result; LOG_ERROR << "Server certificate is not valid"; return false; } @@ -2000,7 +2000,7 @@ bool TcpConnectionImpl::validatePeerCertificate() internal::verifyAltName(cert, sslEncryptionPtr_->hostname_); X509_free(cert); - LOG_DEBUG << "domainIsValid: " << domainIsValid; + LOG_TRACE << "domainIsValid: " << domainIsValid; // if mtlsEnabled, ignore domain validation if (sslEncryptionPtr_->sslPtr_->mtlsEnabled || domainIsValid) From 77ed7a6a58591b7c7f54d93ce55e725a19be462b Mon Sep 17 00:00:00 2001 From: An Tao Date: Sun, 25 Sep 2022 22:56:58 +0800 Subject: [PATCH 085/182] Bump version to 1.5.7 (#227) --- CMakeLists.txt | 2 +- ChangeLog.md | 32 +++++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3ec011e2..fe393cbe 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) set(TRANTOR_MAJOR_VERSION 1) set(TRANTOR_MINOR_VERSION 5) -set(TRANTOR_PATCH_VERSION 6) +set(TRANTOR_PATCH_VERSION 7) set(TRANTOR_VERSION ${TRANTOR_MAJOR_VERSION}.${TRANTOR_MINOR_VERSION}.${TRANTOR_PATCH_VERSION}) diff --git a/ChangeLog.md b/ChangeLog.md index c09de8a3..54139364 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -3,6 +3,34 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [1.5.7] - 2022-09-25 + +### API changes list + +- Add utc methods for trantor::Date. + +### Changed + +- Remove an unnecessary semi-colon. + +- Added support for Solaris. + +- Define ssize_t as std::intptr_t on Windows. + +- Add an environment without openssl to github actions. + +- Added SSL Error Trace Log and mTLS Samples. + +- Use LOG_TRACE instead of LOG_DEBUG. + +### Fixed + +- Fix a race condition. + +- Fix iterator invalidation bug when stopping TCP server. + +- Partial fix of exception safety in the event loop. + ## [1.5.6] - 2022-07-09 ### API changes list @@ -444,7 +472,9 @@ All notable changes to this project will be documented in this file. ## [1.0.0-rc1] - 2019-06-11 -[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.6...HEAD +[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.7...HEAD + +[1.5.7]: https://github.com/an-tao/trantor/compare/v1.5.6...v1.5.7 [1.5.6]: https://github.com/an-tao/trantor/compare/v1.5.5...v1.5.6 From 62c9027d952b90f763cd9482fb729fa9a0924428 Mon Sep 17 00:00:00 2001 From: Martin Chang Date: Sat, 8 Oct 2022 23:17:41 +0800 Subject: [PATCH 086/182] Fix socket fd leak if Connector destruct before connection callback is made (#230) --- trantor/net/inner/Connector.cc | 36 ++++++++++++++++++++++++++-------- trantor/net/inner/Connector.h | 3 +++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/trantor/net/inner/Connector.cc b/trantor/net/inner/Connector.cc index 9d4d3ed6..1f139042 100644 --- a/trantor/net/inner/Connector.cc +++ b/trantor/net/inner/Connector.cc @@ -27,6 +27,18 @@ Connector::Connector(EventLoop *loop, InetAddress &&addr, bool retry) { } +Connector::~Connector() +{ + if (socketHanded_ == false && fd_ != -1) + { +#ifndef _WIN32 + ::close(fd_); +#else + closesocket(fd_); +#endif + } +} + void Connector::start() { connect_ = true; @@ -65,9 +77,10 @@ void Connector::startInLoop() } void Connector::connect() { - int sockfd = Socket::createNonblockingSocketOrDie(serverAddr_.family()); + socketHanded_ = false; + fd_ = Socket::createNonblockingSocketOrDie(serverAddr_.family()); errno = 0; - int ret = Socket::connect(sockfd, serverAddr_); + int ret = Socket::connect(fd_, serverAddr_); int savedErrno = (ret == 0) ? 0 : errno; switch (savedErrno) { @@ -76,7 +89,7 @@ void Connector::connect() case EINTR: case EISCONN: LOG_TRACE << "connecting"; - connecting(sockfd); + connecting(fd_); break; case EAGAIN: @@ -86,7 +99,7 @@ void Connector::connect() case ENETUNREACH: if (retry_) { - retry(sockfd); + retry(fd_); } break; @@ -99,10 +112,11 @@ void Connector::connect() case ENOTSOCK: LOG_SYSERR << "connect error in Connector::startInLoop " << savedErrno; + socketHanded_ = true; #ifndef _WIN32 - ::close(sockfd); + ::close(fd_); #else - closesocket(sockfd); + closesocket(fd_); #endif if (errorCallback_) errorCallback_(); @@ -111,10 +125,11 @@ void Connector::connect() default: LOG_SYSERR << "Unexpected error in Connector::startInLoop " << savedErrno; + socketHanded_ = true; #ifndef _WIN32 - ::close(sockfd); + ::close(fd_); #else - closesocket(sockfd); + closesocket(fd_); #endif if (errorCallback_) errorCallback_(); @@ -154,6 +169,7 @@ int Connector::removeAndResetChannel() void Connector::handleWrite() { + socketHanded_ = true; if (status_ == Status::Connecting) { int sockfd = removeAndResetChannel(); @@ -168,6 +184,7 @@ void Connector::handleWrite() } else { + socketHanded_ = true; #ifndef _WIN32 ::close(sockfd); #else @@ -188,6 +205,7 @@ void Connector::handleWrite() } else { + socketHanded_ = true; #ifndef _WIN32 ::close(sockfd); #else @@ -208,6 +226,7 @@ void Connector::handleWrite() } else { + socketHanded_ = true; #ifndef _WIN32 ::close(sockfd); #else @@ -225,6 +244,7 @@ void Connector::handleWrite() void Connector::handleError() { + socketHanded_ = true; if (status_ == Status::Connecting) { status_ = Status::Disconnected; diff --git a/trantor/net/inner/Connector.h b/trantor/net/inner/Connector.h index eda33736..91bce8e8 100644 --- a/trantor/net/inner/Connector.h +++ b/trantor/net/inner/Connector.h @@ -30,6 +30,7 @@ class Connector : public NonCopyable, using ConnectionErrorCallback = std::function; Connector(EventLoop *loop, const InetAddress &addr, bool retry = true); Connector(EventLoop *loop, InetAddress &&addr, bool retry = true); + ~Connector(); void setNewConnectionCallback(const NewConnectionCallback &cb) { newConnectionCallback_ = cb; @@ -76,6 +77,8 @@ class Connector : public NonCopyable, int maxRetryInterval_{kMaxRetryDelayMs}; bool retry_; + bool socketHanded_{false}; + int fd_{-1}; void startInLoop(); void connect(); From c581105c4e90de7dce37e676d67c1c7a25e93174 Mon Sep 17 00:00:00 2001 From: Nitromelon Date: Wed, 26 Oct 2022 12:44:20 +0800 Subject: [PATCH 087/182] Fix Date::timezoneOffset(). (#232) --- trantor/utils/Date.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trantor/utils/Date.h b/trantor/utils/Date.h index a3677979..73386c4c 100644 --- a/trantor/utils/Date.h +++ b/trantor/utils/Date.h @@ -77,7 +77,7 @@ class TRANTOR_EXPORT Date static int64_t timezoneOffset() { static int64_t offset = - 0 - Date::fromDbStringLocal("1970-01-01").secondsSinceEpoch(); + -Date::fromDbStringLocal("1970-01-01 00:00:00").secondsSinceEpoch(); return offset; } From 625458079fd4344dba2f5e9093596fe8b9d4fa0c Mon Sep 17 00:00:00 2001 From: an-tao Date: Fri, 11 Nov 2022 22:17:00 +0800 Subject: [PATCH 088/182] Bump version to 1.5.8 --- CMakeLists.txt | 2 +- ChangeLog.md | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fe393cbe..7f3c3522 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) set(TRANTOR_MAJOR_VERSION 1) set(TRANTOR_MINOR_VERSION 5) -set(TRANTOR_PATCH_VERSION 7) +set(TRANTOR_PATCH_VERSION 8) set(TRANTOR_VERSION ${TRANTOR_MAJOR_VERSION}.${TRANTOR_MINOR_VERSION}.${TRANTOR_PATCH_VERSION}) diff --git a/ChangeLog.md b/ChangeLog.md index 54139364..4c813356 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,8 +1,21 @@ # Changelog + All notable changes to this project will be documented in this file. ## [Unreleased] +## [1.5.8] - 2022-11-11 + +### API Changes list + +### Changed + +### Fixed + +- Fix Date::timezoneOffset(). + +- Fix socket fd leak if Connector destruct before connection callback is made. + ## [1.5.7] - 2022-09-25 ### API changes list @@ -401,7 +414,6 @@ All notable changes to this project will be documented in this file. - Use the std::chrono::steady_clock for timers - ## [1.0.0-rc7] - 2019-11-21 ### Changed @@ -430,7 +442,7 @@ All notable changes to this project will be documented in this file. - Add the Resolver class that provides high-performance DNS functionality(with c-ares library) - Add some unit tests. - + ## [1.0.0-rc4] - 2019-08-08 ### API changes list @@ -472,7 +484,9 @@ All notable changes to this project will be documented in this file. ## [1.0.0-rc1] - 2019-06-11 -[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.7...HEAD +[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.8...HEAD + +[1.5.8]: https://github.com/an-tao/trantor/compare/v1.5.7...v1.5.8 [1.5.7]: https://github.com/an-tao/trantor/compare/v1.5.6...v1.5.7 From 894849377f55d6144eecafd4874c38ce5a6d9bf5 Mon Sep 17 00:00:00 2001 From: sbrajchuk <78134229+sbrajchuk@users.noreply.github.com> Date: Wed, 23 Nov 2022 05:30:21 +0300 Subject: [PATCH 089/182] Search for \ if under msvc (#236) --- trantor/utils/Logger.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/trantor/utils/Logger.h b/trantor/utils/Logger.h index 862186cd..40e2600c 100644 --- a/trantor/utils/Logger.h +++ b/trantor/utils/Logger.h @@ -56,7 +56,11 @@ class TRANTOR_EXPORT Logger : public NonCopyable inline SourceFile(const char (&arr)[N]) : data_(arr), size_(N - 1) { // std::cout< Date: Mon, 23 Jan 2023 09:33:51 +0800 Subject: [PATCH 090/182] Bump version to 1.5.9 --- CMakeLists.txt | 2 +- ChangeLog.md | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f3c3522..8e742c6b 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) set(TRANTOR_MAJOR_VERSION 1) set(TRANTOR_MINOR_VERSION 5) -set(TRANTOR_PATCH_VERSION 8) +set(TRANTOR_PATCH_VERSION 9) set(TRANTOR_VERSION ${TRANTOR_MAJOR_VERSION}.${TRANTOR_MINOR_VERSION}.${TRANTOR_PATCH_VERSION}) diff --git a/ChangeLog.md b/ChangeLog.md index 4c813356..0e762a4d 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [1.5.9] - 2023-01-23 + +### API Changes list + +### Changed + +- Search for \ if under msvc + +### Fixed + ## [1.5.8] - 2022-11-11 ### API Changes list @@ -484,7 +494,9 @@ All notable changes to this project will be documented in this file. ## [1.0.0-rc1] - 2019-06-11 -[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.8...HEAD +[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.9...HEAD + +[1.5.9]: https://github.com/an-tao/trantor/compare/v1.5.8...v1.5.9 [1.5.8]: https://github.com/an-tao/trantor/compare/v1.5.7...v1.5.8 From 26575e1dff7c99876d5a41643847ee86307ae70a Mon Sep 17 00:00:00 2001 From: An Tao Date: Mon, 23 Jan 2023 10:37:10 +0800 Subject: [PATCH 091/182] Use gtest 1.13 in github actions (#237) --- .github/workflows/cmake.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index f847d2ba..1aeef4de 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -100,9 +100,9 @@ jobs: sudo apt install dos2unix - name: install gtest run: | - wget https://github.com/google/googletest/archive/release-1.10.0.tar.gz - tar xf release-1.10.0.tar.gz - cd googletest-release-1.10.0 + wget https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz + tar xf v1.13.0.tar.gz + cd googletest-1.13.0 cmake . make sudo make install From ae4fa2e474ca2079825637a5cbc06c5bcbbbbe1b Mon Sep 17 00:00:00 2001 From: an-tao Date: Mon, 23 Jan 2023 10:43:28 +0800 Subject: [PATCH 092/182] Bump version to 1.5.10 --- CMakeLists.txt | 2 +- ChangeLog.md | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e742c6b..027c41a7 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules/) set(TRANTOR_MAJOR_VERSION 1) set(TRANTOR_MINOR_VERSION 5) -set(TRANTOR_PATCH_VERSION 9) +set(TRANTOR_PATCH_VERSION 10) set(TRANTOR_VERSION ${TRANTOR_MAJOR_VERSION}.${TRANTOR_MINOR_VERSION}.${TRANTOR_PATCH_VERSION}) diff --git a/ChangeLog.md b/ChangeLog.md index 0e762a4d..9a29498e 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [1.5.10] - 2023-01-23 + +### API Changes list + +### Changed + +- Use gtest 1.13 in github actions + +### Fixed + ## [1.5.9] - 2023-01-23 ### API Changes list @@ -494,7 +504,9 @@ All notable changes to this project will be documented in this file. ## [1.0.0-rc1] - 2019-06-11 -[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.9...HEAD +[Unreleased]: https://github.com/an-tao/trantor/compare/v1.5.10...HEAD + +[1.5.10]: https://github.com/an-tao/trantor/compare/v1.5.9...v1.5.10 [1.5.9]: https://github.com/an-tao/trantor/compare/v1.5.8...v1.5.9 From 41fa158cae1c015bde3c52bc485c363d54a501b2 Mon Sep 17 00:00:00 2001 From: An Tao Date: Sun, 19 Feb 2023 15:38:20 +0800 Subject: [PATCH 093/182] Add a method to the Logger class to enable local time displaying (#240) --- trantor/utils/Logger.cc | 49 ++++++++++++++++++++++++++++++++--------- trantor/utils/Logger.h | 22 ++++++++++++++++++ 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/trantor/utils/Logger.cc b/trantor/utils/Logger.cc index b86d8ccc..6db2f2d1 100644 --- a/trantor/utils/Logger.cc +++ b/trantor/utils/Logger.cc @@ -88,24 +88,51 @@ void Logger::formatTime() if (now != lastSecond_) { lastSecond_ = now; + if (displayLocalTime_()) + { #ifndef _MSC_VER - strncpy(lastTimeString_, - date_.toFormattedString(false).c_str(), + strncpy(lastTimeString_, + date_.toFormattedStringLocal(false).c_str(), + sizeof(lastTimeString_) - 1); +#else + strncpy_s( + lastTimeString_, + date_.toFormattedStringLocal(false).c_str(), sizeof(lastTimeString_) - 1); +#endif + } + else + { +#ifndef _MSC_VER + strncpy(lastTimeString_, + date_.toFormattedString(false).c_str(), + sizeof(lastTimeString_) - 1); #else - strncpy_s( - lastTimeString_, - date_.toFormattedString(false).c_str(), - sizeof(lastTimeString_) - 1); + strncpy_s( + lastTimeString_, + date_.toFormattedString(false).c_str(), + sizeof(lastTimeString_) - 1); #endif + } } logStream_ << T(lastTimeString_, 17); char tmp[32]; - snprintf(tmp, - sizeof(tmp), - ".%06llu UTC ", - static_cast(microSec)); - logStream_ << T(tmp, 12); + if (displayLocalTime_()) + { + snprintf(tmp, + sizeof(tmp), + ".%06llu ", + static_cast(microSec)); + logStream_ << T(tmp, 8); + } + else + { + snprintf(tmp, + sizeof(tmp), + ".%06llu UTC ", + static_cast(microSec)); + logStream_ << T(tmp, 12); + } #ifdef __linux__ if (threadId_ == 0) threadId_ = static_cast(::syscall(SYS_gettid)); diff --git a/trantor/utils/Logger.h b/trantor/utils/Logger.h index 40e2600c..d21e0d6b 100644 --- a/trantor/utils/Logger.h +++ b/trantor/utils/Logger.h @@ -141,6 +141,22 @@ class TRANTOR_EXPORT Logger : public NonCopyable return logLevel_(); } + /** + * @brief Check whether it shows local time or UTC time. + */ + static bool displayLocalTime() + { + return displayLocalTime_(); + } + + /** + * @brief Set whether it shows local time or UTC time. the default is UTC. + */ + static void setDisplayLocalTime(bool showLocalTime) + { + displayLocalTime_() = showLocalTime; + } + protected: static void defaultOutputFunction(const char *msg, const uint64_t len) { @@ -151,6 +167,12 @@ class TRANTOR_EXPORT Logger : public NonCopyable fflush(stdout); } void formatTime(); + static bool &displayLocalTime_() + { + static bool showLocalTime = false; + return showLocalTime; + } + static LogLevel &logLevel_() { #ifdef RELEASE From 8da2d4015120b486864ac1420dd70ce3b433d3f6 Mon Sep 17 00:00:00 2001 From: Nitromelon Date: Sat, 25 Feb 2023 00:20:34 +0800 Subject: [PATCH 094/182] Refactor TcpServer I/O loop logic. (#239) --- trantor/net/EventLoop.cc | 3 +- trantor/net/TcpServer.cc | 70 +++++++++++++++------------------------- trantor/net/TcpServer.h | 62 +++++++++++++++++++++++++++-------- 3 files changed, 77 insertions(+), 58 deletions(-) diff --git a/trantor/net/EventLoop.cc b/trantor/net/EventLoop.cc index a2231192..acaa4713 100644 --- a/trantor/net/EventLoop.cc +++ b/trantor/net/EventLoop.cc @@ -241,7 +241,8 @@ void EventLoop::loop() catch (std::exception &e) { LOG_WARN << "Exception thrown from event loop, rethrowing after " - "running functions on quit"; + "running functions on quit: " + << e.what(); loopException = std::current_exception(); } diff --git a/trantor/net/TcpServer.cc b/trantor/net/TcpServer.cc index 23b57d48..e22b8cb2 100644 --- a/trantor/net/TcpServer.cc +++ b/trantor/net/TcpServer.cc @@ -12,31 +12,33 @@ * */ -#include "Acceptor.h" -#include "inner/TcpConnectionImpl.h" #include #include #include #include +#include "Acceptor.h" +#include "inner/TcpConnectionImpl.h" using namespace trantor; using namespace std::placeholders; TcpServer::TcpServer(EventLoop *loop, const InetAddress &address, - const std::string &name, + std::string name, bool reUseAddr, bool reUsePort) : loop_(loop), acceptorPtr_(new Acceptor(loop, address, reUseAddr, reUsePort)), - serverName_(name), + serverName_(std::move(name)), recvMessageCallback_([](const TcpConnectionPtr &, MsgBuffer *buffer) { LOG_ERROR << "unhandled recv message [" << buffer->readableBytes() << " bytes]"; buffer->retrieveAll(); - }) + }), + ioLoops_({loop}), + numIoLoops_(1) { acceptorPtr_->setNewConnectionCallback( - std::bind(&TcpServer::newConnection, this, _1, _2)); + [this](int fd, const InetAddress &peer) { newConnection(fd, peer); }); } TcpServer::~TcpServer() @@ -49,21 +51,12 @@ void TcpServer::newConnection(int sockfd, const InetAddress &peer) { LOG_TRACE << "new connection:fd=" << sockfd << " address=" << peer.toIpPort(); - // test code for blocking or nonblocking - // std::vector str(1024*1024*100); - // for(int i=0;iassertInLoopThread(); - EventLoop *ioLoop = NULL; - if (loopPoolPtr_ && loopPoolPtr_->size() > 0) + EventLoop *ioLoop = ioLoops_[nextLoopIdx_]; + if (++nextLoopIdx_ >= numIoLoops_) { - ioLoop = loopPoolPtr_->getNextLoop(); + nextLoopIdx_ = 0; } - if (ioLoop == NULL) - ioLoop = loop_; std::shared_ptr newPtr; if (sslCtxPtr_) { @@ -102,7 +95,10 @@ void TcpServer::newConnection(int sockfd, const InetAddress &peer) if (writeCompleteCallback_) writeCompleteCallback_(connectionPtr); }); - newPtr->setCloseCallback(std::bind(&TcpServer::connectionClosed, this, _1)); + + newPtr->setCloseCallback([this](const TcpConnectionPtr &closeConnPtr) { + connectionClosed(closeConnPtr); + }); connSet_.insert(newPtr); newPtr->connectEstablished(); } @@ -114,29 +110,15 @@ void TcpServer::start() started_ = true; if (idleTimeout_ > 0) { - timingWheelMap_[loop_] = - std::make_shared(loop_, - idleTimeout_, - 1.0F, - idleTimeout_ < 500 - ? idleTimeout_ + 1 - : 100); - if (loopPoolPtr_) + for (EventLoop *loop : ioLoops_) { - auto loopNum = loopPoolPtr_->size(); - while (loopNum > 0) - { - // LOG_TRACE << "new Wheel loopNum=" << loopNum; - auto poolLoop = loopPoolPtr_->getNextLoop(); - timingWheelMap_[poolLoop] = - std::make_shared(poolLoop, - idleTimeout_, - 1.0F, - idleTimeout_ < 500 - ? idleTimeout_ + 1 - : 100); - --loopNum; - } + timingWheelMap_[loop] = + std::make_shared(loop, + idleTimeout_, + 1.0F, + idleTimeout_ < 500 + ? idleTimeout_ + 1 + : 100); } } LOG_TRACE << "map size=" << timingWheelMap_.size(); @@ -156,7 +138,7 @@ void TcpServer::stop() { connPtrs.push_back(conn); } - for (auto connection : connPtrs) + for (auto &connection : connPtrs) { connection->forceClose(); } @@ -173,7 +155,7 @@ void TcpServer::stop() { connPtrs.push_back(conn); } - for (auto connection : connPtrs) + for (auto &connection : connPtrs) { connection->forceClose(); } @@ -223,7 +205,7 @@ void TcpServer::connectionClosed(const TcpConnectionPtr &connectionPtr) } } -const std::string TcpServer::ipPort() const +std::string TcpServer::ipPort() const { return acceptorPtr_->addr().toIpPort(); } diff --git a/trantor/net/TcpServer.h b/trantor/net/TcpServer.h index d5b77062..700d3689 100644 --- a/trantor/net/TcpServer.h +++ b/trantor/net/TcpServer.h @@ -13,18 +13,19 @@ */ #pragma once -#include -#include -#include +#include #include #include #include +#include +#include +#include #include -#include -#include +#include #include #include -#include +#include + namespace trantor { class Acceptor; @@ -48,7 +49,7 @@ class TRANTOR_EXPORT TcpServer : NonCopyable */ TcpServer(EventLoop *loop, const InetAddress &address, - const std::string &name, + std::string name, bool reUseAddr = true, bool reUsePort = true); ~TcpServer(); @@ -68,6 +69,7 @@ class TRANTOR_EXPORT TcpServer : NonCopyable /** * @brief Set the number of event loops in which the I/O of connections to * the server is handled. + * An EventLoopThreadPool is created and managed by TcpServer. * * @param num */ @@ -76,11 +78,14 @@ class TRANTOR_EXPORT TcpServer : NonCopyable assert(!started_); loopPoolPtr_ = std::make_shared(num); loopPoolPtr_->start(); + ioLoops_ = loopPoolPtr_->getLoops(); + numIoLoops_ = ioLoops_.size(); } /** * @brief Set the event loops pool in which the I/O of connections to * the server is handled. + * A shared_ptr of EventLoopThreadPool is copied. * * @param pool */ @@ -89,7 +94,26 @@ class TRANTOR_EXPORT TcpServer : NonCopyable assert(pool->size() > 0); assert(!started_); loopPoolPtr_ = pool; - loopPoolPtr_->start(); + loopPoolPtr_->start(); // TODO: should not start by TcpServer + ioLoops_ = loopPoolPtr_->getLoops(); + numIoLoops_ = ioLoops_.size(); + } + + /** + * @brief Set the event loops in which the I/O of connections to + * the server is handled. + * The loops are managed by caller. Caller should ensure that ioLoops + * lives longer than TcpServer. + * + * @param ioLoops + */ + void setIoLoops(const std::vector &ioLoops) + { + assert(!ioLoops.empty()); + assert(!started_); + ioLoops_ = ioLoops; + numIoLoops_ = ioLoops_.size(); + loopPoolPtr_.reset(); } /** @@ -152,7 +176,7 @@ class TRANTOR_EXPORT TcpServer : NonCopyable * * @return const std::string */ - const std::string ipPort() const; + std::string ipPort() const; /** * @brief Get the address of the server. @@ -178,7 +202,7 @@ class TRANTOR_EXPORT TcpServer : NonCopyable */ std::vector getIoLoops() const { - return loopPoolPtr_->getLoops(); + return ioLoops_; } /** @@ -215,10 +239,12 @@ class TRANTOR_EXPORT TcpServer : NonCopyable const std::string &caPath = ""); private: - EventLoop *loop_; void handleCloseInLoop(const TcpConnectionPtr &connectionPtr); - std::unique_ptr acceptorPtr_; void newConnection(int fd, const InetAddress &peer); + void connectionClosed(const TcpConnectionPtr &connectionPtr); + + EventLoop *loop_; + std::unique_ptr acceptorPtr_; std::string serverName_; std::set connSet_; @@ -228,8 +254,18 @@ class TRANTOR_EXPORT TcpServer : NonCopyable size_t idleTimeout_{0}; std::map> timingWheelMap_; - void connectionClosed(const TcpConnectionPtr &connectionPtr); + + // `loopPoolPtr_` may and may not hold the internal thread pool. + // We should not access it directly in codes. + // Instead, we should use its delegation variable `ioLoops_`. std::shared_ptr loopPoolPtr_; + // If one of `setIoLoopNum()`, `setIoLoopThreadPool()` and `setIoLoops()` is + // called, `ioLoops_` will hold the loops passed in. + // Otherwise, it should contain only one element, which is `loop_`. + std::vector ioLoops_; + size_t nextLoopIdx_{0}; + size_t numIoLoops_{0}; + #ifndef _WIN32 class IgnoreSigPipe { From b7ae847d5a7adba0e03b3aa232fc91c8f66c3be3 Mon Sep 17 00:00:00 2001 From: An Tao Date: Sun, 26 Feb 2023 09:38:01 +0800 Subject: [PATCH 095/182] Fix a conan issue (#241) --- .github/workflows/cmake.yml | 5 +++-- conanfile.txt | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 1aeef4de..aa730328 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -36,14 +36,15 @@ jobs: shell: pwsh working-directory: ./build run: | - conan install .. -s compiler="Visual Studio" -s compiler.version=16 -sbuild_type=Debug -g cmake_paths + conan profile detect + conan install .. -s compiler="msvc" -s compiler.version=193 -sbuild_type=Debug --build=missing - name: Create Build Environment & Configure Cmake shell: bash working-directory: ./build run: | [[ ${{ matrix.link }} == "SHARED" ]] && shared="ON" || shared="OFF" - cmake .. -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=on -DBUILD_SHARED_LIBS=$shared -DCMAKE_TOOLCHAIN_FILE="conan_paths.cmake" -DCMAKE_INSTALL_PREFIX=../install + cmake .. -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=on -DBUILD_SHARED_LIBS=$shared -DCMAKE_TOOLCHAIN_FILE="conan_toolchain.cmake" -DCMAKE_INSTALL_PREFIX=../install -DCMAKE_POLICY_DEFAULT_CMP0091=NEW - name: Build working-directory: ${{env.GITHUB_WORKSPACE}} diff --git a/conanfile.txt b/conanfile.txt index 59906b3a..8bfdb864 100644 --- a/conanfile.txt +++ b/conanfile.txt @@ -1,10 +1,10 @@ [requires] gtest/1.10.0 -openssl/1.1.1j +openssl/1.1.1t #c-ares/1.17.1 [generators] -cmake_paths +CMakeToolchain [options] From 5391ff83195da7d5130997820cf63c6c7034f76f Mon Sep 17 00:00:00 2001 From: Ostropik Date: Fri, 17 Mar 2023 05:50:34 +0200 Subject: [PATCH 096/182] TRNANTOR_LOG_COMPACT - compact logs without source code details(line, file name etc) (#242) Co-authored-by: Ostropik Co-authored-by: antao --- trantor/tests/LoggerTest.cc | 1 + trantor/utils/Logger.cc | 31 +++++++++++++++++++++++++--- trantor/utils/Logger.h | 40 ++++++++++++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/trantor/tests/LoggerTest.cc b/trantor/tests/LoggerTest.cc index 408fcb77..fafd849c 100644 --- a/trantor/tests/LoggerTest.cc +++ b/trantor/tests/LoggerTest.cc @@ -4,6 +4,7 @@ int main() { int i; + LOG_COMPACT_DEBUG << "Hello, world!"; LOG_DEBUG << (float)3.14; LOG_DEBUG << (const char)'8'; LOG_DEBUG << &i; diff --git a/trantor/utils/Logger.cc b/trantor/utils/Logger.cc index 6db2f2d1..74f1315b 100644 --- a/trantor/utils/Logger.cc +++ b/trantor/utils/Logger.cc @@ -67,6 +67,7 @@ inline LogStream &operator<<(LogStream &s, const Logger::SourceFile &v) s.append(v.data_, v.size_); return s; } + } // namespace trantor using namespace trantor; @@ -174,6 +175,7 @@ static const char *logLevelStr[Logger::LogLevel::kNumberOfLogLevels] = { " ERROR ", " FATAL ", }; + Logger::Logger(SourceFile file, int line) : sourceFile_(file), fileLine_(line), level_(kInfo) { @@ -202,6 +204,28 @@ Logger::Logger(SourceFile file, int line, bool) logStream_ << strerror_tl(errno) << " (errno=" << errno << ") "; } } + +// LOG_COMPACT +Logger::Logger() : level_(kInfo) +{ + formatTime(); + logStream_ << T(logLevelStr[level_], 7); +} +Logger::Logger(LogLevel level) : level_(level) +{ + formatTime(); + logStream_ << T(logLevelStr[level_], 7); +} +Logger::Logger(bool) : level_(kFatal) +{ + formatTime(); + logStream_ << T(logLevelStr[level_], 7); + if (errno != 0) + { + logStream_ << strerror_tl(errno) << " (errno=" << errno << ") "; + } +} + RawLogger::~RawLogger() { if (index_ < 0) @@ -221,7 +245,10 @@ RawLogger::~RawLogger() } Logger::~Logger() { - logStream_ << T(" - ", 3) << sourceFile_ << ':' << fileLine_ << '\n'; + if (sourceFile_.data_) + logStream_ << T(" - ", 3) << sourceFile_ << ':' << fileLine_ << '\n'; + else + logStream_ << '\n'; if (index_ < 0) { auto &oFunc = Logger::outputFunc_(); @@ -240,8 +267,6 @@ Logger::~Logger() if (level_ >= kError) Logger::flushFunc_(index_)(); } - - // logStream_.resetBuffer(); } LogStream &Logger::stream() { diff --git a/trantor/utils/Logger.h b/trantor/utils/Logger.h index d21e0d6b..8d19c71f 100644 --- a/trantor/utils/Logger.h +++ b/trantor/utils/Logger.h @@ -68,8 +68,13 @@ class TRANTOR_EXPORT Logger : public NonCopyable } } - explicit SourceFile(const char *filename) : data_(filename) + explicit SourceFile(const char *filename = nullptr) : data_(filename) { + if (!filename) + { + size_ = 0; + return; + } #ifndef _MSC_VER const char *slash = strrchr(filename, '/'); #else @@ -89,6 +94,12 @@ class TRANTOR_EXPORT Logger : public NonCopyable Logger(SourceFile file, int line, LogLevel level); Logger(SourceFile file, int line, bool isSysErr); Logger(SourceFile file, int line, LogLevel level, const char *func); + + // LOG_COMPACT only