diff --git a/.gitignore b/.gitignore index 25d1efd..07f3eb7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ *.o *.a +*.exe main -main.exe main-sb* main-lb* -output.txt +output.txt \ No newline at end of file diff --git a/config.ini b/config.ini new file mode 100644 index 0000000..49d93a6 --- /dev/null +++ b/config.ini @@ -0,0 +1,11 @@ +version=1.3.8r +discord_webhook= # too lazy atm +shroomin_api_key= + +default_print_interval=256 +default_threads=16 + +ULB_sizethreshold=128000000 +LB_sizethreshold=96000000 +USB_sizethreshold=10000000 +SB_sizethreshold=10000000 \ No newline at end of file diff --git a/makefile b/makefile index 0669883..a1ad716 100644 --- a/makefile +++ b/makefile @@ -3,6 +3,7 @@ CUBIOMES_SRC := $(addprefix cubiomes/,biomenoise.c biomes.c finders.c generator. LARGE_BIOMES ?= 0 UNBOUND ?= 0 PRINT_INTERVAL ?= 256 +OUT ?= main.exe # Auto-detect GPU architecture: # - RTX 40xx/50xx series: sm_89 is faster than native sm_120 # - Everything else: use native @@ -20,12 +21,19 @@ endif $(info Using ARCH = $(ARCH)) override CFLAGS += -O3 -override CXXFLAGS += -O3 -std=c++20 -I asio/asio/include -DOMISSION_LARGE_BIOMES=$(LARGE_BIOMES) -DOMISSION_UNBOUND=$(UNBOUND) -DPRINT_INTERVAL=$(PRINT_INTERVAL) +override CXXFLAGS += -O3 -std=c++20 -I asio/asio/include \ + -DOMISSION_LARGE_BIOMES=$(LARGE_BIOMES) \ + -DOMISSION_UNBOUND=$(UNBOUND) \ + -DPRINT_INTERVAL=$(PRINT_INTERVAL) + override NVCC_FLAGS += $(CXXFLAGS) --expt-relaxed-constexpr --default-stream per-thread -arch=$(ARCH) -use_fast_math ifeq ($(OS),Windows_NT) + all: main.exe +all4: SB.exe USB.exe LB.exe ULB.exe + SRC_CPP := $(wildcard src/*.cpp) SRC_C := $(wildcard src/*.c) SRC_CU := $(wildcard src/*.cu) @@ -33,16 +41,50 @@ SRC := $(SRC_CPP) $(SRC_C) $(SRC_CU) clean: del /Q main.exe - -# nvcc src/*.cpp src/*.c src/*.cu -o main.exe cubiomes/biomenoise.c cubiomes/biomes.c cubiomes/finders.c cubiomes/generator.c cubiomes/layers.c cubiomes/noise.c -arch=native -O3 -std=c++20 -I asio-1.34.2/include -DOMISSION_LARGE_BIOMES=1 --expt-relaxed-constexpr --default-stream per-thread -D_WIN32_WINNT=0x0601 + main.exe: $(SRC) $(CUBIOMES_SRC) nvcc $(SRC) $(CUBIOMES_SRC) -o $@ $(NVCC_FLAGS) -D_WIN32_WINNT=0x0601 + +SB.exe: + $(MAKE) OUT=SB.exe LARGE_BIOMES=0 UNBOUND=0 build + +USB.exe: + $(MAKE) OUT=USB.exe LARGE_BIOMES=0 UNBOUND=1 build + +LB.exe: + $(MAKE) OUT=LB.exe LARGE_BIOMES=1 UNBOUND=0 build + +ULB.exe: + $(MAKE) OUT=ULB.exe LARGE_BIOMES=1 UNBOUND=1 build + +build: $(SRC) $(CUBIOMES_SRC) + nvcc $(SRC) $(CUBIOMES_SRC) -o $(OUT) \ + -O3 -std=c++20 \ + -I asio/asio/include \ + -DOMISSION_LARGE_BIOMES=$(LARGE_BIOMES) \ + -DOMISSION_UNBOUND=$(UNBOUND) \ + -DPRINT_INTERVAL=$(PRINT_INTERVAL) \ + --expt-relaxed-constexpr \ + --default-stream per-thread \ + -arch=native \ + -D_WIN32_WINNT=0x0601 + else + override NVCC_FLAGS += -ccbin $(CXX) MAIN_SRC := src/main.cpp MAIN_DEP := $(MAIN_SRC) src/common.h +MAIN_SRC += config.o shroomposter.o +MAIN_DEP += config.o shroomposter.o src/config.h src/shroomposter.h + +ifneq ($(wildcard /usr/include/openssl/err.h),) +MAIN_SRC += cpp20_http_client.o +MAIN_DEP += cpp20_http_client.o src/cpp20_http_client.hpp +MAIN_CXXFLAGS += -lssl -lcrypto +endif + ifndef NO_GPU MAIN_SRC += gpu.o MAIN_DEP += gpu.o src/gpu.h @@ -70,7 +112,7 @@ endif all: main clean: - rm -f main libcubiomes.a biomenoise.o biomes.o finders.o generator.o layers.o noise.o cubiomes.o gpu.o cpu.o client.o server.o + rm -f main libcubiomes.a biomenoise.o biomes.o finders.o generator.o layers.o noise.o cubiomes.o gpu.o cpu.o client.o server.o config.o shroomposter.o cpp20_http_client.o libcubiomes.a: $(CUBIOMES_SRC) $(CC) -c $(CUBIOMES_SRC) -fwrapv $(CFLAGS) @@ -91,6 +133,16 @@ client.o: src/client.cpp src/client.h src/common.h server.o: src/server.cpp src/server.h src/common.h $(CXX) -c $< -o $@ $(CXXFLAGS) +config.o: src/config.cpp src/config.h + $(CXX) -c $< -o $@ $(CXXFLAGS) + +shroomposter.o: src/shroomposter.cpp src/shroomposter.h src/config.h + $(CXX) -c $< -o $@ $(CXXFLAGS) + +cpp20_http_client.o: src/cpp20_http_client.cpp src/cpp20_http_client.hpp + $(CXX) -c $< -o $@ $(CXXFLAGS) + main: $(MAIN_DEP) $(MAIN_CXX) $(MAIN_SRC) -o $@ $(MAIN_CXXFLAGS) -endif + +endif \ No newline at end of file diff --git a/src/config.cpp b/src/config.cpp new file mode 100644 index 0000000..f67b190 --- /dev/null +++ b/src/config.cpp @@ -0,0 +1,51 @@ +#include "config.h" + +#include +#include + +std::unordered_map config; +std::string version; +std::string shroomin_api_key; + +void loadConfig() +{ + std::ifstream file("config.ini"); + if (!file) + return; + + std::string line; + + while (std::getline(file, line)) + { + if (line.empty()) + continue; + + auto pos = line.find('='); + + if (pos == std::string::npos) + continue; + + std::string key = line.substr(0, pos); + std::string value = line.substr(pos + 1); + + if (key == "version") + { + version = value; + continue; + } + + if (key == "shroomin_api_key") + { + shroomin_api_key = value; + continue; + } + + try + { + config[key] = std::stoll(value); + } + catch (...) + { + } + } +} \ No newline at end of file diff --git a/src/config.h b/src/config.h new file mode 100644 index 0000000..6726eba --- /dev/null +++ b/src/config.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +#include + +extern std::unordered_map config; +extern std::string version; +extern std::string shroomin_api_key; + +void loadConfig(); + +inline long long cfg(const std::string& key, long long defaultValue) +{ + auto it = config.find(key); + return it == config.end() ? defaultValue : it->second; +} \ No newline at end of file diff --git a/src/cpp20_http_client.cpp b/src/cpp20_http_client.cpp new file mode 100644 index 0000000..500d114 --- /dev/null +++ b/src/cpp20_http_client.cpp @@ -0,0 +1,1390 @@ +/* +MIT License + +Copyright (c) 2021-2023 Björn Sundin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#include "cpp20_http_client.hpp" + +//--------------------------------------------------------- + +#include +#include +#include +#include + +using namespace std::chrono_literals; + +//--------------------------------------------------------- + +#ifdef _WIN32 +# ifndef NOMINMAX +# define NOMINMAX +# endif + +// Required by SSPI API headers for some reason. +# ifndef SECURITY_WIN32 +# define SECURITY_WIN32 +# endif + +// Windows socket API. +# include +# include + +// Windows secure channel API. +# include +# include + +// Mingw does not define this. +# ifndef SECBUFFER_ALERT + constexpr auto SECBUFFER_ALERT = 17; +# endif +#elif __has_include() // This header must exist on platforms that conform to the POSIX specifications. +// The POSIX library is available on this platform. +# define IS_POSIX + +# include +# include +# include +# include +# include +# include +# include + +# include +# include + +// Name clash +# ifdef unix +# undef unix +# endif +#endif // __has_include() + +//--------------------------------------------------------- + +namespace http_client { + +// Platform-specific utilities. +namespace utils { + +void enable_utf8_console() { +#ifdef _WIN32 + SetConsoleOutputCP(CP_UTF8); +#endif + // Pretty much everyone else uses utf-8 by default. +} + +#ifdef _WIN32 +namespace win { + +[[nodiscard]] +std::wstring utf8_to_wide(std::string_view const input) { + auto result = std::wstring(MultiByteToWideChar( + CP_UTF8, 0, + input.data(), static_cast(input.size()), + 0, 0 + ), '\0'); + + MultiByteToWideChar( + CP_UTF8, 0, + input.data(), static_cast(input.size()), + result.data(), static_cast(result.size()) + ); + + return result; +} + +void utf8_to_wide(std::string_view const input, std::span const output) { + auto const length = MultiByteToWideChar( + CP_UTF8, 0, + input.data(), static_cast(input.size()), + output.data(), static_cast(output.size()) + ); + + if (length > 0) { + output[length] = 0; + } +} + +[[nodiscard]] +std::string wide_to_utf8(std::wstring_view const input) { + auto result = std::string(WideCharToMultiByte( + CP_UTF8, 0, + input.data(), static_cast(input.size()), + 0, 0, nullptr, nullptr + ), '\0'); + + WideCharToMultiByte( + CP_UTF8, 0, + input.data(), static_cast(input.size()), + result.data(), static_cast(result.size()), + nullptr, nullptr + ); + + return result; +} + +void wide_to_utf8(std::wstring_view const input, std::span const output) { + auto const length = WideCharToMultiByte( + CP_UTF8, 0, + input.data(), static_cast(input.size()), + output.data(), static_cast(output.size()), + nullptr, nullptr + ); + + if (length > 0) { + output[length] = 0; + } +} + +[[nodiscard]] +std::string get_error_message(DWORD const message_id) { + auto buffer = static_cast(nullptr); + + [[maybe_unused]] + auto const buffer_cleanup = Cleanup{[&]{::LocalFree(buffer);}}; + + auto const size = ::FormatMessageW( + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS | + FORMAT_MESSAGE_ALLOCATE_BUFFER, + nullptr, + message_id, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + reinterpret_cast(&buffer), + 1, + nullptr + ); + + return wide_to_utf8(std::wstring_view{buffer, size}); +} + +} // namespace win + +#endif // _WIN32 + +#ifdef IS_POSIX + +namespace unix { + +using UniqueBio = std::unique_ptr; + +[[nodiscard]] +std::string get_openssl_error_string() { + auto const memory_file_handle = UniqueBio{::BIO_new(::BIO_s_mem())}; + ::ERR_print_errors(memory_file_handle.get()); + + auto buffer = static_cast(nullptr); + auto const length = ::BIO_get_mem_data(memory_file_handle.get(), &buffer); + + return std::string(static_cast(buffer), length); +} + +} // namespace unix + +#endif // IS_POSIX + +//--------------------------------------------------------- + +#ifdef _WIN32 + +[[noreturn]] +void throw_connection_error( + std::string reason, + int const error_code = static_cast(GetLastError()), + bool const is_tls_error = false +) { + throw errors::ConnectionFailed{ + std::format("{} with code {}: {}", reason, error_code, win::get_error_message(error_code)), + is_tls_error + }; +} + +#endif // _WIN32 + +#ifdef IS_POSIX + +[[noreturn]] +void throw_connection_error(std::string reason, int const error_code = errno, bool const is_tls_error = false) { + throw errors::ConnectionFailed{ + std::format("{} with code {}: {}", reason, error_code, std::generic_category().message(error_code)), + is_tls_error + }; +} + +#endif // IS_POSIX + +} // namespace utils + +#ifdef _WIN32 + +class WinSockLifetime { +public: + WinSockLifetime() { + auto api_info = ::WSADATA{}; + if (auto const result = ::WSAStartup(MAKEWORD(2, 2), &api_info)) { + utils::throw_connection_error("Failed to initialize Winsock API 2.2", result); + } + } + ~WinSockLifetime() { + if (!is_moved_) { + ::WSACleanup(); + } + } + + WinSockLifetime(WinSockLifetime&& other) noexcept { + other.is_moved_ = true; + } + WinSockLifetime& operator=(WinSockLifetime&& other) noexcept { + other.is_moved_ = true; + is_moved_ = false; + return *this; + } + + WinSockLifetime(WinSockLifetime const&) = delete; + WinSockLifetime& operator=(WinSockLifetime const&) = delete; + +private: + bool is_moved_{false}; +}; + +using SocketHandle = utils::UniqueHandle< + SOCKET, + decltype([](auto const socket) { + if (shutdown(socket, SD_BOTH) == SOCKET_ERROR) { + utils::throw_connection_error("Failed to shut down socket connection", WSAGetLastError()); + } + closesocket(socket); + }), + INVALID_SOCKET +>; + +class RawSocket { +public: + void set_is_nonblocking(bool const p_is_nonblocking) + { + if (is_nonblocking_ != p_is_nonblocking) + { + is_nonblocking_ = p_is_nonblocking; + + auto is_nonblocking = static_cast(p_is_nonblocking); + ioctlsocket(handle_.get(), FIONBIO, &is_nonblocking); + } + } + [[nodiscard]] + SOCKET get_winsock_handle() { + return handle_.get(); + } + + void write(std::span const data) + { + if (is_closed_) { + reconnect_(); + } + + if (::send( + handle_.get(), + reinterpret_cast(data.data()), + static_cast(data.size()), + 0 + ) == SOCKET_ERROR) + { + utils::throw_connection_error("Failed to send data through socket", WSAGetLastError()); + } + } + [[nodiscard]] + auto read(std::span const buffer, bool const is_nonblocking = false) + -> std::variant + { + if (is_closed_) { + return std::size_t{}; + } + + set_is_nonblocking(is_nonblocking); + + if (auto const receive_result = recv( + handle_.get(), + reinterpret_cast(buffer.data()), + static_cast(buffer.size()), + 0 + ); receive_result >= 0) + { + if (receive_result == 0) { + is_closed_ = true; + return ConnectionClosed{}; + } + return static_cast(receive_result); + } + else if (is_nonblocking && WSAGetLastError() == WSAEWOULDBLOCK) { + return std::size_t{}; + } + else utils::throw_connection_error("Failed to receive data through socket"); + } + [[nodiscard]] + auto read_available(std::span const buffer) + -> std::variant + { + return read(buffer, true); + } + + RawSocket(std::string_view const server, Port const port) : + address_info_{get_address_info_(server, port)}, + handle_{create_handle_()} + {} + +private: + using AddressInfo = std::unique_ptr; + + [[nodiscard]] + static AddressInfo get_address_info_(std::string_view const server, Port const port) { + auto const wide_server_name = utils::win::utf8_to_wide(server); + auto const wide_port_string = std::to_wstring(port); + auto const hints = addrinfoW{ + .ai_family = AF_UNSPEC, + .ai_socktype = SOCK_STREAM, + .ai_protocol = IPPROTO_TCP, + }; + auto address_info = static_cast(nullptr); + + if (auto const result = GetAddrInfoW( + wide_server_name.data(), + wide_port_string.data(), + &hints, + &address_info + )) + { + throw errors::ConnectionFailed{ + std::format("Failed to get address info for socket creation: {}", utils::win::get_error_message(result)) + }; + } + + return AddressInfo{address_info}; + } + + [[nodiscard]] + SocketHandle create_handle_() const { + auto const handle_error = [](auto const error_message) { + if (auto const error_code = WSAGetLastError(); error_code != WSAEINPROGRESS) { + utils::throw_connection_error(error_message, error_code); + } + constexpr auto time_to_wait_between_attempts = 1ms; + std::this_thread::sleep_for(time_to_wait_between_attempts); + }; + + auto socket_handle = SocketHandle{}; + while ((socket_handle = socket( + address_info_->ai_family, + address_info_->ai_socktype, + address_info_->ai_protocol + )).get() == INVALID_SOCKET) + { + handle_error("Failed to create socket"); + } + + while (connect( + socket_handle.get(), + address_info_->ai_addr, + static_cast(address_info_->ai_addrlen) + ) == SOCKET_ERROR) + { + handle_error("Failed to connect socket"); + } + + return socket_handle; + } + + void reconnect_() { + handle_ = create_handle_(); + is_closed_ = false; + } + + WinSockLifetime api_lifetime_; + AddressInfo address_info_; + SocketHandle handle_; + bool is_nonblocking_{false}; + bool is_closed_{false}; +}; + +using DllHandle = utils::UniqueHandle; + +struct SspiLibrary { + DllHandle dll_handle; + + PSecurityFunctionTableW functions; + + SspiLibrary() : + dll_handle{LoadLibraryW(L"secur32.dll")} + { + auto const throw_error = []{ + throw std::system_error{static_cast(GetLastError()), std::system_category(), "Failed to initialize the SSPI library"}; + }; + + if (!dll_handle) { + throw_error(); + } + + // ew :) + auto const init_security_interface = reinterpret_cast( + reinterpret_cast(GetProcAddress(dll_handle.get(), "InitSecurityInterfaceW")) + ); + + functions = init_security_interface(); + if (!functions) { + throw_error(); + } + } + ~SspiLibrary() = default; + + SspiLibrary(SspiLibrary const&) = delete; + SspiLibrary& operator=(SspiLibrary const&) = delete; + SspiLibrary(SspiLibrary&&) = delete; + SspiLibrary& operator=(SspiLibrary&&) = delete; +}; + +auto const sspi_library = SspiLibrary{}; + +[[nodiscard]] +constexpr bool operator==(CredHandle const& first, CredHandle const& second) noexcept { + return first.dwLower == second.dwLower && first.dwUpper == second.dwUpper; +} +[[nodiscard]] +constexpr bool operator!=(CredHandle const& first, CredHandle const& second) noexcept { + return !(first == second); +} + +[[nodiscard]] +constexpr bool operator==(SecBuffer const& first, SecBuffer const& second) noexcept { + return first.pvBuffer == second.pvBuffer; +} +[[nodiscard]] +constexpr bool operator!=(SecBuffer const& first, SecBuffer const& second) noexcept { + return !(first == second); +} + +using SecurityContextHandle = utils::UniqueHandleDeleteSecurityContext(&h); })>; + +SecBufferDesc create_single_schannel_buffer_description(SecBuffer& buffer) { + return { + .ulVersion = SECBUFFER_VERSION, + .cBuffers = 1ul, + .pBuffers = &buffer, + }; +} +SecBufferDesc create_schannel_buffers_description(std::span const buffers) { + return { + .ulVersion = SECBUFFER_VERSION, + .cBuffers = static_cast(buffers.size()), + .pBuffers = buffers.data(), + }; +} + +/* + Holds either received handshake data or TLS message data. + The required size of the TLS handshake message buffer cannot be retrieved + through any API call. See the block comment in the class. + After the handshake is complete, the buffer is resized because then the TLS message + header, trailer and message sizes can be retrieved using QueryContextAttributesW. +*/ +class TlsMessageReceiveBuffer { +public: + std::span extra_data; + + using iterator = utils::DataVector::iterator; + + [[nodiscard]] + iterator begin() { + return buffer_.begin(); + } + [[nodiscard]] + iterator end() { + return buffer_.end(); + } + + void grow_to_size(std::size_t const new_size) { + assert(new_size >= buffer_.size()); + + if (!extra_data.empty()) { + auto const extra_data_start = extra_data.data() - buffer_.data(); + assert(extra_data_start > 0 && extra_data_start < static_cast(buffer_.size())); + buffer_.resize(new_size); + extra_data = std::span{buffer_}.subspan(extra_data_start, extra_data.size()); + } + else { + buffer_.resize(new_size); + } + } + [[nodiscard]] + std::span get_full_buffer() { + return buffer_; + } + + [[nodiscard]] + static TlsMessageReceiveBuffer allocate_new() { + return TlsMessageReceiveBuffer{maximum_handshake_message_size}; + } + + TlsMessageReceiveBuffer() = default; + ~TlsMessageReceiveBuffer() = default; + + TlsMessageReceiveBuffer(TlsMessageReceiveBuffer const&) = delete; + TlsMessageReceiveBuffer& operator=(TlsMessageReceiveBuffer const&) = delete; + TlsMessageReceiveBuffer(TlsMessageReceiveBuffer&&) noexcept = default; + TlsMessageReceiveBuffer& operator=(TlsMessageReceiveBuffer&&) noexcept = default; + +private: + /* + When the buffer is too small to fit the whole handshake message received from the peer, the return + code from InitializeSecurityContextW is not SEC_E_INCOMPLETE_MESSAGE, but SEC_E_INVALID_TOKEN. + Trying to grow the buffer after getting that return code does not work. The server closes the + connection when trying to read more data afterwards. It seems that we need a fixed maximum + handshake message/token size. + + It is not clear exactly what this maximum size should be. + The only thing Microsoft's documentation says about this is + "[...] the value of this parameter is a pointer to a + buffer allocated with enough memory to hold the + token returned by the remote computer." + (https://docs.microsoft.com/en-us/windows/win32/api/sspi/nf-sspi-initializesecuritycontextw) + The TLS 1.3 standard specification says: + "The record layer fragments information blocks into TLSPlaintext + records carrying data in chunks of 2^14 bytes or less." + (https://tools.ietf.org/html/rfc8446) + + Looking at a few implementations of TLS sockets using Schannel: + 1. https://github.com/adobe/chromium/blob/master/net/socket/ssl_client_socket_win.cc + Uses 5 + 16*1024 + 64 = 16453 bytes. + 2. https://github.com/curl/curl/blob/master/lib/vtls/schannel.c + Uses 4096 + 1024 = 5120 bytes. + 3. https://github.com/odzhan/shells/tree/master/s6 + Uses 32768 bytes. + 4. https://docs.microsoft.com/en-us/windows/win32/secauthn/using-sspi-with-a-windows-sockets-client + Uses 12000 bytes. + + ALL of these implementations use DIFFERENT maximum handshake message sizes. + I decided to follow the TLS specification and use 2^14 bytes for the handshake message buffer, + as this should be the maximum allowed size of any TLSPlaintext record block, which includes handshake messages. + */ + static constexpr auto maximum_handshake_message_size = std::size_t{1 << 14}; + + utils::DataVector buffer_; + + explicit TlsMessageReceiveBuffer(std::size_t const size) : + buffer_(size) + {} +}; + +class SchannelConnectionInitializer { +public: + /* + Returns the resulting security context and a vector of any extra non-handshake data + that should be processed as part of the next message. + */ + std::pair operator()() && { + do_handshake_(); + return {std::move(security_context_), std::move(receive_buffer_)}; + } + + [[nodiscard]] + SchannelConnectionInitializer(RawSocket* const socket, std::string_view const server) : + socket_{socket}, + server_name_{utils::win::utf8_to_wide(server)} + {} + ~SchannelConnectionInitializer() = default; + + SchannelConnectionInitializer(SchannelConnectionInitializer&&) noexcept = delete; + SchannelConnectionInitializer& operator=(SchannelConnectionInitializer&&) noexcept = delete; + SchannelConnectionInitializer(SchannelConnectionInitializer const&) = delete; + SchannelConnectionInitializer& operator=(SchannelConnectionInitializer const&) = delete; + +private: + using CredentialsHandle = utils::UniqueHandleFreeCredentialHandle(&h); })>; + + using HandshakeOutputBuffer = utils::UniqueHandle< + SecBuffer, decltype([](auto const& buffer) { + if (buffer.pvBuffer) { + sspi_library.functions->FreeContextBuffer(buffer.pvBuffer); + } + }) + >; + + void do_handshake_() { + if (auto const [return_code, output_buffer] = process_handshake_data_({}); + return_code != SEC_I_CONTINUE_NEEDED) // First call should always yield this return code. + { + utils::throw_connection_error("Schannel TLS handshake initialization failed", return_code, true); + } + else send_handshake_message_(output_buffer); + + auto offset = std::size_t{}; + while (true) { + auto const read_span = read_response_(offset); + if (auto const [return_code, output_buffer] = process_handshake_data_(read_span); + return_code == SEC_I_CONTINUE_NEEDED) + { + if (output_buffer->cbBuffer) { + send_handshake_message_(output_buffer); + } + offset = 0; + } + else if (return_code == SEC_E_INCOMPLETE_MESSAGE) { + offset = read_span.size(); + } + else if (return_code == SEC_E_OK) { + return; + } + else { + utils::throw_connection_error("Schannel TLS handshake failed", return_code); + } + } + } + + /* + Returns a span over the total read data. + */ + std::span read_response_(std::size_t const offset = {}) { + auto const buffer_span = receive_buffer_.get_full_buffer(); + + if (!receive_buffer_.extra_data.empty()) { + assert(offset == 0); + + auto const extra_data_size = receive_buffer_.extra_data.size(); + std::ranges::copy_backward(receive_buffer_.extra_data, buffer_span.begin() + extra_data_size); + + receive_buffer_.extra_data = {}; + return buffer_span.first(extra_data_size); + } + else if (auto const read_result = socket_->read(buffer_span.subspan(offset)); + std::holds_alternative(read_result)) + { + throw errors::ConnectionFailed{"The connection closed unexpectedly while reading handshake data.", true}; + } + else { + return buffer_span.subspan(0, offset + std::get(read_result)); + } + } + + struct [[nodiscard]] HandshakeProcessResult { + SECURITY_STATUS status_code; + HandshakeOutputBuffer output_buffer; + }; + HandshakeProcessResult process_handshake_data_(std::span const input_buffer) { + constexpr auto request_flags = ISC_REQ_SEQUENCE_DETECT | ISC_REQ_REPLAY_DETECT | + ISC_REQ_CONFIDENTIALITY | ISC_REQ_ALLOCATE_MEMORY | ISC_REQ_STREAM; + + // The second input buffer is used to indicate that extra data + // from the next message was at the end of the input buffer, + // and should be processed in the next call. There's not actually + // any buffer pointer in that SecBuffer. + auto input_buffers = std::array{ + SecBuffer{ + .cbBuffer = static_cast(input_buffer.size()), + .BufferType = SECBUFFER_TOKEN, + .pvBuffer = input_buffer.data(), + }, + SecBuffer{}, + }; + auto input_buffer_description = create_schannel_buffers_description(input_buffers); + + auto output_buffers = std::array{ + SecBuffer{.BufferType = SECBUFFER_TOKEN}, + SecBuffer{.BufferType = SECBUFFER_ALERT}, + SecBuffer{}, + }; + auto output_buffer_description = create_schannel_buffers_description(output_buffers); + + unsigned long returned_flags; + + auto const return_code = sspi_library.functions->InitializeSecurityContextW( + &credentials_.get(), + security_context_ ? &security_context_ : nullptr, // Null on first call, input security context handle + server_name_.data(), + request_flags, + 0, // Reserved + 0, // Not used with Schannel + input_buffer.empty() ? nullptr : &input_buffer_description, // Null on first call + 0, // Reserved + &security_context_, // Output security context handle + &output_buffer_description, + &returned_flags, + nullptr // Don't care about expiration date right now + ); + + if (returned_flags != request_flags) { + utils::throw_connection_error("The schannel security context flags were not supported"); + } + + return HandshakeProcessResult{[&]{ + if (input_buffers[1].BufferType == SECBUFFER_EXTRA) { + receive_buffer_.extra_data = input_buffer.last(input_buffers[1].cbBuffer); + } + + if (return_code == SEC_I_COMPLETE_AND_CONTINUE || return_code == SEC_I_COMPLETE_NEEDED) { + sspi_library.functions->CompleteAuthToken(&security_context_, &output_buffer_description); + + if (return_code == SEC_I_COMPLETE_AND_CONTINUE) { + return SEC_I_CONTINUE_NEEDED; + } + return SEC_E_OK; + } + return return_code; + }(), HandshakeOutputBuffer{output_buffers[0]}}; + } + void send_handshake_message_(HandshakeOutputBuffer const& message_buffer) { + socket_->write(std::span{ + static_cast(message_buffer->pvBuffer), + static_cast(message_buffer->cbBuffer) + }); + } + + [[nodiscard]] + static CredentialsHandle acquire_credentials_handle_() { + auto credentials_data = SCHANNEL_CRED{ + .dwVersion = SCHANNEL_CRED_VERSION, + }; + CredHandle credentials_handle; + TimeStamp credentials_time_limit; + + auto const security_status = sspi_library.functions->AcquireCredentialsHandleW( + nullptr, + const_cast(UNISP_NAME_W), + SECPKG_CRED_OUTBOUND, + nullptr, + &credentials_data, + nullptr, + nullptr, + &credentials_handle, + &credentials_time_limit + ); + if (security_status != SEC_E_OK) { + utils::throw_connection_error("Failed to acquire credentials", security_status, true); + } + + return CredentialsHandle{credentials_handle}; + } + + CredentialsHandle credentials_{acquire_credentials_handle_()}; + RawSocket* socket_; + std::wstring server_name_; + + SecurityContextHandle security_context_; + TlsMessageReceiveBuffer receive_buffer_{TlsMessageReceiveBuffer::allocate_new()}; +}; + +class TlsSocket { +public: + void write(std::span data) { + while (!data.empty()) { + auto const message_length = std::min(data.size(), static_cast(stream_sizes_.cbMaximumMessage)); + + auto const output_buffer = encrypt_message_(data.first(message_length)); + raw_socket_->write(output_buffer); + + data = data.subspan(message_length); + } + } + + [[nodiscard]] + auto read(std::span const buffer, bool const is_nonblocking = false) + -> std::variant + { + if (decrypted_message_left_.empty()) { + auto const receive_buffer_span = receive_buffer_.get_full_buffer(); + auto read_offset = std::size_t{}; + + while (true) { + if (auto const read_result = read_encrypted_data_(read_offset, is_nonblocking); + std::holds_alternative(read_result)) + { + return read_result; + } + else if (decrypt_message_(receive_buffer_span.first(read_offset + std::get(read_result))) + || is_nonblocking) + { + break; + } + else { + read_offset += std::get(read_result); + } + } + } + if (decrypted_message_left_.empty()) { + return std::size_t{}; + } + + auto const size = std::min(decrypted_message_left_.size(), buffer.size()); + std::ranges::copy(decrypted_message_left_.first(size), buffer.begin()); + decrypted_message_left_ = decrypted_message_left_.subspan(size); + + return size; + } + [[nodiscard]] + auto read_available(std::span const buffer) + -> std::variant + { + return read(buffer, true); + } + + TlsSocket(std::string_view const server, Port const port) + { + initialize_connection_(server, port); + } + +private: + void initialize_connection_(std::string_view const server, Port const port) { + if (raw_socket_) { + return; + } + + raw_socket_ = std::make_unique(server, port); + + std::tie(security_context_, receive_buffer_) = SchannelConnectionInitializer{raw_socket_.get(), server}(); + initialize_stream_sizes_(); + } + + void initialize_stream_sizes_() { + if (auto const result = sspi_library.functions->QueryContextAttributesW(&security_context_, SECPKG_ATTR_STREAM_SIZES, &stream_sizes_); + result != SEC_E_OK) + { + utils::throw_connection_error("Failed to query Schannel security context stream sizes", result, true); + } + receive_buffer_.grow_to_size(stream_sizes_.cbHeader + stream_sizes_.cbMaximumMessage + stream_sizes_.cbTrailer); + } + + [[nodiscard]] + utils::DataVector encrypt_message_(std::span const data) { + // https://docs.microsoft.com/en-us/windows/win32/api/sspi/nf-sspi-encryptmessage + + auto full_buffer = utils::DataVector(stream_sizes_.cbHeader + data.size() + stream_sizes_.cbTrailer); + std::ranges::copy(data, full_buffer.begin() + stream_sizes_.cbHeader); + + auto buffers = std::array{ + SecBuffer{ + .cbBuffer = stream_sizes_.cbHeader, + .BufferType = SECBUFFER_STREAM_HEADER, + .pvBuffer = full_buffer.data(), + }, + SecBuffer{ + .cbBuffer = static_cast(data.size()), + .BufferType = SECBUFFER_DATA, + .pvBuffer = full_buffer.data() + stream_sizes_.cbHeader, + }, + SecBuffer{ + .cbBuffer = stream_sizes_.cbTrailer, + .BufferType = SECBUFFER_STREAM_TRAILER, + .pvBuffer = full_buffer.data() + stream_sizes_.cbHeader + data.size(), + }, + // Empty buffer that must be supplied at the end. + SecBuffer{}, + }; + + auto buffers_description = create_schannel_buffers_description(buffers); + + if (auto const result = sspi_library.functions->EncryptMessage(&security_context_, 0, &buffers_description, 0); + result != SEC_E_OK) + { + utils::throw_connection_error("Failed to encrypt TLS message", result, true); + } + + return full_buffer; + } + + [[nodiscard]] + auto read_encrypted_data_(std::size_t const offset, bool const is_nonblocking) + -> std::variant + { + auto const buffer_span = receive_buffer_.get_full_buffer(); + + if (!receive_buffer_.extra_data.empty()) { + assert(offset == 0); + + auto const extra_data_size = receive_buffer_.extra_data.size(); + std::ranges::copy_backward(receive_buffer_.extra_data, buffer_span.begin() + extra_data_size); + + receive_buffer_.extra_data = {}; + return extra_data_size; + } + else { + return raw_socket_->read(buffer_span.subspan(offset), is_nonblocking); + } + } + + // Returns false if the encrypted message was incomplete + [[nodiscard]] + bool decrypt_message_(std::span const message) { + // https://docs.microsoft.com/en-us/windows/win32/secauthn/stream-contexts + auto buffers = std::array{ + SecBuffer{ // This will hold the message header afterwards + .cbBuffer = static_cast(message.size()), + .BufferType = SECBUFFER_DATA, + .pvBuffer = message.data(), + }, + SecBuffer{}, // Will hold the decrypted data + SecBuffer{}, // Will hold the message trailer + SecBuffer{}, // May hold size of extra undecrypted data (from the next message) + }; + auto message_buffer_description = create_schannel_buffers_description(buffers); + + if (auto const status_code = sspi_library.functions->DecryptMessage(&security_context_, &message_buffer_description, 0, nullptr); + status_code == SEC_E_OK) + { + decrypted_message_left_ = {static_cast(buffers[1].pvBuffer), static_cast(buffers[1].cbBuffer)}; + + // https://docs.microsoft.com/en-us/windows/win32/secauthn/extra-buffers-returned-by-schannel + // Data from the next message. Always at the end. + if (buffers[3].BufferType == SECBUFFER_EXTRA) { + receive_buffer_.extra_data = message.last(buffers[3].cbBuffer); + } + return true; + } + else if (status_code == SEC_E_INCOMPLETE_MESSAGE) { + return false; + } + else { + utils::throw_connection_error("Failed to decrypt a received TLS message", status_code, true); + } + } + + std::unique_ptr raw_socket_; + + SecurityContextHandle security_context_; + + SecPkgContext_StreamSizes stream_sizes_; + + TlsMessageReceiveBuffer receive_buffer_; + + // This is the part of the message buffer that contains the + // rest of the decrypted message data that has not been read yet. + std::span decrypted_message_left_; +}; + +#endif // _WIN32 + +#ifdef IS_POSIX + +using PosixSocketHandle = int; + +using SocketHandle = utils::UniqueHandle< + PosixSocketHandle, + decltype([](auto const handle) { + if (::shutdown(handle, SHUT_RDWR) == -1) { + utils::throw_connection_error("Failed to shut down socket connection"); + } + ::close(handle); + }), + PosixSocketHandle{-1} +>; + +class RawSocket { +public: + void make_nonblocking() { + if (!is_nonblocking_) { + auto const flags = ::fcntl(handle_.get(), F_GETFL); + if (-1 == ::fcntl(handle_.get(), F_SETFL, flags | O_NONBLOCK)) { + utils::throw_connection_error("Failed to turn on nonblocking mode on socket"); + } + is_nonblocking_ = true; + } + } + void make_blocking() { + if (is_nonblocking_) { + auto const flags = ::fcntl(handle_.get(), F_GETFL); + if (-1 == ::fcntl(handle_.get(), F_SETFL, flags & ~O_NONBLOCK)) { + utils::throw_connection_error("Failed to turn off nonblocking mode on socket"); + } + is_nonblocking_ = false; + } + } + [[nodiscard]] + PosixSocketHandle get_posix_handle() const { + return handle_.get(); + } + + void reconnect_() { + handle_ = create_handle_(); + is_closed_ = false; + } + + void write(std::span const data) { + if (is_closed_) { + reconnect_(); + } + + if (::send( + handle_.get(), + data.data(), + static_cast(data.size()), + 0 + ) == -1) + { + utils::throw_connection_error("Failed to send data through socket"); + } + } + [[nodiscard]] + auto read(std::span const buffer, bool is_nonblocking = false) + -> std::variant + { + if (is_closed_) { + return std::size_t{}; + } + + if (auto const receive_result = ::recv( + handle_.get(), + reinterpret_cast(buffer.data()), + static_cast(buffer.size()), + is_nonblocking ? MSG_DONTWAIT : 0 + ); receive_result >= 0) + { + if (receive_result == 0) { + is_closed_ = true; + return ConnectionClosed{}; + } + return static_cast(receive_result); + } + else if (is_nonblocking && (errno == EWOULDBLOCK || errno == EAGAIN)) { + return std::size_t{}; + } + utils::throw_connection_error("Failed to receive data through socket"); + } + [[nodiscard]] + auto read_available(std::span const buffer) + -> std::variant + { + return read(buffer, true); + } + + RawSocket(std::string_view const server, Port const port) : + address_info_{get_address_info_(std::string{server}, port)}, + handle_{create_handle_()} + {} + +private: + using AddressInfo = std::unique_ptr; + + [[nodiscard]] + static AddressInfo get_address_info_(std::string const server, Port const port) { + auto const port_string = std::to_string(port); + auto const hints = addrinfo{ + .ai_flags{}, + .ai_family{AF_UNSPEC}, + .ai_socktype{SOCK_STREAM}, + .ai_protocol{IPPROTO_TCP}, + .ai_addrlen{}, + .ai_addr{}, + .ai_canonname{}, + .ai_next{} + }; + auto address_info = static_cast(nullptr); + + if (auto const result = ::getaddrinfo( + reinterpret_cast(server.data()), + port_string.data(), + &hints, + &address_info + )) + { + throw errors::ConnectionFailed{ + std::format("Failed to get address info for socket creation: {}", gai_strerror(result)) + }; + } + + return AddressInfo{address_info}; + } + + [[nodiscard]] + SocketHandle create_handle_() const { + auto socket_handle = SocketHandle{::socket( + address_info_->ai_family, + address_info_->ai_socktype, + address_info_->ai_protocol + )}; + if (!socket_handle) { + utils::throw_connection_error("Failed to create socket"); + } + + while (::connect( + socket_handle.get(), + address_info_->ai_addr, + static_cast(address_info_->ai_addrlen) + ) == -1) + { + if (auto const error_code = errno; error_code != EINPROGRESS) { + utils::throw_connection_error("Failed to connect socket", error_code); + } + constexpr auto time_to_wait_between_attempts = 1ms; + std::this_thread::sleep_for(time_to_wait_between_attempts); + } + + return socket_handle; + } + + AddressInfo address_info_; + + SocketHandle handle_; + + bool is_nonblocking_{false}; + + bool is_closed_{false}; +}; + +class TlsSocket { +public: + void write(std::span const data) { + ensure_connected_(); + + if (::SSL_write( + tls_connection_.get(), + data.data(), + static_cast(data.size()) + ) == -1) + { + utils::throw_connection_error("Failed to send data through socket"); + } + } + [[nodiscard]] + auto read(std::span const buffer) + -> std::variant + { + if (is_closed_) { + return std::size_t{}; + } + + raw_socket_->make_blocking(); + if (auto const read_result = ::SSL_read( + tls_connection_.get(), + buffer.data(), + static_cast(buffer.size()) + ); read_result >= 0) + { + if (read_result == 0) { + is_closed_ = true; + return ConnectionClosed{}; + } + return static_cast(read_result); + } + utils::throw_connection_error("Failed to receive data from socket"); + } + [[nodiscard]] + auto read_available(std::span const buffer) + -> std::variant + { + if (is_closed_) { + return std::size_t{}; + } + + raw_socket_->make_nonblocking(); + if (auto const read_result = ::SSL_read( + tls_connection_.get(), + buffer.data(), + static_cast(buffer.size()) + ); read_result > 0) + { + return static_cast(read_result); + } + else switch (auto const error_code = ::SSL_get_error(tls_connection_.get(), read_result)) { + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + // No available data to read at the moment. + return std::size_t{}; + case SSL_ERROR_ZERO_RETURN: + case SSL_ERROR_SYSCALL: + if (errno == 0) { + is_closed_ = true; + // Peer shut down the connection. + return ConnectionClosed{}; + } + [[fallthrough]]; + default: + utils::throw_connection_error("Failed to read available data from socket", error_code); + } + utils::unreachable(); + } + + TlsSocket(std::string_view const server, Port const port) { + initialize_connection_(server, port); + } + +private: + using TlsContext = std::unique_ptr; + using TlsConnection = std::unique_ptr; + + static void throw_tls_error_() { + throw errors::ConnectionFailed{utils::unix::get_openssl_error_string(), true}; + } + + void ensure_connected_() { + if (is_closed_) { + raw_socket_->reconnect_(); + update_tls_socket_handle_(); + } + } + + void initialize_connection_(std::string_view const server, Port const port) { + if (raw_socket_) { + return; + } + + configure_tls_context_(); + configure_tls_connection_(std::string{server}, port); + connect_(); + } + + void configure_tls_context_() { + if (1 != SSL_CTX_set_default_verify_paths(tls_context_.get())) { + throw_tls_error_(); + } + SSL_CTX_set_read_ahead(tls_context_.get(), true); + } + + void configure_tls_connection_(std::string const server, Port const port) { + auto const host_name_c_string = server.data(); + + // For SNI (Server Name Identification) + // The macro casts the string to a void* for some reason. Ew. + // The casts are to suppress warnings about it. + if (1 != SSL_set_tlsext_host_name(tls_connection_.get(), reinterpret_cast(const_cast(host_name_c_string)))) { + throw_tls_error_(); + } + // Configure automatic hostname check + if (1 != SSL_set1_host(tls_connection_.get(), host_name_c_string)) { + throw_tls_error_(); + } + + // Set the socket to be used by the tls connection + raw_socket_ = std::make_unique(server, port); + update_tls_socket_handle_(); + } + + void update_tls_socket_handle_() { + if (1 != SSL_set_fd(tls_connection_.get(), raw_socket_->get_posix_handle())) { + throw_tls_error_(); + } + } + + void connect_() { + SSL_connect(tls_connection_.get()); + + // Just to check that a certificate was presented by the server + if (auto const certificate = SSL_get_peer_certificate(tls_connection_.get())) { + X509_free(certificate); + } + else throw_tls_error_(); + + // Get result of the certificate verification + auto const verify_result = SSL_get_verify_result(tls_connection_.get()); + if (X509_V_OK != verify_result) { + throw_tls_error_(); + } + } + + std::unique_ptr raw_socket_; + bool is_closed_{false}; + + TlsContext tls_context_ = []{ + if (auto const method = TLS_client_method()) { + if (auto const tls = SSL_CTX_new(method)) { + return TlsContext{tls}; + } + } + throw_tls_error_(); + return TlsContext{}; + }(); + + TlsConnection tls_connection_ = [this]{ + if (auto const tls_connection = SSL_new(tls_context_.get())) { + return TlsConnection{tls_connection}; + } + throw_tls_error_(); + return TlsConnection{}; + }(); +}; +#endif // IS_POSIX + +class Socket::Implementation { +public: + void write(std::span const buffer) { + if (std::holds_alternative(socket_)) { + std::get(socket_).write(buffer); + } + else std::get(socket_).write(buffer); + } + [[nodiscard]] + auto read(std::span const buffer) + -> std::variant + { + if (std::holds_alternative(socket_)) { + return std::get(socket_).read(buffer); + } + return std::get(socket_).read(buffer); + } + [[nodiscard]] + auto read_available(std::span const buffer) + -> std::variant + { + if (std::holds_alternative(socket_)) { + return std::get(socket_).read_available(buffer); + } + return std::get(socket_).read_available(buffer); + } + + Implementation(std::string_view const server, Port const port, bool const is_tls_encrypted) : + socket_{select_socket_(server, port, is_tls_encrypted)} + {} + +private: + using SocketVariant = std::variant; + + [[nodiscard]] + static SocketVariant select_socket_(std::string_view const server, Port const port, bool const is_tls_encrypted) + { + if (is_tls_encrypted) { + return TlsSocket{server, port}; + } + return RawSocket{server, port}; + } + + SocketVariant socket_; +}; + +void Socket::write(std::span data) const { + implementation_->write(data); +} + +auto Socket::read(std::span buffer) const + -> std::variant +{ + return implementation_->read(buffer); +} + +auto Socket::read_available(std::span buffer) const + -> std::variant +{ + return implementation_->read_available(buffer); +} + +Socket::Socket(std::string_view const server, Port const port, bool const is_tls_encrypted) : + implementation_{std::make_unique(server, port, is_tls_encrypted)} +{} +Socket::~Socket() = default; + +Socket::Socket(Socket&&) noexcept = default; +Socket& Socket::operator=(Socket&&) noexcept = default; + +} // namespace http_client diff --git a/src/cpp20_http_client.hpp b/src/cpp20_http_client.hpp new file mode 100644 index 0000000..68ba810 --- /dev/null +++ b/src/cpp20_http_client.hpp @@ -0,0 +1,1998 @@ +/* +MIT License + +Copyright (c) 2021-2023 Björn Sundin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#ifdef __cpp_lib_source_location +# include +#endif + +/* +Namespaces: + +http_client { + utils + errors + algorithms +} +*/ + +namespace http_client { + +using Port = int; + +/* + An enumeration of the transfer protocols that are supported by the library. +*/ +enum class Protocol : Port { + Http = 80, + Https = 443, + Unknown = -1, +}; + +/* + This is everything that doesn't have anything to do with the core functionality, + but are utilities that are used within the library. +*/ +namespace utils { + +/* + IsAnyOf is true if T is the same type as one or more of U, V, W, ... +*/ +template +concept IsAnyOf = (std::same_as || ...); + +//--------------------------------------------------------- + +template +concept IsTrivial = std::is_trivial_v; + +/* + Aliasing with types for which IsByte is true is allowed and does not invoke undefined behavior. + https://en.cppreference.com/w/cpp/language/reinterpret_cast +*/ +template +concept IsByte = IsAnyOf, std::byte, char, unsigned char>; + +//--------------------------------------------------------- + +/* + Used to invoke a lambda at the end of a scope. +*/ +template +class [[nodiscard]] Cleanup { +public: + [[nodiscard]] + Cleanup(T&& callable) : + callable_{std::forward(callable)} + {} + + Cleanup() = delete; + ~Cleanup() { + callable_(); + } + + Cleanup(Cleanup&&) noexcept = delete; + Cleanup& operator=(Cleanup&&) noexcept = delete; + + Cleanup(Cleanup const&) = delete; + Cleanup& operator=(Cleanup const&) = delete; + +private: + T callable_; +}; + +//--------------------------------------------------------- + +/* + Similar to std::unique_ptr except that non-pointer types can be held + and that a custom deleter must be specified. + + This is useful for OS handles that are integer types, for example a native socket handle. + Use C++20 lambdas in unevaluated contexts to specify a deleter, or use an already defined + functor type. + + Example: + using DllHandle = utils::UniqueHandle; +*/ +template Deleter_, T invalid_handle = T{}> +class UniqueHandle { +public: + [[nodiscard]] + constexpr explicit operator T() const noexcept { + return handle_; + } + [[nodiscard]] + constexpr T get() const noexcept { + return handle_; + } + [[nodiscard]] + constexpr T& get() noexcept { + return handle_; + } + + [[nodiscard]] + constexpr T const* operator->() const noexcept { + return &handle_; + } + [[nodiscard]] + constexpr T* operator->() noexcept { + return &handle_; + } + + [[nodiscard]] + constexpr T const* operator&() const noexcept { + return &handle_; + } + [[nodiscard]] + constexpr T* operator&() noexcept { + return &handle_; + } + + [[nodiscard]] + constexpr explicit operator bool() const noexcept { + return handle_ != invalid_handle; + } + [[nodiscard]] + constexpr bool operator!() const noexcept { + return handle_ == invalid_handle; + } + + [[nodiscard]] + constexpr bool operator==(UniqueHandle const&) const noexcept + requires std::equality_comparable + = default; + + constexpr explicit UniqueHandle(T const handle) noexcept : + handle_{handle} + {} + constexpr UniqueHandle& operator=(T const handle) { + close_(); + handle_ = handle; + return *this; + } + + constexpr UniqueHandle() = default; + constexpr ~UniqueHandle() { + close_(); + } + + constexpr UniqueHandle(UniqueHandle&& handle) noexcept : + handle_{handle.handle_} + { + handle.handle_ = invalid_handle; + } + constexpr UniqueHandle& operator=(UniqueHandle&& handle) noexcept { + handle_ = handle.handle_; + handle.handle_ = invalid_handle; + return *this; + } + + constexpr UniqueHandle(UniqueHandle const&) = delete; + constexpr UniqueHandle& operator=(UniqueHandle const&) = delete; + +private: + T handle_{invalid_handle}; + + constexpr void close_() { + if (handle_ != invalid_handle) { + Deleter_{}(handle_); + handle_ = invalid_handle; + } + } +}; + +//--------------------------------------------------------- + +/* + This can be called when the program reaches a path that should never be reachable. + It prints error output and exits the program. +*/ +#ifdef __cpp_lib_source_location +[[noreturn]] +inline void unreachable(std::source_location const& source_location = std::source_location::current()) { + std::cerr << std::format("Reached an unreachable code path in file {}, in function {}, on line {}.\n", + source_location.file_name(), source_location.function_name(), source_location.line()); + std::exit(1); +} +#else +[[noreturn]] +inline void unreachable() { + std::cerr << "Reached an unreachable code path, exiting.\n"; + std::exit(1); +} +#endif + +/* + Prints an error message to the error output stream and exits the program. +*/ +[[noreturn]] +inline void panic(std::string_view const message) { + std::cerr << message << '\n'; + std::exit(1); +} + +//--------------------------------------------------------- + +template +concept IsInputRangeOf = std::ranges::input_range && std::same_as, Value_>; + +template +concept IsSizedRangeOf = IsInputRangeOf && std::ranges::sized_range; + +/* + Converts a range of contiguous characters to a std::basic_string_view. + + TODO: Remove this in C++23; std::views::split will return contiguous ranges and std::basic_string_view will have a range constructor. +*/ +constexpr auto range_to_string_view = []< + /* + std::views::split returns a range of ranges. + The ranges unfortunately are not std::ranges::contiguous_range + even when the base type is contiguous, so we can't use that constraint. + + This will be fixed :^D + http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p2210r2.html + */ + IsInputRangeOf Range_ +> (Range_&& range) { + return std::string_view{ + &*std::ranges::begin(range), + static_cast(std::ranges::distance(range)) + }; +}; + +//--------------------------------------------------------- + +void enable_utf8_console(); + +//--------------------------------------------------------- + +/* + Copies a sized range to a std::basic_string of any type. +*/ +template Range_> +[[nodiscard]] +inline std::string range_to_string(Range_ const& range) { + auto result = std::string(range.size(), char{}); + std::ranges::copy(range, std::ranges::begin(result)); + return result; +} + +/* + Copies a range of unknown size to a std::basic_string of any type. +*/ +template Range_> +[[nodiscard]] +inline std::string range_to_string(Range_ const& range) { + auto result = std::string(); + std::ranges::copy(range, std::back_inserter(result)); + return result; +} + +/* + Reinterprets a span of any byte-sized trivial type as a string view of a specified byte-sized character type. +*/ +template +[[nodiscard]] +std::string_view data_to_string(std::span const data) { + return std::string_view{reinterpret_cast(data.data()), data.size()}; +} +/* + Reinterprets a string view of any byte-sized character type as a span of any byte-sized trivial type. +*/ +template +[[nodiscard]] +std::span string_to_data(std::string_view const string) { + return std::span{reinterpret_cast(string.data()), string.size()}; +} + +//--------------------------------------------------------- + +using DataVector = std::vector; + +//--------------------------------------------------------- + +template +void append_to_vector(std::vector& vector, std::span const data) { + vector.insert(vector.end(), data.begin(), data.end()); +} + +//--------------------------------------------------------- + +template +concept IsByteData = IsByte || std::ranges::range && IsByte>; + +/* + Returns the size of any trivial byte-sized element or range of trivial byte-sized elements. +*/ +template +[[nodiscard]] +std::size_t size_of_byte_data(T const& data) { + if constexpr (std::ranges::range) { + return std::ranges::distance(data); + } + else { + return sizeof(data); + } +} + +/* + Copies any type of trivial byte-sized element(s) from data to range. +*/ +template> +[[nodiscard]] +auto copy_byte_data(Data_ const& data, Range_& range) + -> std::ranges::iterator_t +{ + if constexpr (IsByte) { + *std::ranges::begin(range) = *reinterpret_cast(&data); + return std::ranges::begin(range) + 1; + } + else { + return std::ranges::copy(std::span{ + reinterpret_cast(std::ranges::data(data)), + std::ranges::size(data) + }, std::ranges::begin(range)).out; + } +} + +/* + Concatenates any kind of sequence of trivial byte-sized elements like char and std::byte. + The arguments can be individual bytes and/or ranges of bytes. + Returns a utils::DataVector (std::vector). +*/ +template +[[nodiscard]] +DataVector concatenate_byte_data(T const& ... arguments) { + auto buffer = DataVector((size_of_byte_data(arguments) + ...)); + auto buffer_span = std::span{buffer}; + ((buffer_span = std::span{copy_byte_data(arguments, buffer_span), buffer_span.end()}), ...); + return buffer; +} + +//--------------------------------------------------------- + +/* + Parses a string as an integer type in a given base. + For more details, see std::from_chars. This is just an abstraction layer on top of it. +*/ +template +[[nodiscard]] +std::optional string_to_integral(std::string_view const string, int const base = 10) +{ + auto number_result = T{}; + if (std::from_chars(string.data(), string.data() + string.size(), number_result, base).ec == std::errc{}) { + return number_result; + } + return {}; +} + +//--------------------------------------------------------- + +template requires IsByte> +void write_to_file(DataRange_ const& data, std::string const& file_name) { + // std::string because std::ofstream does not take std::string_view. + auto file_stream = std::ofstream{file_name, std::ios::binary}; + file_stream.write(reinterpret_cast(std::ranges::data(data)), std::ranges::size(data)); +} + +//--------------------------------------------------------- + +constexpr auto filter_true = std::views::filter([](auto const& x){ return static_cast(x); }); +constexpr auto dereference_move = std::views::transform([](auto&& x) { return std::move(*x); }); + +/* + Transforms a range of chars into its lowercase equivalent. +*/ +constexpr auto ascii_lowercase_transform = std::views::transform([](char const c) { + return static_cast(std::tolower(static_cast(c))); +}); + +/* + Returns whether lhs and rhs are equal, regardless of casing, assuming both are encoded in ASCII. +*/ +[[nodiscard]] +constexpr bool equal_ascii_case_insensitive(std::string_view const lhs, std::string_view const rhs) noexcept { + return std::ranges::equal(lhs | ascii_lowercase_transform, rhs | ascii_lowercase_transform); +} + +//--------------------------------------------------------- + +/* + Returns the default port corresponding to the specified protocol. +*/ +[[nodiscard]] +constexpr Port default_port_for_protocol(Protocol const protocol) noexcept { + return static_cast(protocol); +} + +[[nodiscard]] +constexpr bool is_protocol_tls_encrypted(Protocol const protocol) noexcept { + return protocol == Protocol::Https; +} + +/* + Returns the protocol that corresponds to the specified case-insensitive string. + For example, "http" converts to Protocol::Http. +*/ +[[nodiscard]] +constexpr Protocol get_protocol_from_string(std::string_view const protocol_string) noexcept { + if (equal_ascii_case_insensitive(protocol_string, "http")) { + return Protocol::Http; + } + else if (equal_ascii_case_insensitive(protocol_string, "https")) { + return Protocol::Https; + } + return Protocol::Unknown; +} + +/* + The result of the split_url function. +*/ +struct UrlComponents { + Protocol protocol{Protocol::Unknown}; + std::string_view host; + Port port{default_port_for_protocol(Protocol::Unknown)}; + std::string_view path; +}; + + +struct HostAndPort { + std::string_view host; + std::optional port; +}; + +/* + Splits a domain name into a name and an optional port. + For example, "localhost:8080" returns "localhost" and 8080, while + "google.com" returns "google.com" and no port (std::nullopt). +*/ +[[nodiscard]] +inline HostAndPort split_domain_name(std::string_view const domain_name) +{ + if (auto const colon_position = domain_name.rfind(':'); + colon_position != std::string_view::npos) + { + if (auto const port = string_to_integral(domain_name.substr(colon_position + 1))) + { + return HostAndPort{ + .host{domain_name.substr(0, colon_position)}, + .port{port} + }; + } + else + { + return HostAndPort{ + .host{domain_name.substr(0, colon_position)}, + .port = std::nullopt + }; + } + } + return HostAndPort{ + .host{domain_name}, + .port = std::nullopt + }; +} + +/* + Splits an URL into its components. +*/ +[[nodiscard]] +inline UrlComponents split_url(std::string_view const url) noexcept { + using namespace std::string_view_literals; + + if (url.empty()) { + return {}; + } + + auto result = UrlComponents{}; + + constexpr auto whitespace_characters = " \t\r\n"sv; + + // Find the start position of the protocol. + auto start_position = url.find_first_not_of(whitespace_characters); + if (start_position == std::string_view::npos) { + return {}; + } + + constexpr auto protocol_suffix = "://"sv; + + // Find the end position of the protocol. + if (auto const position = url.find(protocol_suffix, start_position); + position != std::string_view::npos) + { + result.protocol = get_protocol_from_string(url.substr(start_position, position - start_position)); + result.port = default_port_for_protocol(result.protocol); + + // The start position of the domain name. + start_position = position + protocol_suffix.length(); + } + + // Find the end position of the domain name and start of the path. + if (auto const slash_position = url.find('/', start_position); + slash_position != std::string_view::npos) + { + auto [host, port] = split_domain_name(url.substr(start_position, slash_position - start_position)); + + result.host = host; + + if (port) { + result.port = *port; + } + + start_position = slash_position; + } + else { + // There was nothing after the domain name. + auto [host, port] = split_domain_name(url.substr(start_position)); + + result.host = host; + + if (port) { + result.port = *port; + } + + result.path = "/"sv; + return result; + } + + // Find the end position of the path. + auto const end_position = url.find_last_not_of(whitespace_characters) + 1; + result.path = url.substr(start_position, end_position - start_position); + return result; +} + +/* + Returns the file name part of a URL (or file path with only forward slashes). +*/ +[[nodiscard]] +constexpr std::string_view extract_filename(std::string_view const url) +{ + if (auto const slash_pos = url.rfind('/'); + slash_pos != std::string_view::npos) + { + if (auto const question_mark_pos = url.find('?', slash_pos + 1); + question_mark_pos != std::string_view::npos) + { + return url.substr(slash_pos + 1, question_mark_pos - slash_pos - 1); + } + + return url.substr(slash_pos + 1); + } + return {}; +} + +/* + Returns whether character is allowed in a URI-encoded string or not. +*/ +[[nodiscard]] +constexpr bool get_is_allowed_uri_character(char const character) noexcept { + constexpr auto other_characters = std::string_view{"%-._~:/?#[]@!$&'()*+,;="}; + + return (character >= '0' && character <= '9') || + (character >= 'a' && character <= 'z') || + (character >= 'A' && character <= 'Z') || + other_characters.find(character) != std::string_view::npos; +} + +/* + Returns the URI-encoded equivalent of uri. +*/ +[[nodiscard]] +inline std::string uri_encode(std::string_view const uri) { + auto result_string = std::string(); + result_string.reserve(uri.size()); + + for (auto const character : uri) { + if (get_is_allowed_uri_character(character)) { + result_string += character; + } + else { + result_string += "%xx"; + std::to_chars( + &result_string.back() - 1, + &result_string.back() + 1, + static_cast(character), + 16 + ); + } + } + return result_string; +} + +} // namespace utils + +//--------------------------------------------------------- + +namespace errors { + +/* + The connection to the server failed in some way. + For example, there is no internet connection or the server name is invalid. +*/ +class ConnectionFailed : public std::exception { +public: + [[nodiscard]] + char const* what() const noexcept override { + return reason_.c_str(); + } + + [[nodiscard]] + bool get_is_tls_failure() const noexcept { + return is_tls_failure_; + } + + ConnectionFailed(std::string reason, bool const is_tls_failure = false) noexcept : + reason_(std::move(reason)), + is_tls_failure_{is_tls_failure} + {} + +private: + std::string reason_; + bool is_tls_failure_; +}; + +class ResponseParsingFailed : public std::exception { +public: + [[nodiscard]] + char const* what() const noexcept override { + return reason_.c_str(); + } + + ResponseParsingFailed(std::string reason) : + reason_(std::move(reason)) + {} + +private: + std::string reason_; +}; + +} // namespace errors + +//--------------------------------------------------------- + +/* + This type is used by the Socket class to signify that + the peer closed the connection during a read call. +*/ +struct ConnectionClosed {}; + +/* + An abstraction on top of low level socket and TLS encryption APIs. + Marking a Socket as const only means it won't be moved from or move assigned to. +*/ +class Socket { +public: + /* + Sends data to the peer through the socket. + */ + void write(std::span data) const; + /* + Sends a string to the peer through the socket. + This function takes a basic_string_view, think about + whether you want it to be null terminated or not. + */ + void write(std::string_view const string_view) const { + write(utils::string_to_data(string_view)); + } + + /* + Receives data from the socket and reads it into a buffer. + This function blocks until there is some data available. + The data that was read may be smaller than the buffer. + The function either returns the number of bytes that were read + or a ConnectionClosed value if the peer closed the connection. + */ + [[nodiscard("The result is important as it contains the size that was actually read.")]] + std::variant read(std::span buffer) const; + /* + Receives data from the socket. + This function blocks until there is some data available. + The function either returns the buffer that was read + or a ConnectionClosed value if the peer closed the connection. + The returned DataVector may be smaller than what was requested. + */ + [[nodiscard]] + auto read(std::size_t const number_of_bytes = 512) const + -> std::variant + { + auto result = utils::DataVector(number_of_bytes); + if (auto const read_result = read(result); std::holds_alternative(read_result)) { + result.resize(std::get(read_result)); + return result; + } + return ConnectionClosed{}; + } + + /* + Reads any available data from the socket into a buffer. + This function is nonblocking, and may return std::size_t{} if + there was no data available. The function either returns the number + of bytes that were read or a ConnectionClosed value if the peer + closed the connection. + */ + [[nodiscard("The result is important as it contains the size that was actually read.")]] + std::variant read_available(std::span buffer) const; + /* + Reads any available data from the socket into a buffer. + This function is nonblocking, and may return an empty vector if + there was no data available. The function either returns a utils::DataVector + of the data that was read or a ConnectionClosed value if the peer + closed the connection. + */ + template + [[nodiscard]] + std::variant read_available() const { + auto buffer = utils::DataVector(read_buffer_size); + auto read_offset = std::size_t{}; + + while (true) { + if (auto const read_result = read_available( + std::span{buffer.data() + read_offset, read_buffer_size} + ); std::holds_alternative(read_result)) + { + if (auto const bytes_read = std::get(read_result)) { + read_offset += bytes_read; + buffer.resize(read_offset + read_buffer_size); + } + else return buffer; + } + else return ConnectionClosed{}; + } + return {}; + } + + Socket() = delete; + ~Socket(); // = default in .cpp + + Socket(Socket&&) noexcept; // = default in .cpp + Socket& operator=(Socket&&) noexcept; // = default in .cpp + + Socket(Socket const&) = delete; + Socket& operator=(Socket const&) = delete; + +private: + class Implementation; + std::unique_ptr implementation_; + + Socket(std::string_view server, Port port, bool is_tls_encrypted); + friend Socket open_socket(std::string_view, Port, bool); +}; + +/* + Opens a socket to a server through a port. + If port is 443 OR is_tls_encrypted is true, TLS encryption is used. + Otherwise it is unencrypted. +*/ +[[nodiscard]] +inline Socket open_socket(std::string_view const server, Port const port, bool const is_tls_encrypted = false) { + return Socket{server, port, is_tls_encrypted}; +} + +//--------------------------------------------------------- + +struct Header; + +/* + Represents a HTTP header whose data was copied from somewhere at some point. + It consists of std::string objects instead of std::string_view. +*/ +struct HeaderCopy { + std::string name, value; + + [[nodiscard]] + inline explicit operator Header() const; +}; +/* + Represents a HTTP header whose data is not owned by this object. + It consists of std::string_view objects instead of std::string. +*/ +struct Header { + std::string_view name, value; + + [[nodiscard]] + explicit operator HeaderCopy() const { + return HeaderCopy{ + .name = std::string{name}, + .value = std::string{value}, + }; + } +}; +HeaderCopy::operator Header() const { + return Header{ + .name = std::string_view{name}, + .value = std::string_view{value}, + }; +} + +template +concept IsHeader = utils::IsAnyOf; + +/* + Compares two headers, taking into account case insensitivity. +*/ +[[nodiscard]] +bool operator==(IsHeader auto const& lhs, IsHeader auto const& rhs) { + return lhs.value == rhs.value && utils::equal_ascii_case_insensitive(lhs.name, rhs.name); +} + +enum class StatusCode { + Continue = 100, + SwitchingProtocols = 101, + Processing = 102, + EarlyHints = 103, + + Ok = 200, + Created = 201, + Accepted = 202, + NonAuthoritativeInformation = 203, + NoContent = 204, + ResetContent = 205, + PartialContent = 206, + MultiStatus = 207, + AlreadyReported = 208, + ImUsed = 226, + + MultipleChoices = 300, + MovedPermanently = 301, + Found = 302, + SeeOther = 303, + NotModified = 304, + UseProxy = 305, + SwitchProxy = 306, + TemporaryRedirect = 307, + PermanentRedirect = 308, + + BadRequest = 400, + Unauthorized = 401, + PaymentRequired = 402, + Forbidden = 403, + NotFound = 404, + MethodNotAllowed = 405, + NotAcceptable = 406, + ProxyAuthenticationRequired = 407, + RequestTimeout = 408, + Conflict = 409, + Gone = 410, + LengthRequired = 411, + PreconditionFailed = 412, + PayloadTooLarge = 413, + UriTooLong = 414, + UnsupportedMediaType = 415, + RangeNotSatisfiable = 416, + ExpectationFailed = 417, + ImATeapot = 418, + MisdirectedRequest = 421, + UnprocessableEntity = 422, + Locked = 423, + FailedDependency = 424, + TooEarly = 425, + UpgradeRequired = 426, + PreconditionRequired = 428, + TooManyRequests = 429, + RequestHeaderFieldsTooLarge = 431, + UnavailableForLegalReasons = 451, + + InternalServerError = 500, + NotImplemented = 501, + BadGateway = 502, + ServiceUnavailable = 503, + GatewayTimeout = 504, + HttpVersionNotSupported = 505, + VariantAlsoNegotiates = 506, + InsufficientStorage = 507, + LoopDetected = 508, + NotExtended = 510, + NetworkAuthenticationRequired = 511, + + Unknown = -1 +}; + +struct StatusLine { + std::string http_version; + StatusCode status_code = StatusCode::Unknown; + std::string status_message; + + [[nodiscard]] + bool operator==(StatusLine const&) const noexcept = default; +}; + +namespace algorithms { + +[[nodiscard]] +inline StatusLine parse_status_line(std::string_view const line) { + auto status_line = StatusLine{}; + + auto cursor = std::size_t{}; + + if (auto const http_version_end = line.find(' '); http_version_end != std::string_view::npos) + { + status_line.http_version = line.substr(0, http_version_end); + cursor = http_version_end + 1; + } + else return status_line; + + if (auto const status_code_end = line.find(' ', cursor); status_code_end != std::string_view::npos) + { + if (auto const status_code = utils::string_to_integral(line.substr(cursor, status_code_end))) + { + status_line.status_code = static_cast(*status_code); + } + else return status_line; + cursor = status_code_end + 1; + } + else return status_line; + + status_line.status_message = line.substr(cursor, line.find_last_not_of("\r\n ") + 1 - cursor); + return status_line; +} + +[[nodiscard]] +constexpr std::optional
parse_header(std::string_view const line) { + /* + "An HTTP header consists of its case-insensitive name followed by a colon (:), + then by its value. Whitespace before the value is ignored." + (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers) + + So we're just ignoring whitespace before the value, and after because there may be + an \r there if the line endings are CRLF. + */ + + auto const colon_pos = line.find(':'); + if (colon_pos == std::string_view::npos) { + return {}; + } + + constexpr auto whitespace_characters = std::string_view{" \t\r"}; + + auto const value_start = line.find_first_not_of(whitespace_characters, colon_pos + 1); + if (value_start == std::string_view::npos) { + return {}; + } + + // This will never be npos, assuming the header + // string isn't mutated by some other thread. + auto const value_end = line.find_last_not_of(whitespace_characters); + + return Header{ + .name = line.substr(0, colon_pos), + .value = line.substr(value_start, value_end + 1 - value_start) + }; +} + +[[nodiscard]] +inline std::vector
parse_headers_string(std::string_view const headers) +{ + auto result = std::vector
(); + + std::ranges::copy( + headers + | std::views::split('\n') | std::views::transform(utils::range_to_string_view) + | std::views::transform(parse_header) | utils::filter_true | utils::dereference_move, + std::back_inserter(result) + ); + + return result; +} + +template> +[[nodiscard]] +inline Header_ const* find_header_by_name(Range_ const& headers, std::string_view const name) +{ + auto const lowercase_name_to_search = utils::range_to_string( + name | utils::ascii_lowercase_transform + ); + auto const pos = std::ranges::find_if(headers, [&](Header_ const& header) { + return std::ranges::equal(lowercase_name_to_search, header.name | utils::ascii_lowercase_transform); + }); + if (pos == std::ranges::end(headers)) { + return nullptr; + } + else { + return &*pos; + } +} + +struct ParsedResponse { + StatusLine status_line; + std::string headers_string; + std::vector
headers; // Points into headers_string + utils::DataVector body_data; + + [[nodiscard]] + bool operator==(ParsedResponse const&) const noexcept = default; + + ParsedResponse() = default; + ParsedResponse(StatusLine p_status_line, std::string p_headers_string = {}, std::vector
p_headers = {}, utils::DataVector p_body_data = {}) : + status_line{std::move(p_status_line)}, + headers_string(std::move(p_headers_string)), + headers(std::move(p_headers)), + body_data(std::move(p_body_data)) + {} + + ParsedResponse(ParsedResponse&&) = default; + ParsedResponse& operator=(ParsedResponse&&) = default; + + ParsedResponse(ParsedResponse const&) = delete; + ParsedResponse& operator=(ParsedResponse const&) = delete; +}; + +struct ParsedHeadersInterface { + virtual ~ParsedHeadersInterface() = default; + + constexpr virtual ParsedResponse const& get_parsed_response() const noexcept = 0; + + /* + Returns the status code from the response header. + */ + [[nodiscard]] + StatusCode get_status_code() const { + return get_parsed_response().status_line.status_code; + } + /* + Returns the status code description from the response header. + */ + [[nodiscard]] + std::string_view get_status_message() const { + return get_parsed_response().status_line.status_message; + } + /* + Returns the HTTP version from the response header. + */ + [[nodiscard]] + std::string_view get_http_version() const { + return get_parsed_response().status_line.http_version; + } + /* + Returns a const reference to the parsed status line object. + */ + [[nodiscard]] + StatusLine const& get_status_line() const { + return get_parsed_response().status_line; + } + + /* + Returns the headers of the response as a string. + The returned string_view shall not outlive this Response object. + */ + [[nodiscard]] + std::string_view get_headers_string() const { + return get_parsed_response().headers_string; + } + + /* + Returns the headers of the response as Header objects. + The returned span shall not outlive this Response object. + */ + [[nodiscard]] + std::span
get_headers() const { + return get_parsed_response().headers; + } + /* + Returns a header of the response by its name. + The returned header shall not outlive this Response object. + */ + [[nodiscard]] + std::optional
get_header(std::string_view const name) const { + if (auto const header = algorithms::find_header_by_name(get_parsed_response().headers, name)) { + return *header; + } + else return {}; + } + /* + Returns a header value of the response by its name. + The returned std::string_view shall not outlive this Response object. + */ + [[nodiscard]] + std::optional get_header_value(std::string_view const name) const { + if (auto const header = algorithms::find_header_by_name(get_parsed_response().headers, name)) { + return header->value; + } + else return {}; + } +}; + +class ResponseParser; + +} // namespace algorithms + +class ResponseProgressRaw { + friend class algorithms::ResponseParser; + +public: + constexpr void stop() noexcept { + is_stopped_ = true; + } + + std::span data; + std::size_t new_data_start; + + explicit constexpr ResponseProgressRaw(std::span const p_data, std::size_t const p_new_data_start) noexcept : + data{p_data}, new_data_start{p_new_data_start} + {} + +private: + bool is_stopped_{}; +}; + +class ResponseProgressHeaders : public algorithms::ParsedHeadersInterface { +public: + ResponseProgressRaw raw_progress; + + constexpr void stop() noexcept { + raw_progress.stop(); + } + + [[nodiscard]] + constexpr algorithms::ParsedResponse const& get_parsed_response() const noexcept override { + return parsed_response_; + } + + ResponseProgressHeaders(ResponseProgressRaw const p_raw_progress, algorithms::ParsedResponse const& parsed_response) : + raw_progress{p_raw_progress}, parsed_response_{parsed_response} + {} + + ResponseProgressHeaders() = delete; + ~ResponseProgressHeaders() = default; + + ResponseProgressHeaders(ResponseProgressHeaders const&) = delete; + ResponseProgressHeaders& operator=(ResponseProgressHeaders const&) = delete; + + ResponseProgressHeaders(ResponseProgressHeaders&&) noexcept = delete; + ResponseProgressHeaders& operator=(ResponseProgressHeaders&&) noexcept = delete; + +private: + algorithms::ParsedResponse const& parsed_response_; +}; + +class ResponseProgressBody : public algorithms::ParsedHeadersInterface { +public: + ResponseProgressRaw raw_progress; + + std::span body_data_so_far; + /* + This may not have a value if the transfer encoding is chunked, in which + case the full body length is not known ahead of time. + */ + std::optional total_expected_body_size; + + constexpr void stop() noexcept { + raw_progress.stop(); + } + + [[nodiscard]] + constexpr algorithms::ParsedResponse const& get_parsed_response() const noexcept override { + return parsed_response_; + } + + ResponseProgressBody( + ResponseProgressRaw const p_raw_progress, + algorithms::ParsedResponse const& parsed_response, + std::span const p_body_data_so_far, + std::optional const p_total_expected_body_size + ) : + raw_progress{p_raw_progress}, + body_data_so_far{p_body_data_so_far}, + total_expected_body_size{p_total_expected_body_size}, + parsed_response_{parsed_response} + {} + + ResponseProgressBody() = delete; + ~ResponseProgressBody() = default; + + ResponseProgressBody(ResponseProgressBody const&) = delete; + ResponseProgressBody& operator=(ResponseProgressBody const&) = delete; + + ResponseProgressBody(ResponseProgressBody&&) noexcept = delete; + ResponseProgressBody& operator=(ResponseProgressBody&&) noexcept = delete; + +private: + algorithms::ParsedResponse const& parsed_response_; +}; + +/* + Represents the response of a HTTP request. +*/ +class Response : public algorithms::ParsedHeadersInterface { +public: + [[nodiscard]] + constexpr algorithms::ParsedResponse const& get_parsed_response() const noexcept override { + return parsed_response_; + } + + /* + Returns the body of the response. + The returned std::span shall not outlive this Response object. + */ + [[nodiscard]] + std::span get_body() const { + return parsed_response_.body_data; + } + /* + Returns the body of the response as a string. + The returned std::string_view shall not outlive this Response object. + */ + [[nodiscard]] + std::string_view get_body_string() const { + return utils::data_to_string(get_body()); + } + + [[nodiscard]] + std::string_view get_url() const { + return url_; + } + + // std::future requires default constructibility on MSVC... Because of ABI stability. + Response() = default; + ~Response() = default; + + Response(Response const&) = delete; + Response& operator=(Response const&) = delete; + + Response(Response&&) noexcept = default; + Response& operator=(Response&&) noexcept = default; + + Response(algorithms::ParsedResponse&& parsed_response, std::string&& url) : + parsed_response_{std::move(parsed_response)}, + url_{std::move(url)} + {} + +private: + algorithms::ParsedResponse parsed_response_; + std::string url_; +}; + +namespace algorithms { + +class ChunkyBodyParser { +public: + [[nodiscard]] + std::optional parse_new_data(std::span const new_data) { + if (has_returned_result_) { + return {}; + } + if (is_finished_) { + has_returned_result_ = true; + return std::move(result_); + } + + auto cursor = start_parse_offset_; + + while (true) { + if (cursor >= new_data.size()) { + start_parse_offset_ = cursor - new_data.size(); + return {}; + } + if (auto const cursor_offset = parse_next_part_(new_data.subspan(cursor))) { + cursor += cursor_offset; + } + else { + has_returned_result_ = true; + return std::move(result_); + } + } + } + [[nodiscard]] + std::span get_result_so_far() const { + return result_; + } + +private: + static constexpr auto newline = std::string_view{"\r\n"}; + + /* + "part" refers to a separately parsed unit of data. + This partitioning makes the parsing algorithm simpler. + Returns the position where the part ended. + It may be past the end of the part. + */ + [[nodiscard]] + std::size_t parse_next_part_(std::span const new_data) { + if (chunk_size_left_) { + return parse_chunk_body_part_(new_data); + } + else return parse_chunk_separator_part_(new_data); + } + + [[nodiscard]] + std::size_t parse_chunk_body_part_(std::span const new_data) { + if (chunk_size_left_ > new_data.size()) + { + chunk_size_left_ -= new_data.size(); + utils::append_to_vector(result_, new_data); + return new_data.size(); + } + else { + utils::append_to_vector(result_, new_data.first(chunk_size_left_)); + + // After each chunk, there is a \r\n and then the size of the next chunk. + // We skip the \r\n so the next part starts at the size number. + auto const part_end = chunk_size_left_ + newline.size(); + chunk_size_left_ = 0; + return part_end; + } + } + + [[nodiscard]] + std::size_t parse_chunk_separator_part_(std::span const new_data) { + auto const data_string = utils::data_to_string(new_data); + + auto const first_newline_character_pos = data_string.find(newline[0]); + + if (first_newline_character_pos == std::string_view::npos) { + chunk_size_string_buffer_ += data_string; + return new_data.size(); + } + else if (chunk_size_string_buffer_.empty()) { + parse_chunk_size_left_(data_string.substr(0, first_newline_character_pos)); + } + else { + chunk_size_string_buffer_ += data_string.substr(0, first_newline_character_pos); + parse_chunk_size_left_(chunk_size_string_buffer_); + chunk_size_string_buffer_.clear(); + } + + if (chunk_size_left_ == 0) { + is_finished_ = true; + return 0; + } + + return first_newline_character_pos + newline.size(); + } + + void parse_chunk_size_left_(std::string_view const string) { + // hexadecimal + if (auto const result = utils::string_to_integral(string, 16)) { + chunk_size_left_ = *result; + } + else throw errors::ResponseParsingFailed{"Failed parsing http body chunk size."}; + } + + utils::DataVector result_; + + bool is_finished_{false}; + bool has_returned_result_{false}; + + std::size_t start_parse_offset_{}; + + std::string chunk_size_string_buffer_; + std::size_t chunk_size_left_{}; +}; + +struct ResponseCallbacks { + std::function handle_raw_progress; + std::function handle_headers; + std::function handle_body_progress; + std::function handle_finish; + std::function handle_stop; +}; + +/* + Separate, testable module that parses a http response. + It has support for optional response progress callbacks. +*/ +class ResponseParser { +public: + /* + Parses a new packet of data from the HTTP response. + If it reached the end of the response, the parsed result is returned. + */ + [[nodiscard]] + std::optional parse_new_data(std::span const data) { + if (is_done_) { + return {}; + } + + auto const new_data_start = buffer_.size(); + + utils::append_to_vector(buffer_, data); + + if (callbacks_ && (*callbacks_)->handle_raw_progress) { + auto raw_progress = ResponseProgressRaw{buffer_, new_data_start}; + (*callbacks_)->handle_raw_progress(raw_progress); + if (raw_progress.is_stopped_) { + finish_(); + } + } + + if (!is_done_ && result_.headers_string.empty()) { + try_parse_headers_(new_data_start); + } + + if (!is_done_ && !result_.headers_string.empty()) { + if (chunky_body_parser_) { + parse_new_chunky_body_data_(new_data_start); + } + else { + parse_new_regular_body_data_(new_data_start); + } + } + if (is_done_) { + return std::move(result_); + } + return {}; + } + + ResponseParser() = default; + ResponseParser(ResponseCallbacks& callbacks) : + callbacks_{&callbacks} + {} + +private: + void finish_() { + is_done_ = true; + if (callbacks_ && (*callbacks_)->handle_stop) { + (*callbacks_)->handle_stop(); + } + } + + void try_parse_headers_(std::size_t const new_data_start) { + // Only do anything if the headers are completely received. + if (auto const headers_string = try_extract_headers_string_(new_data_start)) + { + result_.headers_string = *headers_string; + + auto status_line_end = result_.headers_string.find_first_of("\r\n"); + if (status_line_end == std::string_view::npos) { + status_line_end = result_.headers_string.size(); + } + + result_.status_line = algorithms::parse_status_line( + std::string_view{result_.headers_string}.substr(0, status_line_end) + ); + + if (result_.headers_string.size() > status_line_end) { + result_.headers = algorithms::parse_headers_string( + std::string_view{result_.headers_string}.substr(status_line_end) + ); + } + + if (callbacks_ && (*callbacks_)->handle_headers) { + auto progress_headers = ResponseProgressHeaders{ResponseProgressRaw{buffer_, new_data_start}, result_}; + (*callbacks_)->handle_headers(progress_headers); + if (progress_headers.raw_progress.is_stopped_) { + finish_(); + } + } + + if (auto const body_size_try = get_body_size_()) { + body_size_ = *body_size_try; + } + else if (auto const transfer_encoding = algorithms::find_header_by_name(result_.headers, "transfer-encoding"); + transfer_encoding && transfer_encoding->value == "chunked") + { + chunky_body_parser_ = ChunkyBodyParser{}; + } + } + } + /* + Checks whether the headers have been completely received from the server and if so returns them as a string. + Otherwise returns an empty `std::optional`. + + `new_data_start` is the byte index of the beginning of the new data chunk inside `buffer_`. + */ + [[nodiscard]] + std::optional try_extract_headers_string_(std::size_t const new_data_start) { + // An empty line marks the end of the headers part of the response. + // '\n' line endings are not conformant with the HTTP standard but may appear anyways. + for (std::string_view const empty_line : {"\r\n\r\n", "\n\n"}) + { + // The byte index inside buffer_ to start searching for an empty line at. + // This could just be set to zero but to minimize the amount of text to search, + // we start at the latest point where we have not searched yet. + auto const find_start = new_data_start >= empty_line.length() - 1 + ? new_data_start - (empty_line.length() - 1) + : std::size_t{}; + + // String view of the whole buffer_ which has been received at this point. + auto const string_view_to_search = utils::data_to_string(std::span{buffer_}); + + if (auto const position = string_view_to_search.find(empty_line, find_start); + position != std::string_view::npos) + { + body_start_ = position + empty_line.length(); + return string_view_to_search.substr(0, position); + } + } + return {}; + } + [[nodiscard]] + std::optional get_body_size_() const { + if (auto const content_length_string = + algorithms::find_header_by_name(result_.headers, "content-length")) + { + if (auto const parse_result = + utils::string_to_integral(content_length_string->value)) + { + return *parse_result; + } + } + return {}; + } + + void parse_new_chunky_body_data_(std::size_t const new_data_start) { + // May need to add an offset if this packet is where the headers end and the body starts. + auto const body_parse_start = std::max(new_data_start, body_start_); + if (auto body = chunky_body_parser_->parse_new_data(std::span{buffer_}.subspan(body_parse_start))) + { + result_.body_data = *std::move(body); + + if (callbacks_ && (*callbacks_)->handle_body_progress) { + auto body_progress = ResponseProgressBody{ + ResponseProgressRaw{buffer_, new_data_start}, + result_, + result_.body_data, {} + }; + (*callbacks_)->handle_body_progress(body_progress); + } + + finish_(); + } + else if (callbacks_ && (*callbacks_)->handle_body_progress) { + auto body_progress = ResponseProgressBody{ + ResponseProgressRaw{buffer_, new_data_start}, + result_, + chunky_body_parser_->get_result_so_far(), {} + }; + (*callbacks_)->handle_body_progress(body_progress); + if (body_progress.raw_progress.is_stopped_) { + finish_(); + } + } + } + + void parse_new_regular_body_data_(std::size_t const new_data_start) { + if (buffer_.size() >= body_start_ + body_size_) { + auto const body_begin = buffer_.begin() + body_start_; + result_.body_data = utils::DataVector(body_begin, body_begin + body_size_); + + if (callbacks_ && (*callbacks_)->handle_body_progress) { + auto body_progress = ResponseProgressBody{ + ResponseProgressRaw{buffer_, new_data_start}, + result_, + result_.body_data, + body_size_ + }; + (*callbacks_)->handle_body_progress(body_progress); + } + + finish_(); + } + else if (callbacks_ && (*callbacks_)->handle_body_progress) { + auto body_progress = ResponseProgressBody{ + ResponseProgressRaw{buffer_, new_data_start}, + result_, + std::span{buffer_}.subspan(body_start_), + body_size_ + }; + (*callbacks_)->handle_body_progress(body_progress); + if (body_progress.raw_progress.is_stopped_) { + finish_(); + } + } + } + + utils::DataVector buffer_; + + ParsedResponse result_; + bool is_done_{false}; + + std::size_t body_start_{}; + std::size_t body_size_{}; + + std::optional chunky_body_parser_; + + std::optional callbacks_; +}; + +constexpr auto default_receive_buffer_size = std::size_t{1} << 12; + +template +[[nodiscard]] +inline Response receive_response(Socket const&& socket, std::string&& url, ResponseCallbacks&& callbacks) { + // Does not need to be atomic because handle_stop will always be called from this thread. + auto has_stopped = false; + callbacks.handle_stop = [&has_stopped]{ has_stopped = true; }; + + auto response_parser = algorithms::ResponseParser{callbacks}; + + auto read_buffer = std::array(); + + while (!has_stopped) { + if (auto const read_result = socket.read(read_buffer); + std::holds_alternative(read_result)) + { + if (auto parse_result = response_parser.parse_new_data( + std::span{read_buffer}.first(std::get(read_result)) + )) + { + auto response = Response{std::move(*parse_result), std::move(url)}; + if (callbacks.handle_finish) { + callbacks.handle_finish(response); + } + return response; + } + } + else throw errors::ConnectionFailed{"The peer closed the connection unexpectedly"}; + } + + utils::unreachable(); +} + + + +} // namespace algorithms + +//--------------------------------------------------------- + +/* + Enumeration of the different HTTP request methods that can be used. +*/ +enum class RequestMethod { + Connect, + Delete, + Get, + Head, + Options, + Patch, + Post, + Put, + Trace, +}; + +/* + Converts a RequestMethod to its uppercase string equivalent. + For example, RequestMethod::Get becomes std::string_view{"GET"}. +*/ +[[nodiscard]] +inline std::string_view request_method_to_string(RequestMethod const method) { + using enum RequestMethod; + switch (method) { + case Connect: return "CONNECT"; + case Delete: return "DELETE"; + case Get: return "GET"; + case Head: return "HEAD"; + case Options: return "OPTIONS"; + case Patch: return "PATCH"; + case Post: return "POST"; + case Put: return "PUT"; + case Trace: return "TRACE"; + } + utils::unreachable(); +} + +//--------------------------------------------------------- + +/* + Represents a HTTP request. + It is created by calling any of the HTTP verb functions (http_client::get, http_client::post, http_client::put ...) +*/ +class Request { +public: + /* + Adds headers to the request as a string. + These are in the format: "NAME: [ignored whitespace] VALUE" + The string can be multiple lines for multiple headers. + Non-ASCII bytes are considered opaque data, + according to the HTTP specification. + */ + [[nodiscard]] + Request&& add_headers(std::string_view const headers_string) && { + if (headers_string.empty()) { + return std::move(*this); + } + + headers_ += headers_string; + if (headers_string.back() != '\n') { + headers_ += "\r\n"; // CRLF is the correct line ending for the HTTP protocol + } + + return std::move(*this); + } + /* + Adds headers to the request. + */ + template + [[nodiscard]] + Request&& add_headers(std::span const headers) && { + auto headers_string = std::string{}; + headers_string.reserve(headers.size()*128); + + for (auto const& header : headers) { + headers_string += std::format("{}: {}\r\n", header.name, header.value); + } + + return std::move(*this).add_headers(headers_string); + } + /* + Adds headers to the request. + */ + [[nodiscard]] + Request&& add_headers(std::initializer_list
const headers) && { + return std::move(*this).add_headers(std::span{headers}); + } + /* + Adds headers to the request. + This is a variadic template that can take any number of headers. + */ + template + [[nodiscard]] + Request&& add_headers(Header_&& ... p_headers) && { + auto const headers = std::array{Header{p_headers}...}; + return std::move(*this).add_headers(std::span{headers}); + } + /* + Adds a single header to the request. + Equivalent to add_headers with a single Header argument. + */ + [[nodiscard]] + Request&& add_header(Header const& header) && { + return std::move(*this).add_headers(std::format("{}: {}", header.name, header.value)); + } + + /* + Sets the content of the request as a sequence of bytes. + */ + template + [[nodiscard]] + Request&& set_body(std::span const body_data) && { + body_.resize(body_data.size()); + if constexpr (std::same_as) { + std::ranges::copy(body_data, body_.begin()); + } + else { + std::ranges::copy(std::span{reinterpret_cast(body_data.data()), body_data.size()}, body_.begin()); + } + return std::move(*this); + } + /* + Sets the content of the request as a string view. + */ + [[nodiscard]] + Request&& set_body(std::string_view const body_data) && { + return std::move(*this).set_body(utils::string_to_data(body_data)); + } + /* + Sets a callback to be called after every chunk or buffer of response data has been received from the server. + The callback takes a mutable ResponseProgressRaw reference which is used to retrieve the data + and possibly stop receiving data from the server. + */ + [[nodiscard]] + Request&& set_raw_progress_callback(std::function callback) && { + callbacks_.handle_raw_progress = std::move(callback); + return std::move(*this); + } + /* + Sets a callback to be called as soon as the whole header portion of the response has been received. + The callback takes a mutable ResponseProgressHeaders reference which is used to retrieve the parsed header data + and possibly stop receiving data from the server. + */ + [[nodiscard]] + Request&& set_headers_callback(std::function callback) && { + callbacks_.handle_headers = std::move(callback); + return std::move(*this); + } + [[nodiscard]] + Request&& set_body_progress_callback(std::function callback) && { + callbacks_.handle_body_progress = std::move(callback); + return std::move(*this); + } + [[nodiscard]] + Request&& set_finish_callback(std::function callback) && { + callbacks_.handle_finish = std::move(callback); + return std::move(*this); + } + + /* + Sets a flag such that redirects are followed automatically. + This happens when either StatusCode::MovedPermanently (301) and StatusCode::Found (302) is received + and the server supplies a URL to follow via a "location" header. + */ + [[nodiscard]] + Request&& follow_redirects() && { + follow_redirects_ = true; + return std::move(*this); + } + + // Note: send and send_async are not [[nodiscard]] because callbacks + // could potentially be used exclusively to handle the response. + + /* + Sends the request and blocks until the response has been received. + */ + Response send() && { + return algorithms::receive_response<>(send_and_get_receive_socket_(), std::move(url_), std::move(callbacks_)); + } + /* + Sends the request and blocks until the response has been received. + + The buffer_size template parameter specifies the size of the buffer that data + from the server is read into at a time. If it is small, then data will be received + in many times in smaller pieces, with some time cost. If it is big, then + data will be read few times but in large pieces, with more memory cost. + */ + template + Response send() && { + return algorithms::receive_response(send_and_get_receive_socket_(), std::move(url_), std::move(callbacks_)); + } + /* + Sends the request and returns immediately after the data has been sent. + The returned future receives the response asynchronously. + */ + std::future send_async() && { + return std::async(&algorithms::receive_response<>, send_and_get_receive_socket_(), std::move(url_), std::move(callbacks_)); + } + /* + Sends the request and returns immediately after the data has been sent. + The returned future receives the response asynchronously. + + The buffer_size template parameter specifies the size of the buffer that data + from the server is read into at a time. If it is small, then data will be received + many times in smaller pieces, with some time cost. If it is big, then + data will be read few times but in large pieces, with more memory cost. + */ + template + std::future send_async() && { + return std::async(&algorithms::receive_response, send_and_get_receive_socket_(), std::move(url_), std::move(callbacks_)); + } + + Request() = delete; + ~Request() = default; + + Request(Request&&) noexcept = default; + Request& operator=(Request&&) noexcept = default; + + Request(Request const&) = delete; + Request& operator=(Request const&) = delete; + +private: + [[nodiscard]] + Socket send_and_get_receive_socket_() { + auto socket = open_socket(url_components_.host, url_components_.port, utils::is_protocol_tls_encrypted(url_components_.protocol)); + + using namespace std::string_view_literals; + + if (!body_.empty()) { + headers_ += std::format("Content-Length: {}\r\n", body_.size()); + } + + auto const request_data = utils::concatenate_byte_data( + request_method_to_string(method_), + ' ', + url_components_.path, + " HTTP/1.1\r\nHost: "sv, + url_components_.port == utils::default_port_for_protocol(url_components_.protocol) ? + url_components_.host : + std::format("{}:{}", url_components_.host, url_components_.port), + headers_, + "\r\n"sv, + body_ + ); + + // In case you want to look at the request: + // std::println("{}", std::string_view{reinterpret_cast(request_data.data()), request_data.size()}); + + socket.write(request_data); + + return socket; + } + + RequestMethod method_; + + bool follow_redirects_{}; + + std::string url_; + utils::UrlComponents url_components_; + + std::string headers_{"\r\n"}; + utils::DataVector body_; + + algorithms::ResponseCallbacks callbacks_; + + Request(RequestMethod const method, std::string_view const url, Protocol const default_protocol) : + method_{method}, + url_{utils::uri_encode(url)}, + url_components_{utils::split_url(std::string_view{url_})} + { + if (url_components_.protocol == Protocol::Unknown) + { + url_components_.protocol = default_protocol; + } + if (url_components_.port == utils::default_port_for_protocol(Protocol::Unknown)) + { + url_components_.port = utils::default_port_for_protocol(url_components_.protocol); + } + } + friend Request get(std::string_view, Protocol); + friend Request post(std::string_view, Protocol); + friend Request put(std::string_view, Protocol); + friend Request make_request(RequestMethod, std::string_view, Protocol); +}; + +/* + Creates a GET request. + url is a URL to the server or resource that the request targets. + If url contains a protocol prefix, it is used. Otherwise, default_protocol is used. +*/ +[[nodiscard]] +inline Request get(std::string_view const url, Protocol const default_protocol = Protocol::Http) { + return Request{RequestMethod::Get, url, default_protocol}; +} + +/* + Creates a POST request. + url is a URL to the server or resource that the request targets. + If url contains a protocol prefix, it is used. Otherwise, default_protocol is used. +*/ +[[nodiscard]] +inline Request post(std::string_view const url, Protocol const default_protocol = Protocol::Http) { + return Request{RequestMethod::Post, url, default_protocol}; +} + +/* + Creates a PUT request. + url is a URL to the server or resource that the request targets. + If url contains a protocol prefix, it is used. Otherwise, default_protocol is used. +*/ +[[nodiscard]] +inline Request put(std::string_view const url, Protocol const default_protocol = Protocol::Http) { + return Request{RequestMethod::Put, url, default_protocol}; +} + +/* + Creates a http(s) request. + Can be used to do the same things as get() and post(), but with more method options. + If url contains a protocol prefix, it is used. Otherwise, default_protocol is used. +*/ +[[nodiscard]] +inline Request make_request( + RequestMethod const method, + std::string_view const url, + Protocol const default_protocol = Protocol::Http +) { + return Request{method, url, default_protocol}; +} + +} // namespace http_client diff --git a/src/gpu.cu b/src/gpu.cu index 7d00c6b..6ad9739 100644 --- a/src/gpu.cu +++ b/src/gpu.cu @@ -1,5 +1,6 @@ #include "Random.h" #include "gpu.h" +#include "config.h" #include #include @@ -472,8 +473,7 @@ struct Result { template struct ResultSampler { ImprovedNoise octaves[Octaves]; -__device__ float sample_only_a(const GradDotTable &table, int32_t x, - int32_t y, int32_t z) const { + __device__ float sample_only_a(const GradDotTable &table, int32_t x, int32_t y, int32_t z) const { float val = 0; if constexpr (Octaves >= 1) val += sample_octave(table, octaves[0], x, y, z); @@ -495,8 +495,7 @@ __device__ float sample_only_a(const GradDotTable &table, int32_t x, val += sample_octave(table, octaves[16], x, y, z); return val; } -__device__ float sample(const GradDotTable &table, int32_t x, int32_t y, - int32_t z) const { + __device__ float sample(const GradDotTable &table, int32_t x, int32_t y, int32_t z) const { float val = 0; if constexpr (Octaves >= 1) val += sample_octave(table, octaves[0], x, y, z); @@ -1854,7 +1853,7 @@ void GpuThread::run() { } } - if ((i + 1) % PRINT_INTERVAL == 0) { + if ((i + 1) % cfg("default_print_interval", 4096) == 0) { auto end = std::chrono::steady_clock::now(); double host_total_time = std::chrono::duration_cast(end - start).count() * 1e-9; diff --git a/src/main.cpp b/src/main.cpp index bf6603c..74d3777 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,6 +1,9 @@ #include "common.h" +#include "shroomposter.h" + #ifndef NO_GPU #include "gpu.h" +#include "config.h" #endif #ifndef NO_CPU #include "cpu.h" @@ -178,10 +181,58 @@ int main_inner(int argc, char **argv) { return 1; } - const int threads = args.threads.value_or(args.client ? 0 : 1); - int32_t min_size = args.min_size.value_or(10'000'000 * (large_biomes ? 16 : 1)); + std::FILE *output_file = nullptr; + std::FILE *latest_file = nullptr; + + const int threads = args.threads.value_or( + args.client ? 0 : (int)cfg("default_threads", 16) + ); + int32_t default_size = + large_biomes + ? (unbound + ? (int32_t)cfg("ULB_sizethreshold", 128000000) + : (int32_t)cfg("LB_sizethreshold", 96000000)) + : (unbound + ? (int32_t)cfg("USB_sizethreshold", 10000000) + : (int32_t)cfg("SB_sizethreshold", 10000000)); + + int32_t min_size = args.min_size.value_or(default_size); + if (threads != 0) { - std::printf("min_size = %" PRIi32 "\n", min_size); + + std::string default_output; + + if (large_biomes) { + if (unbound) + default_output = "output/all/ULB.txt"; + else + default_output = "output/all/LB.txt"; + } else { + if (unbound) + default_output = "output/all/USB.txt"; + else + default_output = "output/all/SB.txt"; + } + + const char *output_file_path = + args.output_file + ? args.output_file.value().c_str() + : default_output.c_str(); + + output_file = std::fopen(output_file_path, "a"); + if (!output_file) { + std::fprintf(stderr, "Could not open %s\n", output_file_path); + return 1; + } + + latest_file = std::fopen("latest_output.txt", "w"); + if (!latest_file) { + std::fprintf(stderr, "Could not open latest_output.txt\n"); + return 1; + } + + std::fprintf(output_file, "\n"); + std::fflush(output_file); } if (no_gpu && args.devices.size() != 0) { @@ -197,19 +248,19 @@ int main_inner(int argc, char **argv) { return 1; } - std::printf("Hello! large_biomes = %s\nunbound: %s\nprint interval: %d\n", large_biomes ? "true" : "false", unbound ? "true" : "false", PRINT_INTERVAL); - - std::FILE *output_file = nullptr; - if (threads != 0) { - const char *output_file_path = args.output_file ? args.output_file.value().c_str() : "output.txt"; - output_file = std::fopen(output_file_path, "a"); - if (output_file == nullptr) { - std::fprintf(stderr, "Could not open %s\n", output_file_path); - return 1; - } - std::fprintf(output_file, "\n"); - std::fflush(output_file); - } + std::printf( + "Hello! :3\n" + "version = %s\n" + "large_biomes = %s\n" + "unbound = %s\n" + "print interval = %lld\n" + "min_size = %" PRIi32 "\n", + version.c_str(), + large_biomes ? "true" : "false", + unbound ? "true" : "false", + cfg("default_print_interval", 256), + min_size + ); GpuOutputs gpu_outputs; CpuOutputs cpu_outputs; @@ -250,9 +301,25 @@ int main_inner(int argc, char **argv) { while (!cpu_outputs.queue.empty()) { auto output = cpu_outputs.queue.front(); cpu_outputs.queue.pop(); + + shroomposter_submit( + output.seed, + output.x, + output.z, + output.score + ); std::printf("%" PRIi64 " at %" PRIi32 " %" PRIi32 " with %" PRIi32 "\n", output.seed, output.x, output.z, output.score); - std::fprintf(output_file, "%" PRIi64 " %" PRIi32 " %" PRIi32 " %" PRIi32 "\n", output.seed, output.x, output.z, output.score); - std::fflush(output_file); + if (output_file) { + std::fprintf(output_file, "%" PRIi64 " %" PRIi32 " %" PRIi32 " %" PRIi32 "\n", output.seed, output.x, output.z, output.score); + std::fflush(output_file); + } + if (latest_file) { + std::rewind(latest_file); + std::fprintf(latest_file, + "%" PRIi64 " %" PRIi32 " %" PRIi32 " %" PRIi32 "\n", + output.seed, output.x, output.z, output.score); + std::fflush(latest_file); + } } } @@ -302,18 +369,23 @@ int main_inner(int argc, char **argv) { } #endif - if (output_file != nullptr) { - std::fclose(output_file); + if (output_file) { + std::fclose(output_file); } - return 0; + if (latest_file) { + std::fclose(latest_file); + } } int main(int argc, char **argv) { try { + loadConfig(); + shroomposter_start(); return main_inner(argc, argv); } catch (std::exception &e) { std::fprintf(stderr, "Uncaught exception in main: %s\n", e.what()); + shroomposter_stop(); std::abort(); } -} +} \ No newline at end of file diff --git a/src/shroomposter.cpp b/src/shroomposter.cpp new file mode 100644 index 0000000..4a7e577 --- /dev/null +++ b/src/shroomposter.cpp @@ -0,0 +1,160 @@ +#include "shroomposter.h" +#include "config.h" + +#if defined(_WIN32) || __has_include() +#include "cpp20_http_client.hpp" +#define SHROOMPOSTER_HTTP_AVAILABLE 1 +#else +#define SHROOMPOSTER_HTTP_AVAILABLE 0 +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static std::mutex g_mutex; +static std::condition_variable g_cv; + +static std::queue g_queue; + +static std::thread g_thread; +static std::atomic g_running{false}; +static std::atomic g_http_unavailable_logged{false}; + +static void shroomposter_worker() +{ + while (true) + { + PostResult result; + + { + std::unique_lock lock(g_mutex); + + g_cv.wait(lock, [] { + return !g_queue.empty() || !g_running; + }); + + if (!g_running && g_queue.empty()) + break; + + result = g_queue.front(); + g_queue.pop(); + } + + std::string json = + "{\"data\":[{" + "\"seed\":" + std::to_string(result.seed) + + ",\"x\":" + std::to_string(result.x) + + ",\"z\":" + std::to_string(result.z) + + ",\"claimed_size\":" + std::to_string(result.claimed_size) + + "}]}"; + + std::printf("JSON: %s\n", json.c_str()); + +#ifdef LARGE_BIOMES + constexpr char endpoint[] = + "https://shroomweb.0xa.pw/large_biomes"; +#else + constexpr char endpoint[] = + "https://shroomweb.0xa.pw/small_biomes"; +#endif + +#if SHROOMPOSTER_HTTP_AVAILABLE + try + { + auto response = + http_client::post( + endpoint, + http_client::Protocol::Https + ) + .add_header({ + .name = "api-key", + .value = shroomin_api_key + }) + .add_header({ + .name = "Content-Type", + .value = "application/json" + }) + .set_body(json) + .send(); + + std::printf( + "POST %" PRIi64 " -> HTTP %d %.*s\n", + result.seed, + static_cast(response.get_status_code()), + (int)response.get_status_message().size(), + response.get_status_message().data() + ); + + auto body = response.get_body_string(); + + if (!body.empty()) + { + std::printf( + "Response: %.*s\n", + (int)body.size(), + body.data() + ); + } + } + catch (const std::exception& e) + { + std::fprintf( + stderr, + "POST failed: %s\n", + e.what() + ); + } +#else + if (!g_http_unavailable_logged.exchange(true)) { + std::fprintf(stderr, "HTTP posting disabled: OpenSSL headers not found at build time\n"); + } +#endif + } +} + +void shroomposter_start() +{ + std::printf( + "API KEY = [%s]\n", + shroomin_api_key.c_str() + ); + + g_running = true; + g_thread = std::thread(shroomposter_worker); +} + +void shroomposter_stop() +{ + g_running = false; + g_cv.notify_all(); + + if (g_thread.joinable()) + g_thread.join(); +} + +void shroomposter_submit( + int64_t seed, + int32_t x, + int32_t z, + int64_t claimed_size) +{ + { + std::lock_guard lock(g_mutex); + + g_queue.push({ + seed, + x, + z, + claimed_size + }); + } + + g_cv.notify_one(); +} diff --git a/src/shroomposter.h b/src/shroomposter.h new file mode 100644 index 0000000..189d4a6 --- /dev/null +++ b/src/shroomposter.h @@ -0,0 +1,22 @@ +#pragma once + +#include + +struct PostResult +{ + int64_t seed; + int32_t x; + int32_t z; + int64_t claimed_size; +}; + +void shroomposter_start(); + +void shroomposter_stop(); + +void shroomposter_submit( + int64_t seed, + int32_t x, + int32_t z, + int64_t claimed_size +); \ No newline at end of file