From 90feb460fca5a8dc9d05823a196bae9baa1924b2 Mon Sep 17 00:00:00 2001 From: Uche Mennel Date: Sun, 27 May 2018 23:01:56 +0200 Subject: [PATCH] Make http client connection buffer size configurable --- boost/network/protocol/http/client.hpp | 1 - .../protocol/http/client/async_impl.hpp | 6 +++- .../http/client/connection/async_base.hpp | 5 ++- .../http/client/connection/async_normal.hpp | 33 +++++++++---------- .../connection/async_protocol_handler.hpp | 5 +-- boost/network/protocol/http/client/facade.hpp | 5 ++- boost/network/protocol/http/client/macros.hpp | 18 +++++++--- .../http/policies/async_connection.hpp | 5 ++- 8 files changed, 50 insertions(+), 28 deletions(-) diff --git a/boost/network/protocol/http/client.hpp b/boost/network/protocol/http/client.hpp index c4d197c16..1e47b7f5d 100644 --- a/boost/network/protocol/http/client.hpp +++ b/boost/network/protocol/http/client.hpp @@ -21,7 +21,6 @@ #include #include -#include #include namespace boost { diff --git a/boost/network/protocol/http/client/async_impl.hpp b/boost/network/protocol/http/client/async_impl.hpp index 0c0bc80ce..8e6dda6b1 100644 --- a/boost/network/protocol/http/client/async_impl.hpp +++ b/boost/network/protocol/http/client/async_impl.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include namespace boost { @@ -31,7 +32,10 @@ struct async_client typedef typename resolver::type resolver_type; typedef typename string::type string_type; - typedef typename std::array::type, 1024>::const_iterator const_iterator; + typedef + typename std::array::type, + BOOST_NETWORK_HTTP_CLIENT_CONNECTION_BUFFER_SIZE>:: + const_iterator const_iterator; typedef iterator_range char_const_range; typedef std::function::type string_type; typedef basic_request request; typedef basic_response response; - typedef typename std::array::type, 1024>::const_iterator const_iterator; + typedef + typename std::array::type, + BOOST_NETWORK_HTTP_CLIENT_CONNECTION_BUFFER_SIZE>:: + const_iterator const_iterator; typedef iterator_range char_const_range; typedef std::function body_callback_function_type; diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp index 97af80fe2..b89ad5c65 100644 --- a/boost/network/protocol/http/client/connection/async_normal.hpp +++ b/boost/network/protocol/http/client/connection/async_normal.hpp @@ -38,19 +38,20 @@ namespace network { namespace http { namespace impl { -template +template struct chunk_encoding_parser { + typedef typename buffer_type::const_iterator const_iterator; + typedef boost::iterator_range char_const_range; + chunk_encoding_parser() : state(state_t::header), chunk_size(0) {} enum class state_t { header, header_end, data, data_end }; state_t state; size_t chunk_size; - std::array::type, 1024> buffer; + buffer_type buffer; - void update_chunk_size( - boost::iterator_range::type, 1024>::const_iterator> const &range) { + void update_chunk_size(char_const_range const &range) { if (range.empty()) return; std::stringstream ss; ss << std::hex << range; @@ -60,11 +61,7 @@ struct chunk_encoding_parser { chunk_size = (chunk_size << (range.size() * 4)) | size; } - boost::iterator_range< - typename std::array::type, 1024>::const_iterator> - operator()( - boost::iterator_range::type, 1024>::const_iterator> const &range) { + char_const_range operator()(char_const_range const &range) { auto iter = boost::begin(range); auto begin = iter; auto pos = boost::begin(buffer); @@ -147,6 +144,7 @@ struct http_async_connection typedef typename delegate_factory::type delegate_factory_type; typedef typename delegate_factory_type::connection_delegate_ptr connection_delegate_ptr; + typedef chunk_encoding_parser chunk_encoding_parser_type; http_async_connection(resolver_type& resolver, resolve_function resolve, bool follow_redirect, int timeout, @@ -484,13 +482,14 @@ struct http_async_connection } else { string_type body_string; if (this->is_chunk_encoding && remove_chunk_markers_) { - for (size_t i = 0; i < this->partial_parsed.size(); i += 1024) { + const auto parse_buffer_size = parse_chunk_encoding.buffer.size(); + for (size_t i = 0; i < this->partial_parsed.size(); i += parse_buffer_size) { auto range = parse_chunk_encoding(boost::make_iterator_range( - static_cast::type, 1024>::const_iterator>( - this->partial_parsed.data()) + i, - static_cast::type, 1024>::const_iterator>( - this->partial_parsed.data()) + - std::min(i + 1024, this->partial_parsed.size()))); + static_cast< + typename chunk_encoding_parser_type::const_iterator>(this->partial_parsed.data()) + i, + static_cast< + typename chunk_encoding_parser_type::const_iterator>(this->partial_parsed.data()) + + std::min(i + parse_buffer_size, this->partial_parsed.size()))); body_string.append(boost::begin(range), boost::end(range)); } this->partial_parsed.clear(); @@ -602,7 +601,7 @@ struct http_async_connection connection_delegate_ptr delegate_; boost::asio::streambuf command_streambuf; string_type method; - chunk_encoding_parser parse_chunk_encoding; + chunk_encoding_parser_type parse_chunk_encoding; }; } // namespace impl diff --git a/boost/network/protocol/http/client/connection/async_protocol_handler.hpp b/boost/network/protocol/http/client/connection/async_protocol_handler.hpp index e25b75d87..383b62038 100644 --- a/boost/network/protocol/http/client/connection/async_protocol_handler.hpp +++ b/boost/network/protocol/http/client/connection/async_protocol_handler.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -400,8 +401,8 @@ struct http_async_protocol_handler { } typedef response_parser response_parser_type; - // TODO(dberris): make 1024 go away and become a configurable value. - typedef std::array::type, 1024> buffer_type; + typedef std::array::type, + BOOST_NETWORK_HTTP_CLIENT_CONNECTION_BUFFER_SIZE> buffer_type; response_parser_type response_parser_; boost::promise version_promise; diff --git a/boost/network/protocol/http/client/facade.hpp b/boost/network/protocol/http/client/facade.hpp index 7ec8e2f0a..bfe498060 100644 --- a/boost/network/protocol/http/client/facade.hpp +++ b/boost/network/protocol/http/client/facade.hpp @@ -40,7 +40,10 @@ class basic_client_facade { /** The response type. This models the HTTP Response concept.*/ typedef basic_response response; - typedef typename std::array::type, 1024>::const_iterator const_iterator; + typedef + typename std::array::type, + BOOST_NETWORK_HTTP_CLIENT_CONNECTION_BUFFER_SIZE>:: + const_iterator const_iterator; typedef iterator_range char_const_range; /** diff --git a/boost/network/protocol/http/client/macros.hpp b/boost/network/protocol/http/client/macros.hpp index e89dab45c..8be4028bd 100644 --- a/boost/network/protocol/http/client/macros.hpp +++ b/boost/network/protocol/http/client/macros.hpp @@ -10,11 +10,21 @@ #include #include +#ifndef BOOST_NETWORK_HTTP_CLIENT_CONNECTION_BUFFER_SIZE +/** + * We define the buffer size for each connection that we will use on the client + * side. + */ +#define BOOST_NETWORK_HTTP_CLIENT_CONNECTION_BUFFER_SIZE 4096uL +#endif + #ifndef BOOST_NETWORK_HTTP_BODY_CALLBACK -#define BOOST_NETWORK_HTTP_BODY_CALLBACK(function_name, range_name, \ - error_name) \ - void function_name(boost::iterator_range::const_iterator> (range_name), \ - boost::system::error_code const& (error_name)) +#define BOOST_NETWORK_HTTP_BODY_CALLBACK(function_name, range_name, error_name)\ + void function_name( \ + boost::iterator_range< \ + std::array:: \ + const_iterator>(range_name), \ + boost::system::error_code const&(error_name)) #endif #endif /* BOOST_NETWORK_PROTOCOL_HTTP_CLIENT_MACROS_HPP_20110430 */ diff --git a/boost/network/protocol/http/policies/async_connection.hpp b/boost/network/protocol/http/policies/async_connection.hpp index 64a6b287e..db239eef9 100644 --- a/boost/network/protocol/http/policies/async_connection.hpp +++ b/boost/network/protocol/http/policies/async_connection.hpp @@ -30,7 +30,10 @@ struct async_connection_policy : resolver_policy::type { typedef typename resolver_base::resolve_function resolve_function; typedef typename resolver_base::resolve_completion_function resolve_completion_function; - typedef typename std::array::type, 1024>::const_iterator const_iterator; + typedef + typename std::array::type, + BOOST_NETWORK_HTTP_CLIENT_CONNECTION_BUFFER_SIZE>:: + const_iterator const_iterator; typedef iterator_range char_const_range; typedef std::function