From 16f5c9d7da3d57c5a5f1434000ec541b1efec2f2 Mon Sep 17 00:00:00 2001
From: Uche Mennel <umennel@gmx.ch>
Date: Wed, 14 Dec 2016 10:11:42 +0100
Subject: [PATCH 01/10] Introduced new chunked transfer encoding parser to
 remove chunk markers in streaming clients.

---
 .../protocol/http/client/async_impl.hpp       |  3 +-
 .../http/client/connection/async_base.hpp     |  4 +-
 .../http/client/connection/async_normal.hpp   | 98 +++++++++++++++++--
 boost/network/protocol/http/client/facade.hpp |  3 +-
 .../network/protocol/http/client/options.hpp  | 16 ++-
 boost/network/protocol/http/client/pimpl.hpp  | 10 +-
 .../http/policies/async_connection.hpp        | 14 +--
 7 files changed, 126 insertions(+), 22 deletions(-)

diff --git a/boost/network/protocol/http/client/async_impl.hpp b/boost/network/protocol/http/client/async_impl.hpp
index 31578c2eb..9c9a8e879 100644
--- a/boost/network/protocol/http/client/async_impl.hpp
+++ b/boost/network/protocol/http/client/async_impl.hpp
@@ -42,6 +42,7 @@ struct async_client
 
   async_client(bool cache_resolved, bool follow_redirect,
                bool always_verify_peer, int timeout,
+               bool remove_chunk_markers,
                std::shared_ptr<boost::asio::io_service> service,
                optional<string_type> certificate_filename,
                optional<string_type> verify_path,
@@ -49,7 +50,7 @@ struct async_client
                optional<string_type> private_key_file,
                optional<string_type> ciphers,
                optional<string_type> sni_hostname, long ssl_options)
-      : connection_base(cache_resolved, follow_redirect, timeout),
+      : connection_base(cache_resolved, follow_redirect, timeout, remove_chunk_markers),
         service_ptr(service.get() ? service
                                   : std::make_shared<boost::asio::io_service>()),
         service_(*service_ptr),
diff --git a/boost/network/protocol/http/client/connection/async_base.hpp b/boost/network/protocol/http/client/connection/async_base.hpp
index a31ad30fd..d3e3531e8 100644
--- a/boost/network/protocol/http/client/connection/async_base.hpp
+++ b/boost/network/protocol/http/client/connection/async_base.hpp
@@ -43,7 +43,7 @@ struct async_connection_base {
   // tag.
   static connection_ptr new_connection(
       resolve_function resolve, resolver_type &resolver, bool follow_redirect,
-      bool always_verify_peer, bool https, int timeout,
+      bool always_verify_peer, bool https, int timeout, bool remove_chunk_markers,
       optional<string_type> certificate_filename = optional<string_type>(),
       optional<string_type> const &verify_path = optional<string_type>(),
       optional<string_type> certificate_file = optional<string_type>(),
@@ -59,7 +59,7 @@ struct async_connection_base {
         certificate_filename, verify_path, certificate_file, private_key_file,
         ciphers, sni_hostname, ssl_options);
     auto temp = std::make_shared<async_connection>(
-        resolver, resolve, follow_redirect, timeout, std::move(delegate));
+        resolver, resolve, follow_redirect, timeout, remove_chunk_markers, std::move(delegate));
     BOOST_ASSERT(temp != nullptr);
     return temp;
   }
diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp
index 00ec41a37..5fad8c82c 100644
--- a/boost/network/protocol/http/client/connection/async_normal.hpp
+++ b/boost/network/protocol/http/client/connection/async_normal.hpp
@@ -11,6 +11,7 @@
 
 #include <iterator>
 #include <cstdint>
+#include <iostream>
 #include <boost/algorithm/string/trim.hpp>
 #include <boost/asio/steady_timer.hpp>
 #include <boost/asio/placeholders.hpp>
@@ -37,6 +38,77 @@ namespace network {
 namespace http {
 namespace impl {
 
+template <class Tag>
+struct chunk_encoding_parser {
+
+  chunk_encoding_parser() : state(state_t::header), chunk_size(0) {}
+
+  enum state_t { header, header_end, data, data_end };
+
+  state_t state;
+  size_t chunk_size;
+  std::array<typename char_<Tag>::type, 1024> buffer;
+
+  void update_chunk_size(boost::iterator_range<typename std::array<typename char_<Tag>::type, 1024>::const_iterator> const& range) {
+    if (range.empty())
+        return;
+    std::stringstream ss;
+    ss << std::hex << range;
+    size_t size;
+    ss >> size;
+    chunk_size = (chunk_size << (range.size()*4)) + size;
+  }
+
+  boost::iterator_range<typename std::array<typename char_<Tag>::type, 1024>::const_iterator> operator()(boost::iterator_range<typename std::array<typename char_<Tag>::type, 1024>::const_iterator> const& range) {
+    auto iter = boost::begin(range);
+    auto begin = iter;
+    auto pos = boost::begin(buffer);
+
+    while (iter != boost::end(range))
+      switch(state) {
+        case state_t::header:
+          iter = std::find(iter, boost::end(range), '\r');
+          update_chunk_size(boost::make_iterator_range(begin, iter));
+          if (iter != boost::end(range)) {
+            state = state_t::header_end;
+            ++iter;
+          }
+          break;
+
+        case state_t::header_end:
+          BOOST_ASSERT(*iter == '\n');
+          ++iter;
+          state = state_t::data;
+          break;
+
+        case state_t::data:
+          if (chunk_size == 0) {
+            BOOST_ASSERT(*iter == '\r');
+            ++iter;
+            state = state_t::data_end;
+          } else {
+            auto len = std::min(chunk_size, (size_t)std::distance(iter, boost::end(range)));
+            begin = iter;
+            iter = std::next(iter, len);
+            pos = std::copy(begin, iter, pos);
+            chunk_size -= len;
+          }
+          break;
+
+        case state_t::data_end:
+          BOOST_ASSERT (*iter == '\n');
+          ++iter;
+          begin = iter;
+          state = state_t::header;
+          break;
+
+        default:
+          BOOST_ASSERT(false && "Bug, report this to the developers!");
+      }
+    return boost::make_iterator_range(boost::begin(buffer), pos);
+  }
+};
+
 template <class Tag, unsigned version_major, unsigned version_minor>
 struct async_connection_base;
 
@@ -73,8 +145,10 @@ struct http_async_connection
 
   http_async_connection(resolver_type& resolver, resolve_function resolve,
                         bool follow_redirect, int timeout,
+                        bool remove_chunk_markers,
                         connection_delegate_ptr delegate)
       : timeout_(timeout),
+        remove_chunk_markers_(remove_chunk_markers),
         timer_(resolver.get_io_service()),
         is_timedout_(false),
         follow_redirect_(follow_redirect),
@@ -397,13 +471,23 @@ struct http_async_connection
               callback(make_iterator_range(begin, end), ec);
             } else {
               string_type body_string;
-              std::swap(body_string, this->partial_parsed);
-			  auto it = this->part.begin();
-			  std::advance(it, bytes_transferred);
-              body_string.append(this->part.begin(), it);
-              if (this->is_chunk_encoding) {
-                this->body_promise.set_value(parse_chunk_encoding(body_string));
+              if (this->is_chunk_encoding && remove_chunk_markers_) {
+                for (size_t i = 0; i < this->partial_parsed.size(); i += 1024) {
+                  auto range = parse_chunk_encoding(
+                    boost::make_iterator_range(this->partial_parsed.data() + i, 
+                                               this->partial_parsed.data() + std::min(i+1024, this->partial_parsed.size())));
+                  body_string.append(boost::begin(range), boost::end(range));
+                }
+                this->partial_parsed.clear();
+                auto range = parse_chunk_encoding(boost::make_iterator_range(this->part.begin(), 
+                                                                             this->part.begin() + bytes_transferred));
+                body_string.append(boost::begin(range), boost::end(range));
+                this->body_promise.set_value(body_string);
               } else {
+                std::swap(body_string, this->partial_parsed);
+                auto it = this->part.begin();
+                std::advance(it, bytes_transferred);
+                body_string.append(this->part.begin(), it);
                 this->body_promise.set_value(body_string);
               }
             }
@@ -520,6 +604,7 @@ struct http_async_connection
   }
 
   int timeout_;
+  bool remove_chunk_markers_;
   boost::asio::steady_timer timer_;
   bool is_timedout_;
   bool follow_redirect_;
@@ -529,6 +614,7 @@ struct http_async_connection
   connection_delegate_ptr delegate_;
   boost::asio::streambuf command_streambuf;
   string_type method;
+  chunk_encoding_parser<Tag> parse_chunk_encoding;
 };
 
 }  // namespace impl
diff --git a/boost/network/protocol/http/client/facade.hpp b/boost/network/protocol/http/client/facade.hpp
index 0d175acd9..7ec8e2f0a 100644
--- a/boost/network/protocol/http/client/facade.hpp
+++ b/boost/network/protocol/http/client/facade.hpp
@@ -312,7 +312,8 @@ class basic_client_facade {
         options.openssl_verify_path(), options.openssl_certificate_file(),
         options.openssl_private_key_file(), options.openssl_ciphers(),
         options.openssl_sni_hostname(), options.openssl_options(),
-        options.io_service(), options.timeout()));
+        options.io_service(), options.timeout(),
+        options.remove_chunk_markers()));
   }
 };
 
diff --git a/boost/network/protocol/http/client/options.hpp b/boost/network/protocol/http/client/options.hpp
index 73d685577..b55be01b8 100644
--- a/boost/network/protocol/http/client/options.hpp
+++ b/boost/network/protocol/http/client/options.hpp
@@ -34,7 +34,8 @@ class client_options {
         openssl_options_(0),
         io_service_(),
         always_verify_peer_(true),
-        timeout_(0) {}
+        timeout_(0),
+        remove_chunk_markers_(false) {}
 
   client_options(client_options const& other)
       : cache_resolved_(other.cache_resolved_),
@@ -48,7 +49,8 @@ class client_options {
         openssl_options_(other.openssl_options_),
         io_service_(other.io_service_),
         always_verify_peer_(other.always_verify_peer_),
-        timeout_(other.timeout_) {}
+        timeout_(other.timeout_),
+        remove_chunk_markers_(other.remove_chunk_markers) {}
 
   client_options& operator=(client_options other) {
     other.swap(*this);
@@ -69,6 +71,7 @@ class client_options {
     swap(io_service_, other.io_service_);
     swap(always_verify_peer_, other.always_verify_peer_);
     swap(timeout_, other.timeout_);
+    swap(remove_chunk_markers_, other.remove_chunk_markers_);
   }
 
   /// Specify whether the client should cache resolved endpoints.
@@ -154,6 +157,12 @@ class client_options {
     return *this;
   }
 
+  /// Set an overall timeout for HTTP requests.
+  client_options& remove_chunk_markers(bool v) {
+    remove_chunk_markers_ = v;
+    return *this;
+  }
+
   bool cache_resolved() const { return cache_resolved_; }
 
   bool follow_redirects() const { return follow_redirects_; }
@@ -190,6 +199,8 @@ class client_options {
 
   int timeout() const { return timeout_; }
 
+  bool remove_chunk_markers() const { return remove_chunk_markers_; }
+
  private:
   bool cache_resolved_;
   bool follow_redirects_;
@@ -203,6 +214,7 @@ class client_options {
   std::shared_ptr<boost::asio::io_service> io_service_;
   bool always_verify_peer_;
   int timeout_;
+  bool remove_chunk_markers_;
 };
 
 template <class Tag>
diff --git a/boost/network/protocol/http/client/pimpl.hpp b/boost/network/protocol/http/client/pimpl.hpp
index d62be32ce..01c77ca70 100644
--- a/boost/network/protocol/http/client/pimpl.hpp
+++ b/boost/network/protocol/http/client/pimpl.hpp
@@ -74,10 +74,12 @@ struct basic_client_impl
                     optional<string_type> const& private_key_file,
                     optional<string_type> const& ciphers,
                     optional<string_type> const& sni_hostname, long ssl_options,
-                    std::shared_ptr<boost::asio::io_service> service, int timeout)
-      : base_type(cache_resolved, follow_redirect, always_verify_peer, timeout,
-                  service, certificate_filename, verify_path, certificate_file,
-                  private_key_file, ciphers, sni_hostname, ssl_options) {}
+                    std::shared_ptr<boost::asio::io_service> service, int timeout,
+                    bool remove_chunk_markers)
+      : base_type(cache_resolved, follow_redirect, always_verify_peer, timeout, 
+                  remove_chunk_markers, service, certificate_filename, verify_path,
+                  certificate_file, private_key_file, ciphers, sni_hostname,
+                  ssl_options) {}
 
   ~basic_client_impl() = default;
 };
diff --git a/boost/network/protocol/http/policies/async_connection.hpp b/boost/network/protocol/http/policies/async_connection.hpp
index 990103ce5..69b0800b6 100644
--- a/boost/network/protocol/http/policies/async_connection.hpp
+++ b/boost/network/protocol/http/policies/async_connection.hpp
@@ -40,7 +40,7 @@ struct async_connection_policy : resolver_policy<Tag>::type {
   struct connection_impl {
     connection_impl(
         bool follow_redirect, bool always_verify_peer, resolve_function resolve,
-        resolver_type& resolver, bool https, int timeout,
+        resolver_type& resolver, bool https, int timeout, bool remove_chunk_markers,
         optional<string_type> /*unused*/ const& certificate_filename,
         optional<string_type> const& verify_path,
         optional<string_type> const& certificate_file,
@@ -49,8 +49,8 @@ struct async_connection_policy : resolver_policy<Tag>::type {
         optional<string_type> const& sni_hostname, long ssl_options) {
       pimpl = impl::async_connection_base<Tag, version_major, version_minor>::
           new_connection(resolve, resolver, follow_redirect, always_verify_peer,
-                         https, timeout, certificate_filename, verify_path,
-                         certificate_file, private_key_file, ciphers,
+                         https, timeout, remove_chunk_markers, certificate_filename,
+                         verify_path, certificate_file, private_key_file, ciphers,
                          sni_hostname, ssl_options);
     }
 
@@ -87,7 +87,7 @@ struct async_connection_policy : resolver_policy<Tag>::type {
                std::uint16_t port, resolve_completion_function once_resolved) {
           this->resolve(resolver, host, port, once_resolved);
         },
-        resolver, boost::iequals(protocol_, string_type("https")), timeout_,
+        resolver, boost::iequals(protocol_, string_type("https")), timeout_, remove_chunk_markers_,
         certificate_filename, verify_path, certificate_file, private_key_file,
         ciphers, sni_hostname, ssl_options);
   }
@@ -95,13 +95,15 @@ struct async_connection_policy : resolver_policy<Tag>::type {
   void cleanup() {}
 
   async_connection_policy(bool cache_resolved, bool follow_redirect,
-                          int timeout)
+                          int timeout, bool remove_chunk_markers)
       : resolver_base(cache_resolved),
         follow_redirect_(follow_redirect),
-        timeout_(timeout) {}
+        timeout_(timeout),
+        remove_chunk_markers_(remove_chunk_markers) {}
 
   bool follow_redirect_;
   int timeout_;
+  bool remove_chunk_markers_;
 };
 
 }  // namespace http

From 32cc7a155c32bd0ce212ae551bf8105dade03865 Mon Sep 17 00:00:00 2001
From: Uche Mennel <umennel@gmx.ch>
Date: Wed, 14 Dec 2016 10:12:47 +0100
Subject: [PATCH 02/10] Applied new remove chunk markers option to tests.

---
 .../test/http/client_get_different_port_test.cpp     |  9 ++++++---
 libs/network/test/http/client_get_streaming_test.cpp |  5 ++++-
 libs/network/test/http/client_get_test.cpp           | 12 +++++++++---
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/libs/network/test/http/client_get_different_port_test.cpp b/libs/network/test/http/client_get_different_port_test.cpp
index 78e7e7d35..4592969a4 100644
--- a/libs/network/test/http/client_get_different_port_test.cpp
+++ b/libs/network/test/http/client_get_different_port_test.cpp
@@ -15,9 +15,12 @@ namespace http = boost::network::http;
 TYPED_TEST_CASE(HTTPClientTest, ClientTypes);
 
 TYPED_TEST(HTTPClientTest, GetDifferentPort) {
-  TypeParam client;
-  typename TypeParam::request r("http://www.boost.org:80/");
-  auto response_ = client.get(r);
+  using client = TypeParam;
+  typename client::options options;
+  options.remove_chunk_markers(true);
+  client client_;
+  typename TypeParam::request request("http://www.boost.org:80/");
+  auto response_ = client_.get(request);
   auto range = headers(response_)["Content-Type"];
   EXPECT_TRUE(std::begin(range) != std::end(range));
   EXPECT_NE(0, body(response_).size());
diff --git a/libs/network/test/http/client_get_streaming_test.cpp b/libs/network/test/http/client_get_streaming_test.cpp
index 0de687c82..5c7d55173 100644
--- a/libs/network/test/http/client_get_streaming_test.cpp
+++ b/libs/network/test/http/client_get_streaming_test.cpp
@@ -32,7 +32,10 @@ TYPED_TEST(HTTPClientTest, GetStreamingTest) {
   typename TypeParam::string_type dummy_body;
   body_handler handler_instance(body_string);
   {
-    TypeParam client_;
+    using client = TypeParam;    
+    typename client::options options;
+    options.remove_chunk_markers(true);
+    client client_(options);
     ASSERT_NO_THROW(response = client_.get(request, handler_instance));
     auto range = headers(response)["Content-Type"];
     ASSERT_TRUE(!boost::empty(range));
diff --git a/libs/network/test/http/client_get_test.cpp b/libs/network/test/http/client_get_test.cpp
index 133af7cb8..055333148 100644
--- a/libs/network/test/http/client_get_test.cpp
+++ b/libs/network/test/http/client_get_test.cpp
@@ -15,7 +15,9 @@ TYPED_TEST_CASE(HTTPClientTest, ClientTypes);
 TYPED_TEST(HTTPClientTest, GetTest) {
   using client = TypeParam;
   typename client::request request("http://cpp-netlib.org/");
-  client client_;
+  typename client::options options;
+  options.remove_chunk_markers(true);
+  client client_(options);
   typename client::response response;
   ASSERT_NO_THROW(response = client_.get(request));
   try {
@@ -34,7 +36,9 @@ TYPED_TEST(HTTPClientTest, GetTest) {
 TYPED_TEST(HTTPClientTest, GetHTTPSTest) {
   using client = TypeParam;
   typename client::request request("https://www.github.com/");
-  client client_;
+  typename client::options options;
+  options.remove_chunk_markers(true);
+  client client_(options);
   typename client::response response = client_.get(request);
   EXPECT_TRUE(response.status() == 200 ||
               (response.status() >= 300 && response.status() < 400));
@@ -77,7 +81,9 @@ TYPED_TEST(HTTPClientTest, TemporaryClientObjectTest) {
   using client = TypeParam;
   typename client::request request("http://cpp-netlib.org/");
   typename client::response response;
-  ASSERT_NO_THROW(response = client().get(request));
+  typename client::options options;
+  options.remove_chunk_markers(true);
+  ASSERT_NO_THROW(response = client(options).get(request));
   auto range = headers(response);
   ASSERT_TRUE(!boost::empty(range));
   try {

From 8bf074204ce373f88031291d25d4544da2a062f8 Mon Sep 17 00:00:00 2001
From: Uche Mennel <umennel@gmx.ch>
Date: Thu, 15 Dec 2016 23:43:17 +0100
Subject: [PATCH 03/10] Nicer formatting

---
 .../protocol/http/client/async_impl.hpp       |  6 ++---
 .../http/client/connection/async_base.hpp     |  6 +++--
 .../http/client/connection/async_normal.hpp   | 24 +++++++++++--------
 .../http/policies/async_connection.hpp        |  6 ++---
 4 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/boost/network/protocol/http/client/async_impl.hpp b/boost/network/protocol/http/client/async_impl.hpp
index 9c9a8e879..0c0bc80ce 100644
--- a/boost/network/protocol/http/client/async_impl.hpp
+++ b/boost/network/protocol/http/client/async_impl.hpp
@@ -41,8 +41,7 @@ struct async_client
   typedef std::function<bool(string_type&)> body_generator_function_type;
 
   async_client(bool cache_resolved, bool follow_redirect,
-               bool always_verify_peer, int timeout,
-               bool remove_chunk_markers,
+               bool always_verify_peer, int timeout, bool remove_chunk_markers,
                std::shared_ptr<boost::asio::io_service> service,
                optional<string_type> certificate_filename,
                optional<string_type> verify_path,
@@ -50,7 +49,8 @@ struct async_client
                optional<string_type> private_key_file,
                optional<string_type> ciphers,
                optional<string_type> sni_hostname, long ssl_options)
-      : connection_base(cache_resolved, follow_redirect, timeout, remove_chunk_markers),
+      : connection_base(cache_resolved, follow_redirect, timeout,
+                        remove_chunk_markers),
         service_ptr(service.get() ? service
                                   : std::make_shared<boost::asio::io_service>()),
         service_(*service_ptr),
diff --git a/boost/network/protocol/http/client/connection/async_base.hpp b/boost/network/protocol/http/client/connection/async_base.hpp
index d3e3531e8..b1c1aa67a 100644
--- a/boost/network/protocol/http/client/connection/async_base.hpp
+++ b/boost/network/protocol/http/client/connection/async_base.hpp
@@ -43,7 +43,8 @@ struct async_connection_base {
   // tag.
   static connection_ptr new_connection(
       resolve_function resolve, resolver_type &resolver, bool follow_redirect,
-      bool always_verify_peer, bool https, int timeout, bool remove_chunk_markers,
+      bool always_verify_peer, bool https, int timeout,
+      bool remove_chunk_markers,
       optional<string_type> certificate_filename = optional<string_type>(),
       optional<string_type> const &verify_path = optional<string_type>(),
       optional<string_type> certificate_file = optional<string_type>(),
@@ -59,7 +60,8 @@ struct async_connection_base {
         certificate_filename, verify_path, certificate_file, private_key_file,
         ciphers, sni_hostname, ssl_options);
     auto temp = std::make_shared<async_connection>(
-        resolver, resolve, follow_redirect, timeout, remove_chunk_markers, std::move(delegate));
+        resolver, resolve, follow_redirect, timeout, remove_chunk_markers,
+        std::move(delegate));
     BOOST_ASSERT(temp != nullptr);
     return temp;
   }
diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp
index 5fad8c82c..acc686cf2 100644
--- a/boost/network/protocol/http/client/connection/async_normal.hpp
+++ b/boost/network/protocol/http/client/connection/async_normal.hpp
@@ -40,7 +40,6 @@ namespace impl {
 
 template <class Tag>
 struct chunk_encoding_parser {
-
   chunk_encoding_parser() : state(state_t::header), chunk_size(0) {}
 
   enum state_t { header, header_end, data, data_end };
@@ -49,23 +48,27 @@ struct chunk_encoding_parser {
   size_t chunk_size;
   std::array<typename char_<Tag>::type, 1024> buffer;
 
-  void update_chunk_size(boost::iterator_range<typename std::array<typename char_<Tag>::type, 1024>::const_iterator> const& range) {
-    if (range.empty())
-        return;
+  void update_chunk_size(boost::iterator_range<typename std::array<
+    typename char_<Tag>::type, 1024>::const_iterator> const &range) {
+    if (range.empty()) 
+      return;
     std::stringstream ss;
     ss << std::hex << range;
     size_t size;
     ss >> size;
-    chunk_size = (chunk_size << (range.size()*4)) + size;
+    chunk_size = (chunk_size << (range.size() * 4)) + size;
   }
 
-  boost::iterator_range<typename std::array<typename char_<Tag>::type, 1024>::const_iterator> operator()(boost::iterator_range<typename std::array<typename char_<Tag>::type, 1024>::const_iterator> const& range) {
+  boost::iterator_range<
+    typename std::array<typename char_<Tag>::type, 1024>::const_iterator>
+  operator()(boost::iterator_range<typename std::array<
+               typename char_<Tag>::type, 1024>::const_iterator> const &range) {
     auto iter = boost::begin(range);
     auto begin = iter;
     auto pos = boost::begin(buffer);
 
-    while (iter != boost::end(range))
-      switch(state) {
+    while (iter != boost::end(range)) 
+      switch (state) {
         case state_t::header:
           iter = std::find(iter, boost::end(range), '\r');
           update_chunk_size(boost::make_iterator_range(begin, iter));
@@ -87,7 +90,8 @@ struct chunk_encoding_parser {
             ++iter;
             state = state_t::data_end;
           } else {
-            auto len = std::min(chunk_size, (size_t)std::distance(iter, boost::end(range)));
+            auto len = std::min(chunk_size,
+                                (size_t)std::distance(iter, boost::end(range)));
             begin = iter;
             iter = std::next(iter, len);
             pos = std::copy(begin, iter, pos);
@@ -96,7 +100,7 @@ struct chunk_encoding_parser {
           break;
 
         case state_t::data_end:
-          BOOST_ASSERT (*iter == '\n');
+          BOOST_ASSERT(*iter == '\n');
           ++iter;
           begin = iter;
           state = state_t::header;
diff --git a/boost/network/protocol/http/policies/async_connection.hpp b/boost/network/protocol/http/policies/async_connection.hpp
index 69b0800b6..011917ec8 100644
--- a/boost/network/protocol/http/policies/async_connection.hpp
+++ b/boost/network/protocol/http/policies/async_connection.hpp
@@ -87,9 +87,9 @@ struct async_connection_policy : resolver_policy<Tag>::type {
                std::uint16_t port, resolve_completion_function once_resolved) {
           this->resolve(resolver, host, port, once_resolved);
         },
-        resolver, boost::iequals(protocol_, string_type("https")), timeout_, remove_chunk_markers_,
-        certificate_filename, verify_path, certificate_file, private_key_file,
-        ciphers, sni_hostname, ssl_options);
+        resolver, boost::iequals(protocol_, string_type("https")), timeout_,
+        remove_chunk_markers_, certificate_filename, verify_path,
+        certificate_file, private_key_file, ciphers, sni_hostname, ssl_options);
   }
 
   void cleanup() {}

From 4a059fee150fbd907408b0cf372d6c2e15aa3415 Mon Sep 17 00:00:00 2001
From: Uche Mennel <umennel@gmx.ch>
Date: Thu, 15 Dec 2016 23:48:19 +0100
Subject: [PATCH 04/10] Use strongly typed enum.

---
 boost/network/protocol/http/client/connection/async_normal.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp
index acc686cf2..cf6aa7bb8 100644
--- a/boost/network/protocol/http/client/connection/async_normal.hpp
+++ b/boost/network/protocol/http/client/connection/async_normal.hpp
@@ -42,7 +42,7 @@ template <class Tag>
 struct chunk_encoding_parser {
   chunk_encoding_parser() : state(state_t::header), chunk_size(0) {}
 
-  enum state_t { header, header_end, data, data_end };
+  enum class state_t { header, header_end, data, data_end };
 
   state_t state;
   size_t chunk_size;

From 1289f48208cd11e16923a3875ec7a0994eb2f86e Mon Sep 17 00:00:00 2001
From: Uche Mennel <umennel@gmx.ch>
Date: Fri, 16 Dec 2016 01:03:41 +0100
Subject: [PATCH 05/10] Clarified and commented expression.

---
 boost/network/protocol/http/client/connection/async_normal.hpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp
index cf6aa7bb8..40d6c4e20 100644
--- a/boost/network/protocol/http/client/connection/async_normal.hpp
+++ b/boost/network/protocol/http/client/connection/async_normal.hpp
@@ -56,7 +56,8 @@ struct chunk_encoding_parser {
     ss << std::hex << range;
     size_t size;
     ss >> size;
-    chunk_size = (chunk_size << (range.size() * 4)) + size;
+    // New digits are appended as LSBs
+    chunk_size = (chunk_size << (range.size() * 4)) | size;
   }
 
   boost::iterator_range<

From 6dbceb27ba43409cc78e6bc2738311ae382f1dee Mon Sep 17 00:00:00 2001
From: Uche Mennel <umennel@gmx.ch>
Date: Mon, 19 Dec 2016 00:22:59 +0100
Subject: [PATCH 06/10] Removed commented lines.

---
 .../http/client/connection/async_normal.hpp   | 54 ++++++-------------
 1 file changed, 16 insertions(+), 38 deletions(-)

diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp
index 40d6c4e20..5d8f3a395 100644
--- a/boost/network/protocol/http/client/connection/async_normal.hpp
+++ b/boost/network/protocol/http/client/connection/async_normal.hpp
@@ -433,8 +433,11 @@ struct http_async_connection
 
             // The invocation of the callback is synchronous to allow us to
             // wait before scheduling another read.
-            callback(make_iterator_range(begin, end), ec);
-
+            if (this->is_chunk_encoding && remove_chunk_markers_) {
+              callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec);
+            } else {
+              callback(make_iterator_range(begin, end), ec);
+            }
             auto self = this->shared_from_this();
             delegate_->read_some(
                 boost::asio::mutable_buffers_1(this->part.data(),
@@ -473,7 +476,11 @@ struct http_async_connection
               // We call the callback function synchronously passing the error
               // condition (in this case, end of file) so that it can handle it
               // appropriately.
-              callback(make_iterator_range(begin, end), ec);
+              if (this->is_chunk_encoding && remove_chunk_markers_) {
+                callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec);
+              } else {
+                callback(make_iterator_range(begin, end), ec);
+              }
             } else {
               string_type body_string;
               if (this->is_chunk_encoding && remove_chunk_markers_) {
@@ -490,9 +497,7 @@ struct http_async_connection
                 this->body_promise.set_value(body_string);
               } else {
                 std::swap(body_string, this->partial_parsed);
-                auto it = this->part.begin();
-                std::advance(it, bytes_transferred);
-                body_string.append(this->part.begin(), it);
+                body_string.append(this->part.begin(), this->part.begin() + bytes_transferred);
                 this->body_promise.set_value(body_string);
               }
             }
@@ -514,7 +519,11 @@ struct http_async_connection
                   this->part.begin();
               typename protocol_base::buffer_type::const_iterator end = begin;
               std::advance(end, bytes_transferred);
-              callback(make_iterator_range(begin, end), ec);
+              if (this->is_chunk_encoding && remove_chunk_markers_) {
+                callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec);
+              } else {
+                callback(make_iterator_range(begin, end), ec);
+              }
               auto self = this->shared_from_this();
               delegate_->read_some(
                   boost::asio::mutable_buffers_1(this->part.data(),
@@ -577,37 +586,6 @@ struct http_async_connection
     }
   }
 
-  string_type parse_chunk_encoding(string_type& body_string) {
-    string_type body;
-    string_type crlf = "\r\n";
-
-    typename string_type::iterator begin = body_string.begin();
-    for (typename string_type::iterator iter =
-             std::search(begin, body_string.end(), crlf.begin(), crlf.end());
-         iter != body_string.end();
-         iter =
-             std::search(begin, body_string.end(), crlf.begin(), crlf.end())) {
-      string_type line(begin, iter);
-      if (line.empty()) {
-        break;
-      }
-      std::stringstream stream(line);
-      int len;
-      stream >> std::hex >> len;
-      std::advance(iter, 2);
-      if (len == 0) {
-        break;
-      }
-      if (len <= body_string.end() - iter) {
-        body.insert(body.end(), iter, iter + len);
-        std::advance(iter, len + 2);
-      }
-      begin = iter;
-    }
-
-    return body;
-  }
-
   int timeout_;
   bool remove_chunk_markers_;
   boost::asio::steady_timer timer_;

From 76b4ebf31c718b434330e2187cf4f28163656ef7 Mon Sep 17 00:00:00 2001
From: Uche Mennel <umennel@gmx.ch>
Date: Thu, 23 Mar 2017 08:38:01 +0100
Subject: [PATCH 07/10] Nicer formatting and clean ups

---
 .../http/client/connection/async_normal.hpp   | 34 ++++++++++---------
 .../http/policies/async_connection.hpp        |  6 ++--
 2 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp
index 5d8f3a395..09ffaea44 100644
--- a/boost/network/protocol/http/client/connection/async_normal.hpp
+++ b/boost/network/protocol/http/client/connection/async_normal.hpp
@@ -11,7 +11,6 @@
 
 #include <iterator>
 #include <cstdint>
-#include <iostream>
 #include <boost/algorithm/string/trim.hpp>
 #include <boost/asio/steady_timer.hpp>
 #include <boost/asio/placeholders.hpp>
@@ -48,10 +47,10 @@ struct chunk_encoding_parser {
   size_t chunk_size;
   std::array<typename char_<Tag>::type, 1024> buffer;
 
-  void update_chunk_size(boost::iterator_range<typename std::array<
-    typename char_<Tag>::type, 1024>::const_iterator> const &range) {
-    if (range.empty()) 
-      return;
+  void update_chunk_size(
+      boost::iterator_range<typename std::array<
+          typename char_<Tag>::type, 1024>::const_iterator> const &range) {
+    if (range.empty()) return;
     std::stringstream ss;
     ss << std::hex << range;
     size_t size;
@@ -61,15 +60,15 @@ struct chunk_encoding_parser {
   }
 
   boost::iterator_range<
-    typename std::array<typename char_<Tag>::type, 1024>::const_iterator>
-  operator()(boost::iterator_range<typename std::array<
-               typename char_<Tag>::type, 1024>::const_iterator> const &range) {
+      typename std::array<typename char_<Tag>::type, 1024>::const_iterator>
+      operator()(
+          boost::iterator_range<typename std::array<
+              typename char_<Tag>::type, 1024>::const_iterator> const &range) {
     auto iter = boost::begin(range);
     auto begin = iter;
     auto pos = boost::begin(buffer);
 
-    while (iter != boost::end(range)) 
-      switch (state) {
+    while (iter != boost::end(range)) switch (state) {
         case state_t::header:
           iter = std::find(iter, boost::end(range), '\r');
           update_chunk_size(boost::make_iterator_range(begin, iter));
@@ -485,19 +484,22 @@ struct http_async_connection
               string_type body_string;
               if (this->is_chunk_encoding && remove_chunk_markers_) {
                 for (size_t i = 0; i < this->partial_parsed.size(); i += 1024) {
-                  auto range = parse_chunk_encoding(
-                    boost::make_iterator_range(this->partial_parsed.data() + i, 
-                                               this->partial_parsed.data() + std::min(i+1024, this->partial_parsed.size())));
+                  auto range = parse_chunk_encoding(boost::make_iterator_range(
+                      this->partial_parsed.data() + i,
+                      this->partial_parsed.data() +
+                          std::min(i + 1024, this->partial_parsed.size())));
                   body_string.append(boost::begin(range), boost::end(range));
                 }
                 this->partial_parsed.clear();
-                auto range = parse_chunk_encoding(boost::make_iterator_range(this->part.begin(), 
-                                                                             this->part.begin() + bytes_transferred));
+                auto range = parse_chunk_encoding(boost::make_iterator_range(
+                    this->part.begin(),
+                    this->part.begin() + bytes_transferred));
                 body_string.append(boost::begin(range), boost::end(range));
                 this->body_promise.set_value(body_string);
               } else {
                 std::swap(body_string, this->partial_parsed);
-                body_string.append(this->part.begin(), this->part.begin() + bytes_transferred);
+                body_string.append(this->part.begin(),
+                                   this->part.begin() + bytes_transferred);
                 this->body_promise.set_value(body_string);
               }
             }
diff --git a/boost/network/protocol/http/policies/async_connection.hpp b/boost/network/protocol/http/policies/async_connection.hpp
index 011917ec8..64a6b287e 100644
--- a/boost/network/protocol/http/policies/async_connection.hpp
+++ b/boost/network/protocol/http/policies/async_connection.hpp
@@ -49,9 +49,9 @@ struct async_connection_policy : resolver_policy<Tag>::type {
         optional<string_type> const& sni_hostname, long ssl_options) {
       pimpl = impl::async_connection_base<Tag, version_major, version_minor>::
           new_connection(resolve, resolver, follow_redirect, always_verify_peer,
-                         https, timeout, remove_chunk_markers, certificate_filename,
-                         verify_path, certificate_file, private_key_file, ciphers,
-                         sni_hostname, ssl_options);
+                         https, timeout, remove_chunk_markers,
+                         certificate_filename, verify_path, certificate_file,
+                         private_key_file, ciphers, sni_hostname, ssl_options);
     }
 
     basic_response<Tag> send_request(string_type /*unused*/ const& method,

From e14fe25e65b65634d737ee957389597808e2d269 Mon Sep 17 00:00:00 2001
From: umennel <umennel@gmx.ch>
Date: Tue, 17 Oct 2017 16:28:33 +0200
Subject: [PATCH 08/10] Fix compilation errors with VS2015 (#762)

---
 .../protocol/http/client/connection/async_normal.hpp       | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp
index 09ffaea44..97af80fe2 100644
--- a/boost/network/protocol/http/client/connection/async_normal.hpp
+++ b/boost/network/protocol/http/client/connection/async_normal.hpp
@@ -30,6 +30,7 @@
 #include <boost/network/traits/ostream_iterator.hpp>
 #include <boost/network/version.hpp>
 #include <boost/range/algorithm/transform.hpp>
+#include <boost/range/iterator_range.hpp>
 #include <boost/throw_exception.hpp>
 
 namespace boost {
@@ -485,8 +486,10 @@ struct http_async_connection
               if (this->is_chunk_encoding && remove_chunk_markers_) {
                 for (size_t i = 0; i < this->partial_parsed.size(); i += 1024) {
                   auto range = parse_chunk_encoding(boost::make_iterator_range(
-                      this->partial_parsed.data() + i,
-                      this->partial_parsed.data() +
+                      static_cast<typename std::array<typename char_<Tag>::type, 1024>::const_iterator>(
+                          this->partial_parsed.data()) + i,
+                      static_cast<typename std::array<typename char_<Tag>::type, 1024>::const_iterator>(
+                          this->partial_parsed.data()) +
                           std::min(i + 1024, this->partial_parsed.size())));
                   body_string.append(boost::begin(range), boost::end(range));
                 }

From c8048e0497afbd3ebfc0ec2009098e3cec265c0e Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorpeshansky@github.com>
Date: Thu, 15 Mar 2018 14:12:32 -0400
Subject: [PATCH 09/10] Enable remove_chunk_markers by default. Fix a bug and
 add a comment.

---
 boost/network/protocol/http/client/options.hpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/boost/network/protocol/http/client/options.hpp b/boost/network/protocol/http/client/options.hpp
index b55be01b8..afc265abd 100644
--- a/boost/network/protocol/http/client/options.hpp
+++ b/boost/network/protocol/http/client/options.hpp
@@ -35,7 +35,7 @@ class client_options {
         io_service_(),
         always_verify_peer_(true),
         timeout_(0),
-        remove_chunk_markers_(false) {}
+        remove_chunk_markers_(true) {}
 
   client_options(client_options const& other)
       : cache_resolved_(other.cache_resolved_),
@@ -50,7 +50,7 @@ class client_options {
         io_service_(other.io_service_),
         always_verify_peer_(other.always_verify_peer_),
         timeout_(other.timeout_),
-        remove_chunk_markers_(other.remove_chunk_markers) {}
+        remove_chunk_markers_(other.remove_chunk_markers_) {}
 
   client_options& operator=(client_options other) {
     other.swap(*this);
@@ -157,7 +157,7 @@ class client_options {
     return *this;
   }
 
-  /// Set an overall timeout for HTTP requests.
+  /// Set whether we process chunked-encoded streams.
   client_options& remove_chunk_markers(bool v) {
     remove_chunk_markers_ = v;
     return *this;

From 1117da4e7dac1a6877a033f57a0053feea6817d2 Mon Sep 17 00:00:00 2001
From: Uche Mennel <umennel@gmx.ch>
Date: Sat, 31 Mar 2018 21:49:00 +0200
Subject: [PATCH 10/10] Removed setting chunk markers option to default value

---
 .../test/http/client_get_different_port_test.cpp     |  2 --
 libs/network/test/http/client_get_streaming_test.cpp |  6 ++----
 libs/network/test/http/client_get_test.cpp           | 12 +++---------
 3 files changed, 5 insertions(+), 15 deletions(-)

diff --git a/libs/network/test/http/client_get_different_port_test.cpp b/libs/network/test/http/client_get_different_port_test.cpp
index 4592969a4..1b9d24367 100644
--- a/libs/network/test/http/client_get_different_port_test.cpp
+++ b/libs/network/test/http/client_get_different_port_test.cpp
@@ -16,8 +16,6 @@ TYPED_TEST_CASE(HTTPClientTest, ClientTypes);
 
 TYPED_TEST(HTTPClientTest, GetDifferentPort) {
   using client = TypeParam;
-  typename client::options options;
-  options.remove_chunk_markers(true);
   client client_;
   typename TypeParam::request request("http://www.boost.org:80/");
   auto response_ = client_.get(request);
diff --git a/libs/network/test/http/client_get_streaming_test.cpp b/libs/network/test/http/client_get_streaming_test.cpp
index 5c7d55173..fed26f772 100644
--- a/libs/network/test/http/client_get_streaming_test.cpp
+++ b/libs/network/test/http/client_get_streaming_test.cpp
@@ -32,10 +32,8 @@ TYPED_TEST(HTTPClientTest, GetStreamingTest) {
   typename TypeParam::string_type dummy_body;
   body_handler handler_instance(body_string);
   {
-    using client = TypeParam;    
-    typename client::options options;
-    options.remove_chunk_markers(true);
-    client client_(options);
+    using client = TypeParam;
+    client client_;
     ASSERT_NO_THROW(response = client_.get(request, handler_instance));
     auto range = headers(response)["Content-Type"];
     ASSERT_TRUE(!boost::empty(range));
diff --git a/libs/network/test/http/client_get_test.cpp b/libs/network/test/http/client_get_test.cpp
index 055333148..133af7cb8 100644
--- a/libs/network/test/http/client_get_test.cpp
+++ b/libs/network/test/http/client_get_test.cpp
@@ -15,9 +15,7 @@ TYPED_TEST_CASE(HTTPClientTest, ClientTypes);
 TYPED_TEST(HTTPClientTest, GetTest) {
   using client = TypeParam;
   typename client::request request("http://cpp-netlib.org/");
-  typename client::options options;
-  options.remove_chunk_markers(true);
-  client client_(options);
+  client client_;
   typename client::response response;
   ASSERT_NO_THROW(response = client_.get(request));
   try {
@@ -36,9 +34,7 @@ TYPED_TEST(HTTPClientTest, GetTest) {
 TYPED_TEST(HTTPClientTest, GetHTTPSTest) {
   using client = TypeParam;
   typename client::request request("https://www.github.com/");
-  typename client::options options;
-  options.remove_chunk_markers(true);
-  client client_(options);
+  client client_;
   typename client::response response = client_.get(request);
   EXPECT_TRUE(response.status() == 200 ||
               (response.status() >= 300 && response.status() < 400));
@@ -81,9 +77,7 @@ TYPED_TEST(HTTPClientTest, TemporaryClientObjectTest) {
   using client = TypeParam;
   typename client::request request("http://cpp-netlib.org/");
   typename client::response response;
-  typename client::options options;
-  options.remove_chunk_markers(true);
-  ASSERT_NO_THROW(response = client(options).get(request));
+  ASSERT_NO_THROW(response = client().get(request));
   auto range = headers(response);
   ASSERT_TRUE(!boost::empty(range));
   try {