Initial implementation of HTTP compression for Windows

Author: deeringc

Release/include/cpprest/details/http_helpers.h

@@ -27,6 +27,11 @@
 ****/
 #pragma once
 
+// CPPREST_EXCLUDE_WEBSOCKETS is a flag that now essentially means "no external dependencies". TODO: Rename
+#if defined(_WIN32) && !defined(CPPREST_EXCLUDE_WEBSOCKETS)
+#define CPPREST_HTTP_COMPRESSION
+#endif
+
 #include "cpprest/http_msg.h"
 
 namespace web { namespace http
@@ -124,4 +129,82 @@ namespace details
         _ASYNCRTIMP size_t __cdecl add_chunked_delimiters(_Out_writes_(buffer_size) uint8_t *data, _In_ size_t buffer_size, size_t bytes_read);
     }
 
+    namespace compression
+    {
+        enum class compression_algorithm : int 
+        { 
+            deflate = 15,
+            gzip = 31,
+            invalid = 9999
+        };
+
+        using data_buffer = std::vector<uint8_t>;
+
+        class stream_decompressor
+        {
+        public:
+
+            static compression_algorithm to_compression_algorithm(const utility::string_t& alg)
+            {
+                if (U("gzip") == alg)
+                {
+                    return compression_algorithm::gzip;
+                }
+                else if (U("deflate") == alg)
+                {
+                    return compression_algorithm::deflate;
+                }
+
+                return compression_algorithm::invalid;
+            }
+
+            static bool is_supported()
+            {
+#if !defined(CPPREST_HTTP_COMPRESSION) 
+                return false;
+#else
+                return true;
+#endif
+            }
+
+            _ASYNCRTIMP stream_decompressor(compression_algorithm alg);
+
+            _ASYNCRTIMP data_buffer decompress(const data_buffer& input);
+
+            _ASYNCRTIMP data_buffer decompress(const uint8_t* input, size_t input_size);
+
+            _ASYNCRTIMP bool has_error() const;
+
+        private:
+            class stream_decompressor_impl;
+            std::shared_ptr<stream_decompressor_impl> m_pimpl;
+        };
+
+        class stream_compressor
+        {
+        public:
+
+            static bool is_supported()
+            {
+#if !defined(CPPREST_HTTP_COMPRESSION) 
+                return false;
+#else
+                return true;
+#endif
+            }
+
+            _ASYNCRTIMP stream_compressor(compression_algorithm alg);
+
+            _ASYNCRTIMP data_buffer compress(const data_buffer& input, bool finish);
+
+            _ASYNCRTIMP data_buffer compress(const uint8_t* input, size_t input_size, bool finish);
+
+            _ASYNCRTIMP bool has_error() const;
+
+        private:
+            class stream_compressor_impl;
+            std::shared_ptr<stream_compressor_impl> m_pimpl;
+        };
+
+    }
 }}}

Release/include/cpprest/http_client.h

@@ -104,6 +104,7 @@ class http_client_config
 #endif
 #if defined(_WIN32) && !defined(__cplusplus_winrt)
         , m_buffer_request(false)
+        , m_request_compressed(false)
 #endif
     {
     }
@@ -301,6 +302,27 @@ class http_client_config
     {
         m_buffer_request = buffer_request;
     }
+
+    /// <summary>
+    /// Checks if requesting a compressed response is turned on, the default is off.
+    /// </summary>
+    /// <returns>True if compressed response is enabled, false otherwise</returns>
+    bool request_compressed_response() const
+    {
+        return m_request_compressed;
+    }
+
+    /// <summary>
+    /// Request that the server responds with a compressed body.
+    /// If true, in cases where the server does not support compression, this will have no effect.
+    /// The response body is internally decompressed before the consumer receives the data.
+    /// </summary>
+    /// <param name="buffer_request">True to turn on response body compression, false otherwise.</param>
+    /// <remarks>Please note there is a performance cost due to copying the request data.</remarks>
+    void set_request_compressed_response(bool request_compressed)
+    {
+        m_request_compressed = request_compressed;
+    }
 #endif
 #endif
 
@@ -375,6 +397,7 @@ class http_client_config
 #endif
 #if defined(_WIN32) && !defined(__cplusplus_winrt)
     bool m_buffer_request;
+    bool m_request_compressed;
 #endif
 };
 

Release/src/build/vs14.wod/casablanca140.wod.vcxproj

@@ -21,14 +21,17 @@
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), build.root))\Build\Release.Product.Settings" />
   <ImportGroup Label="PropertySheets">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+    <Import Project="..\..\..\..\packages\zlib.v140.windesktop.msvcstl.static.rt-dyn.1.2.8.8\build\native\zlib.v140.windesktop.msvcstl.static.rt-dyn.targets" Condition="Exists('..\..\..\..\packages\zlib.v140.windesktop.msvcstl.static.rt-dyn.1.2.8.8\build\native\zlib.v140.windesktop.msvcstl.static.rt-dyn.targets')" />
   </ImportGroup>
   <PropertyGroup Condition="'$(Configuration)' == 'Debug'">
     <DebugFileSuffix>d</DebugFileSuffix>
   </PropertyGroup>
   <PropertyGroup>
     <TargetName>$(CppRestBaseFileName)140$(DebugFileSuffix)_$(CppRestSDKVersionFileSuffix).wod</TargetName>
-    <NuGetPackageImportStamp>57c5697d</NuGetPackageImportStamp>
   </PropertyGroup>
+  <ItemGroup>
+    <None Include="packages.config" />
+  </ItemGroup>
   <ItemDefinitionGroup>
     <ClCompile>
       <PreprocessorDefinitions>CPPREST_EXCLUDE_WEBSOCKETS;_ASYNCRT_EXPORT;_PPLX_EXPORT;WIN32;_MBCS;_USRDLL;%(PreprocessorDefinitions)</PreprocessorDefinitions>
@@ -47,4 +50,10 @@
   <Import Project="..\common.vcxitems" Label="Shared" Condition="Exists('..\common.vcxitems')" />
   <Import Project="..\win32.vcxitems" Label="Shared" Condition="Exists('..\win32.vcxitems')" />
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('..\..\..\..\packages\zlib.v140.windesktop.msvcstl.static.rt-dyn.1.2.8.8\build\native\zlib.v140.windesktop.msvcstl.static.rt-dyn.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\zlib.v140.windesktop.msvcstl.static.rt-dyn.1.2.8.8\build\native\zlib.v140.windesktop.msvcstl.static.rt-dyn.targets'))" />
+  </Target>
 </Project>

Release/src/http/client/http_client_winhttp.cpp

@@ -238,6 +238,8 @@ class winhttp_request_context : public request_context
 
     memory_holder m_body_data;
 
+    std::unique_ptr<web::http::details::compression::stream_decompressor> decompressor;
+
     virtual void cleanup()
     {
         if(m_request_handle != nullptr)
@@ -587,6 +589,11 @@ class winhttp_client : public _http_client_communicator
             }
         }
 
+        if(web::http::details::compression::stream_decompressor::is_supported() && client_config().request_compressed_response())
+        {
+            msg.headers().add(web::http::header_names::accept_encoding, U("deflate, gzip"));
+        }
+
         // Add headers.
         if(!msg.headers().empty())
         {
@@ -1135,6 +1142,24 @@ class winhttp_client : public _http_client_communicator
                         }
                     }
 
+                    // If the response body is compressed we will read the encoding header and create a decompressor object which will later decompress the body
+                    utility::string_t encoding;
+                    if (web::http::details::compression::stream_decompressor::is_supported() && response.headers().match(web::http::header_names::content_encoding, encoding))
+                    {
+                        auto alg = web::http::details::compression::stream_decompressor::to_compression_algorithm(encoding);
+
+                        if (alg != web::http::details::compression::compression_algorithm::invalid)
+                        {
+                            p_request_context->decompressor = std::make_unique<web::http::details::compression::stream_decompressor>(alg);
+                        }
+                        else
+                        {
+                            utility::string_t error = U("Unsupported compression algorithm in the Content Encoding header: ");
+                            error += encoding;
+                            p_request_context->report_exception(http_exception(error));
+                        }
+                    }
+
                     // Signal that the headers are available.
                     p_request_context->complete_headers();
 
@@ -1159,12 +1184,21 @@ class winhttp_client : public _http_client_communicator
             case WINHTTP_CALLBACK_STATUS_DATA_AVAILABLE :
                 {
                     // Status information contains pointer to DWORD containing number of bytes available.
-                    DWORD num_bytes = *(PDWORD)statusInfo;
+                    const DWORD num_bytes = *(PDWORD)statusInfo;
 
                     if(num_bytes > 0)
                     {
-                        auto writebuf = p_request_context->_get_writebuffer();
-                        p_request_context->allocate_reply_space(writebuf.alloc(num_bytes), num_bytes);
+                        if (p_request_context->decompressor)
+                        {
+                            // Decompression is too slow to reliably do on this callback. Therefore we need to store it now in order to decompress it at a later stage in the flow.
+                            // However, we want to eventually use the writebuf to store the decompressed body. Therefore we'll store the compressed body as an internal allocation in the request_context
+                            p_request_context->allocate_reply_space(nullptr, num_bytes);
+                        }
+                        else
+                        {
+                            auto writebuf = p_request_context->_get_writebuffer();
+                            p_request_context->allocate_reply_space(writebuf.alloc(num_bytes), num_bytes);
+                        }
 
                         // Read in body all at once.
                         if(!WinHttpReadData(
@@ -1198,7 +1232,7 @@ class winhttp_client : public _http_client_communicator
             case WINHTTP_CALLBACK_STATUS_READ_COMPLETE :
                 {
                     // Status information length contains the number of bytes read.
-                    const DWORD bytesRead = statusInfoLength;
+                    DWORD bytesRead = statusInfoLength;
 
                     // Report progress about downloaded bytes.
                     auto progress = p_request_context->m_request._get_impl()->_progress_handler();
@@ -1220,9 +1254,39 @@ class winhttp_client : public _http_client_communicator
                         break;
                     }
 
+                    auto writebuf = p_request_context->_get_writebuffer();
+
+                    // If we have compressed data it is stored in the local allocation of the p_request_context. We will store the decompressed buffer in the external allocation of the p_request_context.
+                    if (p_request_context->decompressor)
+                    {
+                        web::http::details::compression::data_buffer decompressed = p_request_context->decompressor->decompress(p_request_context->m_body_data.get(), bytesRead);
+
+                        if (p_request_context->decompressor->has_error())
+                        {
+                            p_request_context->report_exception(std::runtime_error("Failed to decompress the response body"));
+                            return;
+                        }
+
+                        // We've decompressed this chunk of the body, need to now store it in the writebuffer.
+                        auto decompressed_size = decompressed.size();
+
+                        if (decompressed_size > 0)
+                        {
+                            auto p = writebuf.alloc(decompressed_size);
+                            p_request_context->allocate_reply_space(p, decompressed_size);
+                            std::memcpy(p_request_context->m_body_data.get(), &decompressed[0], decompressed_size);
+
+                            bytesRead = decompressed_size;
+                        }
+                        else
+                        {
+                            p_request_context->report_exception(std::runtime_error("Failed to decompress the response body, empty decompression result"));
+                            return;
+                        }
+                    }
+
                     // If the data was allocated directly from the buffer then commit, otherwise we still
                     // need to write to the response stream buffer.
-                    auto writebuf = p_request_context->_get_writebuffer();
                     if (p_request_context->is_externally_allocated())
                     {
                         writebuf.commit(bytesRead);