Http/2 hpack encoding

src/Servers/Kestrel/Core/src/Internal/Http2/HPackHeaderWriter.cs

@@ -87,7 +87,7 @@ private static bool EncodeStatusHeader(int statusCode, DynamicHPackEncoder hpack
                 default:
                     const string name = ":status";
                     var value = StatusCodes.ToStatusString(statusCode);
-                    return hpackEncoder.EncodeHeader(buffer, H2StaticTable.Status200, HeaderEncodingHint.Index, name, value, out length);
+                    return hpackEncoder.EncodeHeader(buffer, H2StaticTable.Status200, HeaderEncodingHint.Index, name, value, valueEncoding: null, out length);
             }
         }
 
@@ -99,6 +99,9 @@ private static bool EncodeHeadersCore(DynamicHPackEncoder hpackEncoder, Http2Hea
                 var staticTableId = headersEnumerator.HPackStaticTableId;
                 var name = headersEnumerator.Current.Key;
                 var value = headersEnumerator.Current.Value;
+                var valueEncoding =
+                    ReferenceEquals(headersEnumerator.EncodingSelector, KestrelServerOptions.DefaultHeaderEncodingSelector)
+                    ? null : headersEnumerator.EncodingSelector(name);
 
                 var hint = ResolveHeaderEncodingHint(staticTableId, name);
 
@@ -108,6 +111,7 @@ private static bool EncodeHeadersCore(DynamicHPackEncoder hpackEncoder, Http2Hea
                     hint,
                     name,
                     value,
+                    valueEncoding,
                     out var headerLength))
                 {
                     // If the header wasn't written, and no headers have been written, then the header is too large.

src/Servers/Kestrel/Core/src/Internal/Http2/Http2HeaderEnumerator.cs

@@ -1,9 +1,11 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
 using System.Collections;
 using System.Collections.Generic;
 using System.Net.Http.HPack;
+using System.Text;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
 using Microsoft.Extensions.Primitives;
 
@@ -25,6 +27,8 @@ private enum HeadersType : byte
         private bool _hasMultipleValues;
         private KnownHeaderType _knownHeaderType;
 
+        public Func<string, Encoding?> EncodingSelector { get; set; } = KestrelServerOptions.DefaultHeaderEncodingSelector;
+
         public int HPackStaticTableId => GetResponseHeaderStaticTableId(_knownHeaderType);
         public KeyValuePair<string, string> Current { get; private set; }
         object IEnumerator.Current => Current;
@@ -35,13 +39,15 @@ public Http2HeadersEnumerator()
 
         public void Initialize(HttpResponseHeaders headers)
         {
+            EncodingSelector = headers.EncodingSelector;
             _headersEnumerator = headers.GetEnumerator();
             _headersType = HeadersType.Headers;
             _hasMultipleValues = false;
         }
 
         public void Initialize(HttpResponseTrailers headers)
         {
+            EncodingSelector = headers.EncodingSelector;
             _trailersEnumerator = headers.GetEnumerator();
             _headersType = HeadersType.Trailers;
             _hasMultipleValues = false;

src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs

@@ -1959,6 +1959,7 @@ public async Task ResponseHeaders_WithNonAscii_Throws()
             await InitializeConnectionAsync(async context =>
             {
                 Assert.Throws<InvalidOperationException>(() => context.Response.Headers.Append("Custom你好Name", "Custom Value"));
+                Assert.Throws<InvalidOperationException>(() => context.Response.ContentType = "Custom 你好 Type");
                 Assert.Throws<InvalidOperationException>(() => context.Response.Headers.Append("CustomName", "Custom 你好 Value"));
                 await context.Response.WriteAsync("Hello World");
             });
@@ -1999,14 +2000,15 @@ public async Task ResponseHeaders_WithNonAsciiAndCustomEncoder_Works()
             await InitializeConnectionAsync(async context =>
             {
                 Assert.Throws<InvalidOperationException>(() => context.Response.Headers.Append("Custom你好Name", "Custom Value"));
+                context.Response.ContentType = "Custom 你好 Type";
                 context.Response.Headers.Append("CustomName", "Custom 你好 Value");
                 await context.Response.WriteAsync("Hello World");
             });
 
             await StartStreamAsync(1, _browserRequestHeaders, endStream: true);
 
             var headersFrame = await ExpectAsync(Http2FrameType.HEADERS,
-                withLength: 32,
+                withLength: 84,
                 withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS,
                 withStreamId: 1);
 
@@ -2024,9 +2026,10 @@ await ExpectAsync(Http2FrameType.DATA,
 
             _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: true, handler: this);
 
-            Assert.Equal(3, _decodedHeaders.Count);
+            Assert.Equal(4, _decodedHeaders.Count);
             Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase);
             Assert.Equal("200", _decodedHeaders[HeaderNames.Status]);
+            Assert.Equal("Custom 你好 Type", _decodedHeaders[HeaderNames.ContentType]);
             Assert.Equal("Custom 你好 Value", _decodedHeaders["CustomName"]);
         }
 

src/Shared/Hpack/DynamicHPackEncoder.cs

@@ -4,6 +4,7 @@
 
 #nullable enable
 using System.Diagnostics;
+using System.Text;
 
 namespace System.Net.Http.HPack
 {
@@ -63,7 +64,8 @@ public bool EnsureDynamicTableSizeUpdate(Span<byte> buffer, out int length)
             return true;
         }
 
-        public bool EncodeHeader(Span<byte> buffer, int staticTableIndex, HeaderEncodingHint encodingHint, string name, string value, out int bytesWritten)
+        public bool EncodeHeader(Span<byte> buffer, int staticTableIndex, HeaderEncodingHint encodingHint, string name, string value,
+            Encoding? valueEncoding, out int bytesWritten)
         {
             Debug.Assert(!_pendingTableSizeUpdate, "Dynamic table size update should be encoded before headers.");
 
@@ -73,30 +75,30 @@ public bool EncodeHeader(Span<byte> buffer, int staticTableIndex, HeaderEncoding
                 int index = ResolveDynamicTableIndex(staticTableIndex, name);
 
                 return index == -1
-                    ? HPackEncoder.EncodeLiteralHeaderFieldNeverIndexingNewName(name, value, buffer, out bytesWritten)
-                    : HPackEncoder.EncodeLiteralHeaderFieldNeverIndexing(index, value, buffer, out bytesWritten);
+                    ? HPackEncoder.EncodeLiteralHeaderFieldNeverIndexingNewName(name, value, valueEncoding, buffer, out bytesWritten)
+                    : HPackEncoder.EncodeLiteralHeaderFieldNeverIndexing(index, value, valueEncoding, buffer, out bytesWritten);
             }
 
             // No dynamic table. Only use the static table.
             if (!_allowDynamicCompression || _maxHeaderTableSize == 0 || encodingHint == HeaderEncodingHint.IgnoreIndex)
             {
                 return staticTableIndex == -1
-                    ? HPackEncoder.EncodeLiteralHeaderFieldWithoutIndexingNewName(name, value, buffer, out bytesWritten)
-                    : HPackEncoder.EncodeLiteralHeaderFieldWithoutIndexing(staticTableIndex, value, buffer, out bytesWritten);
+                    ? HPackEncoder.EncodeLiteralHeaderFieldWithoutIndexingNewName(name, value, valueEncoding, buffer, out bytesWritten)
+                    : HPackEncoder.EncodeLiteralHeaderFieldWithoutIndexing(staticTableIndex, value, valueEncoding, buffer, out bytesWritten);
             }
 
             // Header is greater than the maximum table size.
             // Don't attempt to add dynamic header as all existing dynamic headers will be removed.
-            if (HeaderField.GetLength(name.Length, value.Length) > _maxHeaderTableSize)
+            if (HeaderField.GetLength(name.Length, value.Length) > _maxHeaderTableSize) // TODO: Check the encoded value length
             {
                 int index = ResolveDynamicTableIndex(staticTableIndex, name);
 
                 return index == -1
-                    ? HPackEncoder.EncodeLiteralHeaderFieldWithoutIndexingNewName(name, value, buffer, out bytesWritten)
-                    : HPackEncoder.EncodeLiteralHeaderFieldWithoutIndexing(index, value, buffer, out bytesWritten);
+                    ? HPackEncoder.EncodeLiteralHeaderFieldWithoutIndexingNewName(name, value, valueEncoding, buffer, out bytesWritten)
+                    : HPackEncoder.EncodeLiteralHeaderFieldWithoutIndexing(index, value, valueEncoding, buffer, out bytesWritten);
             }
 
-            return EncodeDynamicHeader(buffer, staticTableIndex, name, value, out bytesWritten);
+            return EncodeDynamicHeader(buffer, staticTableIndex, name, value, valueEncoding, out bytesWritten);
         }
 
         private int ResolveDynamicTableIndex(int staticTableIndex, string name)
@@ -110,7 +112,8 @@ private int ResolveDynamicTableIndex(int staticTableIndex, string name)
             return CalculateDynamicTableIndex(name);
         }
 
-        private bool EncodeDynamicHeader(Span<byte> buffer, int staticTableIndex, string name, string value, out int bytesWritten)
+        private bool EncodeDynamicHeader(Span<byte> buffer, int staticTableIndex, string name, string value, Encoding? valueEncoding,
+            out int bytesWritten)
         {
             EncoderHeaderEntry? headerField = GetEntry(name, value);
             if (headerField != null)
@@ -126,8 +129,8 @@ private bool EncodeDynamicHeader(Span<byte> buffer, int staticTableIndex, string
 
                 int index = ResolveDynamicTableIndex(staticTableIndex, name);
                 bool success = index == -1
-                    ? HPackEncoder.EncodeLiteralHeaderFieldIndexingNewName(name, value, buffer, out bytesWritten)
-                    : HPackEncoder.EncodeLiteralHeaderFieldIndexing(index, value, buffer, out bytesWritten);
+                    ? HPackEncoder.EncodeLiteralHeaderFieldIndexingNewName(name, value, valueEncoding, buffer, out bytesWritten)
+                    : HPackEncoder.EncodeLiteralHeaderFieldIndexing(index, value, valueEncoding, buffer, out bytesWritten);
 
                 if (success)
                 {

src/Shared/Http2cat/HPackHeaderWriter.cs

@@ -85,7 +85,7 @@ private static bool EncodeHeaders(IEnumerator<KeyValuePair<string, string>> head
 
         private static bool EncodeHeader(string name, string value, Span<byte> buffer, out int length)
         {
-            return HPackEncoder.EncodeLiteralHeaderFieldWithoutIndexingNewName(name, value, buffer, out length);
+            return HPackEncoder.EncodeLiteralHeaderFieldWithoutIndexingNewName(name, value, valueEncoding: null, buffer, out length);
         }
     }
 }

src/Shared/runtime/Http2/Hpack/HPackEncoder.cs

@@ -78,7 +78,7 @@ public static bool EncodeStatusHeader(int statusCode, Span<byte> destination, ou
         }
 
         /// <summary>Encodes a "Literal Header Field without Indexing".</summary>
-        public static bool EncodeLiteralHeaderFieldWithoutIndexing(int index, string value, Span<byte> destination, out int bytesWritten)
+        public static bool EncodeLiteralHeaderFieldWithoutIndexing(int index, string value, Encoding? valueEncoding, Span<byte> destination, out int bytesWritten)
         {
             // From https://tools.ietf.org/html/rfc7541#section-6.2.2
             // ------------------------------------------------------
@@ -97,7 +97,7 @@ public static bool EncodeLiteralHeaderFieldWithoutIndexing(int index, string val
                 if (IntegerEncoder.Encode(index, 4, destination, out int indexLength))
                 {
                     Debug.Assert(indexLength >= 1);
-                    if (EncodeStringLiteral(value, valueEncoding: null, destination.Slice(indexLength), out int nameLength))
+                    if (EncodeStringLiteral(value, valueEncoding, destination.Slice(indexLength), out int nameLength))
                     {
                         bytesWritten = indexLength + nameLength;
                         return true;
@@ -110,7 +110,7 @@ public static bool EncodeLiteralHeaderFieldWithoutIndexing(int index, string val
         }
 
         /// <summary>Encodes a "Literal Header Field never Indexing".</summary>
-        public static bool EncodeLiteralHeaderFieldNeverIndexing(int index, string value, Span<byte> destination, out int bytesWritten)
+        public static bool EncodeLiteralHeaderFieldNeverIndexing(int index, string value, Encoding? valueEncoding, Span<byte> destination, out int bytesWritten)
         {
             // From https://tools.ietf.org/html/rfc7541#section-6.2.3
             // ------------------------------------------------------
@@ -129,7 +129,7 @@ public static bool EncodeLiteralHeaderFieldNeverIndexing(int index, string value
                 if (IntegerEncoder.Encode(index, 4, destination, out int indexLength))
                 {
                     Debug.Assert(indexLength >= 1);
-                    if (EncodeStringLiteral(value, valueEncoding: null, destination.Slice(indexLength), out int nameLength))
+                    if (EncodeStringLiteral(value, valueEncoding, destination.Slice(indexLength), out int nameLength))
                     {
                         bytesWritten = indexLength + nameLength;
                         return true;
@@ -142,7 +142,7 @@ public static bool EncodeLiteralHeaderFieldNeverIndexing(int index, string value
         }
 
         /// <summary>Encodes a "Literal Header Field with Indexing".</summary>
-        public static bool EncodeLiteralHeaderFieldIndexing(int index, string value, Span<byte> destination, out int bytesWritten)
+        public static bool EncodeLiteralHeaderFieldIndexing(int index, string value, Encoding? valueEncoding, Span<byte> destination, out int bytesWritten)
         {
             // From https://tools.ietf.org/html/rfc7541#section-6.2.2
             // ------------------------------------------------------
@@ -161,7 +161,7 @@ public static bool EncodeLiteralHeaderFieldIndexing(int index, string value, Spa
                 if (IntegerEncoder.Encode(index, 6, destination, out int indexLength))
                 {
                     Debug.Assert(indexLength >= 1);
-                    if (EncodeStringLiteral(value, valueEncoding: null, destination.Slice(indexLength), out int nameLength))
+                    if (EncodeStringLiteral(value, valueEncoding, destination.Slice(indexLength), out int nameLength))
                     {
                         bytesWritten = indexLength + nameLength;
                         return true;
@@ -209,7 +209,7 @@ public static bool EncodeLiteralHeaderFieldWithoutIndexing(int index, Span<byte>
         }
 
         /// <summary>Encodes a "Literal Header Field with Indexing - New Name".</summary>
-        public static bool EncodeLiteralHeaderFieldIndexingNewName(string name, string value, Span<byte> destination, out int bytesWritten)
+        public static bool EncodeLiteralHeaderFieldIndexingNewName(string name, string value, Encoding? valueEncoding, Span<byte> destination, out int bytesWritten)
         {
             // From https://tools.ietf.org/html/rfc7541#section-6.2.2
             // ------------------------------------------------------
@@ -226,11 +226,11 @@ public static bool EncodeLiteralHeaderFieldIndexingNewName(string name, string v
             // | Value String (Length octets)  |
             // +-------------------------------+
 
-            return EncodeLiteralHeaderNewNameCore(0x40, name, value, destination, out bytesWritten);
+            return EncodeLiteralHeaderNewNameCore(0x40, name, value, valueEncoding, destination, out bytesWritten);
         }
 
         /// <summary>Encodes a "Literal Header Field without Indexing - New Name".</summary>
-        public static bool EncodeLiteralHeaderFieldWithoutIndexingNewName(string name, string value, Span<byte> destination, out int bytesWritten)
+        public static bool EncodeLiteralHeaderFieldWithoutIndexingNewName(string name, string value, Encoding? valueEncoding, Span<byte> destination, out int bytesWritten)
         {
             // From https://tools.ietf.org/html/rfc7541#section-6.2.2
             // ------------------------------------------------------
@@ -247,11 +247,11 @@ public static bool EncodeLiteralHeaderFieldWithoutIndexingNewName(string name, s
             // | Value String (Length octets)  |
             // +-------------------------------+
 
-            return EncodeLiteralHeaderNewNameCore(0, name, value, destination, out bytesWritten);
+            return EncodeLiteralHeaderNewNameCore(0, name, value, valueEncoding, destination, out bytesWritten);
         }
 
         /// <summary>Encodes a "Literal Header Field never Indexing - New Name".</summary>
-        public static bool EncodeLiteralHeaderFieldNeverIndexingNewName(string name, string value, Span<byte> destination, out int bytesWritten)
+        public static bool EncodeLiteralHeaderFieldNeverIndexingNewName(string name, string value, Encoding? valueEncoding, Span<byte> destination, out int bytesWritten)
         {
             // From https://tools.ietf.org/html/rfc7541#section-6.2.3
             // ------------------------------------------------------
@@ -268,16 +268,16 @@ public static bool EncodeLiteralHeaderFieldNeverIndexingNewName(string name, str
             // | Value String (Length octets)  |
             // +-------------------------------+
 
-            return EncodeLiteralHeaderNewNameCore(0x10, name, value, destination, out bytesWritten);
+            return EncodeLiteralHeaderNewNameCore(0x10, name, value, valueEncoding, destination, out bytesWritten);
         }
 
-        private static bool EncodeLiteralHeaderNewNameCore(byte mask, string name, string value, Span<byte> destination, out int bytesWritten)
+        private static bool EncodeLiteralHeaderNewNameCore(byte mask, string name, string value, Encoding? valueEncoding, Span<byte> destination, out int bytesWritten)
         {
             if ((uint)destination.Length >= 3)
             {
                 destination[0] = mask;
                 if (EncodeLiteralHeaderName(name, destination.Slice(1), out int nameLength) &&
-                    EncodeStringLiteral(value, valueEncoding: null, destination.Slice(1 + nameLength), out int valueLength))
+                    EncodeStringLiteral(value, valueEncoding, destination.Slice(1 + nameLength), out int valueLength))
                 {
                     bytesWritten = 1 + nameLength + valueLength;
                     return true;
@@ -643,7 +643,7 @@ public static byte[] EncodeLiteralHeaderFieldWithoutIndexingToAllocatedArray(int
 #endif
             while (true)
             {
-                if (EncodeLiteralHeaderFieldWithoutIndexing(index, value, span, out int length))
+                if (EncodeLiteralHeaderFieldWithoutIndexing(index, value, valueEncoding: null, span, out int length))
                 {
                     return span.Slice(0, length).ToArray();
                 }