HttpResponseHeaders.CopyToFast move directly to next rather than if branching

src/Servers/Kestrel/Core/src/Internal/Http/HttpResponseHeaders.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Buffers;
+using System.Diagnostics.CodeAnalysis;
 using System.IO.Pipelines;
 using System.Collections;
 using System.Collections.Generic;
@@ -67,11 +68,18 @@ private static long ParseContentLength(string value)
             return parsed;
         }
 
+        [DoesNotReturn]
         private static void ThrowInvalidContentLengthException(string value)
         {
             throw new InvalidOperationException(CoreStrings.FormatInvalidContentLength_InvalidNumber(value));
         }
 
+        [DoesNotReturn]
+        private static void ThrowInvalidHeaderBits()
+        {
+            throw new InvalidOperationException("Invalid Header Bits");
+        }
+
         [MethodImpl(MethodImplOptions.NoInlining)]
         private void SetValueUnknown(string key, StringValues value)
         {

src/Servers/Kestrel/perf/Microbenchmarks/InMemoryTransportBenchmark.cs

@@ -24,10 +24,10 @@ public class InMemoryTransportBenchmark
     {
         private const string _plaintextExpectedResponse =
             "HTTP/1.1 200 OK\r\n" +
+            "Content-Length: 13\r\n" +
             "Date: Fri, 02 Mar 2018 18:37:05 GMT\r\n" +
             "Content-Type: text/plain\r\n" +
             "Server: Kestrel\r\n" +
-            "Content-Length: 13\r\n" +
             "\r\n" +
             "Hello, World!";
 

src/Servers/Kestrel/perf/Microbenchmarks/ResponseHeadersWritingBenchmark.cs

@@ -17,6 +17,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Microbenchmarks
 {
     public class ResponseHeadersWritingBenchmark
     {
+        private const int Iterations = 1000;
+
         private static readonly byte[] _bytesServer = Encoding.ASCII.GetBytes("\r\nServer: " + Constants.ServerName);
         private static readonly byte[] _helloWorldPayload = Encoding.ASCII.GetBytes("Hello, World!");
 
@@ -32,11 +34,12 @@ public class ResponseHeadersWritingBenchmark
             BenchmarkTypes.PlaintextChunked,
             BenchmarkTypes.PlaintextWithCookie,
             BenchmarkTypes.PlaintextChunkedWithCookie,
-            BenchmarkTypes.LiveAspNet
+            BenchmarkTypes.LiveAspNet,
+            BenchmarkTypes.Common
         )]
         public BenchmarkTypes Type { get; set; }
 
-        [Benchmark]
+        [Benchmark(OperationsPerInvoke = Iterations)]
         public void Output()
         {
             switch (Type)
@@ -56,6 +59,9 @@ public void Output()
                 case BenchmarkTypes.LiveAspNet:
                     LiveAspNet();
                     break;
+                case BenchmarkTypes.Common:
+                    Common();
+                    break;
             }
         }
 
@@ -66,7 +72,11 @@ private void TechEmpowerPlaintext()
             responseHeaders.ContentLength = _helloWorldPayload.Length;
 
             var writer = new BufferWriter<PipeWriter>(_writer);
-            _responseHeaders.CopyTo(ref writer);
+
+            for (var i = 0; i < Iterations; i++)
+            {
+                _responseHeaders.CopyTo(ref writer);
+            }
         }
 
         private void PlaintextChunked()
@@ -75,7 +85,11 @@ private void PlaintextChunked()
             responseHeaders.ContentType = "text/plain";
 
             var writer = new BufferWriter<PipeWriter>(_writer);
-            _responseHeaders.CopyTo(ref writer);
+
+            for (var i = 0; i < Iterations; i++)
+            {
+                _responseHeaders.CopyTo(ref writer);
+            }
         }
 
         private void LiveAspNet()
@@ -88,7 +102,45 @@ private void LiveAspNet()
             _responseHeadersDict["X-Powered-By"] = "ASP.NET";
 
             var writer = new BufferWriter<PipeWriter>(_writer);
-            _responseHeaders.CopyTo(ref writer);
+
+            for (var i = 0; i < Iterations; i++)
+            {
+                _responseHeaders.CopyTo(ref writer);
+            }
+        }
+
+        private void Common()
+        {
+            var responseHeaders = _responseHeadersDict;
+            responseHeaders.ContentType = "text/css";
+            responseHeaders.ContentLength = 421;
+
+            responseHeaders.Connection = "Close";
+            responseHeaders.CacheControl = "public, max-age=30672000";
+            responseHeaders.Vary = "Accept-Encoding";
+            responseHeaders.ContentEncoding = "gzip";
+            responseHeaders.Expires = "Fri, 12 Jan 2018 22:01:55 GMT";
+            responseHeaders.LastModified = "Wed, 22 Jun 2016 20:08:29 GMT";
+            responseHeaders.SetCookie = "prov=20629ccd-8b0f-e8ef-2935-cd26609fc0bc; __qca=P0-1591065732-1479167353442; _ga=GA1.2.1298898376.1479167354; _gat=1; sgt=id=9519gfde_3347_4762_8762_df51458c8ec2; acct=t=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric&s=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric";
+            responseHeaders.ETag = "\"54ef7954-1078\"";
+            responseHeaders.TransferEncoding = "chunked";
+            responseHeaders.ContentLanguage = "en-gb";
+            responseHeaders.Upgrade = "websocket";
+            responseHeaders.Via = "1.1 varnish";
+            responseHeaders.AccessControlAllowOrigin = "*";
+            responseHeaders.AccessControlAllowCredentials = "true";
+            responseHeaders.AccessControlExposeHeaders = "Client-Protocol, Content-Length, Content-Type, X-Bandwidth-Est, X-Bandwidth-Est2, X-Bandwidth-Est-Comp, X-Bandwidth-Avg, X-Walltime-Ms, X-Sequence-Num";
+
+            var dateHeaderValues = _dateHeaderValueManager.GetDateHeaderValues();
+            _responseHeaders.SetRawDate(dateHeaderValues.String, dateHeaderValues.Bytes);
+            _responseHeaders.SetRawServer("Kestrel", _bytesServer);
+
+            var writer = new BufferWriter<PipeWriter>(_writer);
+
+            for (var i = 0; i < Iterations; i++)
+            {
+                _responseHeaders.CopyTo(ref writer);
+            }
         }
 
         private void PlaintextWithCookie()
@@ -99,7 +151,11 @@ private void PlaintextWithCookie()
             responseHeaders.ContentLength = _helloWorldPayload.Length;
 
             var writer = new BufferWriter<PipeWriter>(_writer);
-            _responseHeaders.CopyTo(ref writer);
+
+            for (var i = 0; i < Iterations; i++)
+            {
+                _responseHeaders.CopyTo(ref writer);
+            }
         }
 
         private void PlaintextChunkedWithCookie()
@@ -110,7 +166,11 @@ private void PlaintextChunkedWithCookie()
             responseHeaders.TransferEncoding = "chunked";
 
             var writer = new BufferWriter<PipeWriter>(_writer);
-            _responseHeaders.CopyTo(ref writer);
+
+            for (var i = 0; i < Iterations; i++)
+            {
+                _responseHeaders.CopyTo(ref writer);
+            }
         }
 
         [GlobalSetup]
@@ -151,7 +211,8 @@ public enum BenchmarkTypes
             PlaintextChunked,
             PlaintextWithCookie,
             PlaintextChunkedWithCookie,
-            LiveAspNet
+            LiveAspNet,
+            Common
         }
     }
 }

src/Servers/Kestrel/shared/KnownHeaders.cs

@@ -769,6 +769,7 @@ public static string GeneratedFile()
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Buffers;
 using System.IO.Pipelines;
 using System.Numerics;
@@ -1150,52 +1151,61 @@ internal void ClearInvalidH2H3Headers()
         }}
         internal unsafe void CopyToFast(ref BufferWriter<PipeWriter> output)
         {{
-            var tempBits = (ulong)_bits | (_contentLength.HasValue ? {"0x" + (1L << 63).ToString("x" , CultureInfo.InvariantCulture)}L : 0);
-            var next = 0;
-            var keyStart = 0;
-            var keyLength = 0;
-            ref readonly StringValues values = ref Unsafe.AsRef<StringValues>(null);
+            var tempBits = (ulong)_bits;
+            // Set exact next
+            var next = BitOperations.TrailingZeroCount(tempBits);
+
+            // Output Content-Length now as it isn't contained in the bit flags.
+            if (_contentLength.HasValue)
+            {{
+                output.Write(HeaderBytes.Slice(640, 18));
+                output.WriteNumeric((ulong)ContentLength.GetValueOrDefault());
+            }}
+            if (tempBits == 0)
+            {{
+                return;
+            }}
 
+            ref readonly StringValues values = ref Unsafe.AsRef<StringValues>(null);
             do
             {{
+                int keyStart;
+                int keyLength;
                 switch (next)
-                {{{Each(loop.Headers.OrderBy(h => !h.PrimaryHeader).Select((h, i) => (Header: h, Index: i)), hi => $@"
-                    case {hi.Index}: // Header: ""{hi.Header.Name}""
-                        if ({hi.Header.TestTempBit()})
+                {{{Each(loop.Headers.OrderBy(h => h.Index).Where(h => h.Identifier != "ContentLength"), header => $@"
+                    case {header.Index}: // Header: ""{header.Name}""
+                        Debug.Assert({header.TestTempBit()});{(header.EnhancedSetter == false ? $@"
+                        values = ref _headers._{header.Identifier};
+                        keyStart = {header.BytesOffset};
+                        keyLength = {header.BytesCount};" : $@"
+                        if (_headers._raw{header.Identifier} != null)
                         {{
-                            tempBits ^= {"0x" + (1L << hi.Header.Index).ToString("x" , CultureInfo.InvariantCulture)}L;{(hi.Header.Identifier != "ContentLength" ? $@"{(hi.Header.EnhancedSetter == false ? $@"
-                            values = ref _headers._{hi.Header.Identifier};
-                            keyStart = {hi.Header.BytesOffset};
-                            keyLength = {hi.Header.BytesCount};
-                            next = {hi.Index + 1};
-                            break; // OutputHeader" : $@"
-                            if (_headers._raw{hi.Header.Identifier} != null)
-                            {{
-                                output.Write(_headers._raw{hi.Header.Identifier});
-                            }}
-                            else
-                            {{
-                                values = ref _headers._{hi.Header.Identifier};
-                                keyStart = {hi.Header.BytesOffset};
-                                keyLength = {hi.Header.BytesCount};
-                                next = {hi.Index + 1};
-                                break; // OutputHeader
-                            }}")}" : $@"
-                            output.Write(HeaderBytes.Slice({hi.Header.BytesOffset}, {hi.Header.BytesCount}));
-                            output.WriteNumeric((ulong)ContentLength.GetValueOrDefault());
-                            if (tempBits == 0)
-                            {{
-                                return;
-                            }}")}
+                            // Clear and set next as not using common output.
+                            tempBits ^= {"0x" + (1L << header.Index).ToString("x", CultureInfo.InvariantCulture)}L;
+                            next = BitOperations.TrailingZeroCount(tempBits);
+                            output.Write(_headers._raw{header.Identifier});
+                            continue; // Jump to next, already output header
                         }}
-                        {(hi.Index + 1 < loop.Headers.Length ? $"goto case {hi.Index + 1};" : "return;")}")}
+                        else
+                        {{
+                            values = ref _headers._{header.Identifier};
+                            keyStart = {header.BytesOffset};
+                            keyLength = {header.BytesCount};
+                        }}")}
+                        break; // OutputHeader
+")}
                     default:
+                        ThrowInvalidHeaderBits();
                         return;
                 }}
 
                 // OutputHeader
                 {{
+                    // Clear bit
+                    tempBits ^= (1UL << next);
                     var valueCount = values.Count;
+                    Debug.Assert(valueCount > 0);
+
                     var headerKey = HeaderBytes.Slice(keyStart, keyLength);
                     for (var i = 0; i < valueCount; i++)
                     {{
@@ -1206,6 +1216,8 @@ internal unsafe void CopyToFast(ref BufferWriter<PipeWriter> output)
                             output.WriteAscii(value);
                         }}
                     }}
+                    // Set exact next
+                    next = BitOperations.TrailingZeroCount(tempBits);
                 }}
             }} while (tempBits != 0);
         }}" : "")}{(loop.ClassName == "HttpRequestHeaders" ? $@"

src/Servers/Kestrel/test/FunctionalTests/ListenHandleTests.cs

@@ -37,8 +37,8 @@ public async Task CanListenToOpenTcpSocketHandle()
 
                     await connection.Receive(
                         "HTTP/1.1 200 OK",
-                        $"Date: {server.Context.DateHeaderValue}",
                         "Content-Length: 0",
+                        $"Date: {server.Context.DateHeaderValue}",
                         "",
                         "");
                 }

src/Servers/Kestrel/test/FunctionalTests/RequestTests.cs

@@ -214,13 +214,13 @@ await connection2.Send(
                         "");
 
                     await connection1.Receive($"HTTP/1.1 200 OK",
-                        $"Date: {server.Context.DateHeaderValue}",
                         "Content-Length: 0",
+                        $"Date: {server.Context.DateHeaderValue}",
                         "",
                         "");
                     await connection2.Receive($"HTTP/1.1 200 OK",
-                        $"Date: {server.Context.DateHeaderValue}",
                         "Content-Length: 0",
+                        $"Date: {server.Context.DateHeaderValue}",
                         "",
                         "");
                 }
@@ -319,8 +319,8 @@ await connection.Send(
                     // a more critical log message.
                     await connection.Receive(
                         "HTTP/1.1 200 OK",
-                        $"Date: {server.Context.DateHeaderValue}",
                         "Content-Length: 0",
+                        $"Date: {server.Context.DateHeaderValue}",
                         "",
                         "");
 
@@ -583,9 +583,9 @@ await connection.Send(
                     await connectionClosedTcs.Task.DefaultTimeout();
 
                     await connection.ReceiveEnd($"HTTP/1.1 200 OK",
+                        "Content-Length: 0",
                         "Connection: close",
                         $"Date: {server.Context.DateHeaderValue}",
-                        "Content-Length: 0",
                         "",
                         "");
                 }
@@ -698,8 +698,8 @@ await connection.Send(
 
                     await connection.Receive(
                         "HTTP/1.1 200 OK",
-                        $"Date: {testContext.DateHeaderValue}",
                         "Content-Length: 5",
+                        $"Date: {testContext.DateHeaderValue}",
                         "",
                         "World");
 

src/Servers/Kestrel/test/FunctionalTests/ResponseTests.cs

@@ -880,6 +880,7 @@ await connection.Send(
 
                     await connection.Receive(
                         "HTTP/1.1 200 OK",
+                        "Content-Length: 0",
                         $"Date: {dateHeaderValueManager.GetDateHeaderValues().String}");
 
                     var minResponseSize = headerSize * headerCount;

src/Servers/Kestrel/test/InMemory.FunctionalTests/BadHttpRequestTests.cs

@@ -225,9 +225,9 @@ private async Task ReceiveBadRequestResponse(InMemoryConnection connection, stri
             var lines = new[]
             {
                 $"HTTP/1.1 {expectedResponseStatusCode}",
+                "Content-Length: 0",
                 "Connection: close",
                 $"Date: {expectedDateHeaderValue}",
-                "Content-Length: 0",
                 expectedAllowHeader,
                 "",
                 ""