[Exporter.Geneva] Allow custom string size limit to increase ETW buffer space efficiency

src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Unreleased
 
+* **Experimental**: Added an option for configuring a custom string size limit in
+  the MessagePack serializer. The maximum string length, in characters, can be
+  set using the `PrivatePreviewLogMessagePackStringSizeLimit=<CharCount>`
+  connection string parameter.
+  ([#2813](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/2813))
+
 ## 1.12.0
 
 Released 2025-May-06

src/OpenTelemetry.Exporter.Geneva/Internal/ConnectionStringBuilder.cs

@@ -3,6 +3,7 @@
 
 using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
+using OpenTelemetry.Exporter.Geneva.MsgPack;
 using OpenTelemetry.Exporter.Geneva.Transports;
 using OpenTelemetry.Internal;
 
@@ -81,6 +82,33 @@ public string EtwSession
     public bool PrivatePreviewEnableAFDCorrelationIdEnrichment => this.parts.TryGetValue(nameof(this.PrivatePreviewEnableAFDCorrelationIdEnrichment), out var value)
                 && bool.TrueString.Equals(value, StringComparison.OrdinalIgnoreCase);
 
+    public int PrivatePreviewLogMessagePackStringSizeLimit
+    {
+        get
+        {
+            if (!this.parts.TryGetValue(nameof(this.PrivatePreviewLogMessagePackStringSizeLimit), out var value))
+            {
+                return MessagePackSerializer.DEFAULT_STRING_SIZE_LIMIT_CHAR_COUNT;
+            }
+
+            if (!int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var sizeLimit))
+            {
+                throw new ArgumentException(
+                    $"{nameof(this.PrivatePreviewLogMessagePackStringSizeLimit)} is malformed.",
+                    nameof(this.PrivatePreviewLogMessagePackStringSizeLimit));
+            }
+
+            if (sizeLimit <= 0 || sizeLimit > MsgPackLogExporter.BUFFER_SIZE)
+            {
+                throw new ArgumentOutOfRangeException(
+                    nameof(this.PrivatePreviewLogMessagePackStringSizeLimit),
+                    $"{nameof(this.PrivatePreviewLogMessagePackStringSizeLimit)} should be greater than zero and less than or equal to {MsgPackLogExporter.BUFFER_SIZE} characters.");
+            }
+
+            return sizeLimit;
+        }
+    }
+
     public string Endpoint
     {
         get => this.ThrowIfNotExists<string>(nameof(this.Endpoint));

src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MessagePackSerializer.cs

@@ -43,12 +43,13 @@ internal static class MessagePackSerializer
     public const byte MAP32 = 0xDF;
     public const byte EXT_DATE_TIME = 0xFF;
 
+    internal const int DEFAULT_STRING_SIZE_LIMIT_CHAR_COUNT = (1 << 14) - 1; // 16 * 1024 - 1 = 16383
+
     private const int LIMIT_MIN_FIX_NEGATIVE_INT = -32;
     private const int LIMIT_MAX_FIX_STRING_LENGTH_IN_BYTES = 31;
     private const int LIMIT_MAX_STR8_LENGTH_IN_BYTES = (1 << 8) - 1; // str8 stores 2^8 - 1 bytes
     private const int LIMIT_MAX_FIX_MAP_COUNT = 15;
     private const int LIMIT_MAX_FIX_ARRAY_LENGTH = 15;
-    private const int STRING_SIZE_LIMIT_CHAR_COUNT = (1 << 14) - 1; // 16 * 1024 - 1 = 16383
 
 #if NET
     private const int MAX_STACK_ALLOC_SIZE_IN_BYTES = 256;
@@ -331,7 +332,7 @@ public static void WriteStr8Header(Span<byte> buffer, int nameStartIdx, int vali
     }
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    public static int SerializeAsciiString(byte[] buffer, int cursor, string? value)
+    public static int SerializeAsciiString(byte[] buffer, int cursor, string? value, int stringSizeLimitCharCount = DEFAULT_STRING_SIZE_LIMIT_CHAR_COUNT)
     {
         if (value == null)
         {
@@ -375,14 +376,14 @@ public static int SerializeAsciiString(byte[] buffer, int cursor, string? value)
         }
 
         cursor += 3;
-        if (cch <= STRING_SIZE_LIMIT_CHAR_COUNT)
+        if (cch <= stringSizeLimitCharCount)
         {
             cb = Encoding.ASCII.GetBytes(value, 0, cch, buffer, cursor);
             cursor += cb;
         }
         else
         {
-            cb = Encoding.ASCII.GetBytes(value, 0, STRING_SIZE_LIMIT_CHAR_COUNT - 3, buffer, cursor);
+            cb = Encoding.ASCII.GetBytes(value, 0, stringSizeLimitCharCount - 3, buffer, cursor);
             cursor += cb;
             cb += 3;
 
@@ -401,26 +402,26 @@ public static int SerializeAsciiString(byte[] buffer, int cursor, string? value)
 #if NET
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    public static int SerializeUnicodeString(byte[] buffer, int cursor, string? value)
+    public static int SerializeUnicodeString(byte[] buffer, int cursor, string? value, int stringSizeLimitCharCount = DEFAULT_STRING_SIZE_LIMIT_CHAR_COUNT)
     {
-        return value == null ? SerializeNull(buffer, cursor) : SerializeUnicodeString(buffer, cursor, value.AsSpan());
+        return value == null ? SerializeNull(buffer, cursor) : SerializeUnicodeString(buffer, cursor, value.AsSpan(), stringSizeLimitCharCount);
     }
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    public static int SerializeUnicodeString(byte[] buffer, int cursor, ReadOnlySpan<char> value)
+    public static int SerializeUnicodeString(byte[] buffer, int cursor, ReadOnlySpan<char> value, int stringSizeLimitCharCount = DEFAULT_STRING_SIZE_LIMIT_CHAR_COUNT)
     {
         var start = cursor;
         var cch = value.Length;
         int cb;
         cursor += 3;
-        if (cch <= STRING_SIZE_LIMIT_CHAR_COUNT)
+        if (cch <= stringSizeLimitCharCount)
         {
             cb = Encoding.UTF8.GetBytes(value.Slice(0, cch), buffer.AsSpan(cursor));
             cursor += cb;
         }
         else
         {
-            cb = Encoding.UTF8.GetBytes(value.Slice(0, STRING_SIZE_LIMIT_CHAR_COUNT - 3), buffer.AsSpan(cursor));
+            cb = Encoding.UTF8.GetBytes(value.Slice(0, stringSizeLimitCharCount - 3), buffer.AsSpan(cursor));
             cursor += cb;
             cb += 3;
 
@@ -439,7 +440,7 @@ public static int SerializeUnicodeString(byte[] buffer, int cursor, ReadOnlySpan
 #else
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    public static int SerializeUnicodeString(byte[] buffer, int cursor, string? value)
+    public static int SerializeUnicodeString(byte[] buffer, int cursor, string? value, int stringSizeLimitCharCount = DEFAULT_STRING_SIZE_LIMIT_CHAR_COUNT)
     {
         if (value == null)
         {
@@ -450,14 +451,14 @@ public static int SerializeUnicodeString(byte[] buffer, int cursor, string? valu
         var cch = value.Length;
         int cb;
         cursor += 3;
-        if (cch <= STRING_SIZE_LIMIT_CHAR_COUNT)
+        if (cch <= stringSizeLimitCharCount)
         {
             cb = Encoding.UTF8.GetBytes(value, 0, cch, buffer, cursor);
             cursor += cb;
         }
         else
         {
-            cb = Encoding.UTF8.GetBytes(value, 0, STRING_SIZE_LIMIT_CHAR_COUNT - 3, buffer, cursor);
+            cb = Encoding.UTF8.GetBytes(value, 0, stringSizeLimitCharCount - 3, buffer, cursor);
             cursor += cb;
             cb += 3;
 

src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs

@@ -17,9 +17,9 @@ namespace OpenTelemetry.Exporter.Geneva.MsgPack;
 
 internal sealed class MsgPackLogExporter : MsgPackExporter, IDisposable
 {
-    internal static readonly ThreadLocal<byte[]> Buffer = new();
+    public const int BUFFER_SIZE = 65360; // the maximum ETW payload (inclusive)
 
-    private const int BUFFER_SIZE = 65360; // the maximum ETW payload (inclusive)
+    internal static readonly ThreadLocal<byte[]> Buffer = new();
 
     private static readonly Action<LogRecordScope, MsgPackLogExporter> ProcessScopeForIndividualColumnsAction = OnProcessScopeForIndividualColumns;
     private static readonly Action<LogRecordScope, MsgPackLogExporter> ProcessScopeForEnvPropertiesAction = OnProcessScopeForEnvProperties;
@@ -43,6 +43,7 @@ internal sealed class MsgPackLogExporter : MsgPackExporter, IDisposable
     private readonly List<string>? prepopulatedFieldKeys;
     private readonly byte[] bufferEpilogue;
     private readonly IDataTransport dataTransport;
+    private readonly int stringFieldSizeLimitCharCount; // the maximum string size limit for MsgPack strings
 
     // This is used for Scopes
     private readonly ThreadLocal<SerializationDataForScopes> serializationData = new();
@@ -86,6 +87,7 @@ public MsgPackLogExporter(GenevaExporterOptions options)
                 throw new NotSupportedException($"Protocol '{connectionStringBuilder.Protocol}' is not supported");
         }
 
+        this.stringFieldSizeLimitCharCount = connectionStringBuilder.PrivatePreviewLogMessagePackStringSizeLimit;
         if (options.PrepopulatedFields != null)
         {
             this.prepopulatedFieldKeys = [];
@@ -302,7 +304,7 @@ internal ArraySegment<byte> SerializeLogRecord(LogRecord logRecord)
             if (entry.Key == "{OriginalFormat}")
             {
                 cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "body");
-                cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, logRecord.FormattedMessage ?? Convert.ToString(entry.Value, CultureInfo.InvariantCulture));
+                cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, logRecord.FormattedMessage ?? Convert.ToString(entry.Value, CultureInfo.InvariantCulture), this.stringFieldSizeLimitCharCount);
                 cntFields += 1;
                 bodyPopulated = true;
                 continue;
@@ -324,7 +326,7 @@ internal ArraySegment<byte> SerializeLogRecord(LogRecord logRecord)
                         namePopulated = true;
                     }
 
-                    cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key);
+                    cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key, this.stringFieldSizeLimitCharCount);
                     cursor = MessagePackSerializer.Serialize(buffer, cursor, entry.Value);
                     cntFields += 1;
                 }
@@ -339,14 +341,14 @@ internal ArraySegment<byte> SerializeLogRecord(LogRecord logRecord)
         if (!namePopulated)
         {
             cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "name");
-            cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, categoryName);
+            cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, categoryName, this.stringFieldSizeLimitCharCount);
             cntFields += 1;
         }
 
         if (!bodyPopulated && logRecord.FormattedMessage != null)
         {
             cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "body");
-            cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, logRecord.FormattedMessage);
+            cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, logRecord.FormattedMessage, this.stringFieldSizeLimitCharCount);
             cntFields += 1;
         }
 
@@ -381,7 +383,7 @@ internal ArraySegment<byte> SerializeLogRecord(LogRecord logRecord)
                 }
                 else
                 {
-                    cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key);
+                    cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key, this.stringFieldSizeLimitCharCount);
                     cursor = MessagePackSerializer.Serialize(buffer, cursor, entry.Value);
                     envPropertiesCount += 1;
                 }
@@ -412,11 +414,11 @@ internal ArraySegment<byte> SerializeLogRecord(LogRecord logRecord)
         if (logRecord.Exception != null)
         {
             cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "env_ex_type");
-            cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, logRecord.Exception.GetType().FullName);
+            cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, logRecord.Exception.GetType().FullName, this.stringFieldSizeLimitCharCount);
             cntFields += 1;
 
             cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "env_ex_msg");
-            cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, logRecord.Exception.Message);
+            cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, logRecord.Exception.Message, this.stringFieldSizeLimitCharCount);
             cntFields += 1;
 
             // The current approach relies on the existing trim
@@ -430,7 +432,7 @@ internal ArraySegment<byte> SerializeLogRecord(LogRecord logRecord)
             {
                 var exceptionStack = logRecord.Exception.ToInvariantString();
                 cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "env_ex_stack");
-                cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, exceptionStack);
+                cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, exceptionStack, this.stringFieldSizeLimitCharCount);
                 cntFields += 1;
             }
             else if (this.exportExceptionStack == ExceptionStackExportMode.ExportAsStackTraceString)
@@ -439,7 +441,7 @@ internal ArraySegment<byte> SerializeLogRecord(LogRecord logRecord)
                 if (exceptionStack != null)
                 {
                     cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "env_ex_stack");
-                    cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, exceptionStack);
+                    cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, exceptionStack, this.stringFieldSizeLimitCharCount);
                     cntFields += 1;
                 }
             }

test/OpenTelemetry.Exporter.Geneva.Tests/ConnectionStringBuilderTests.cs

@@ -240,4 +240,19 @@ public void ConnectionStringBuilder_PrivatePreviewEnableUserEvents_No_Default_Va
         builder = new ConnectionStringBuilder("PrivatePreviewEnableAFDCorrelationIdEnrichment=false");
         Assert.False(builder.PrivatePreviewEnableAFDCorrelationIdEnrichment);
     }
+
+    [Fact]
+    public void ConnectionStringBuilder_PrivatePreviewLogMessagePackStringSizeLimit()
+    {
+        var builder = new ConnectionStringBuilder("key1=value1");
+        Assert.Equal(16383, builder.PrivatePreviewLogMessagePackStringSizeLimit);
+        builder = new ConnectionStringBuilder("PrivatePreviewLogMessagePackStringSizeLimit=1024");
+        Assert.Equal(1024, builder.PrivatePreviewLogMessagePackStringSizeLimit);
+        builder = new ConnectionStringBuilder("PrivatePreviewLogMessagePackStringSizeLimit=65360");
+        Assert.Equal(65360, builder.PrivatePreviewLogMessagePackStringSizeLimit);
+        builder = new ConnectionStringBuilder("PrivatePreviewLogMessagePackStringSizeLimit=-1");
+        Assert.Throws<ArgumentOutOfRangeException>(() => builder.PrivatePreviewLogMessagePackStringSizeLimit);
+        builder = new ConnectionStringBuilder("PrivatePreviewLogMessagePackStringSizeLimit=65361");
+        Assert.Throws<ArgumentOutOfRangeException>(() => builder.PrivatePreviewLogMessagePackStringSizeLimit);
+    }
 }

test/OpenTelemetry.Exporter.Geneva.Tests/MessagePackSerializerTests.cs

@@ -32,15 +32,15 @@ private void MessagePackSerializer_TestSerialization<T>(T value)
         this.AssertBytes(MessagePack.MessagePackSerializer.Serialize(value), buffer, length);
     }
 
-    private void MessagePackSerializer_TestASCIIStringSerialization(string input)
+    private void MessagePackSerializer_TestASCIIStringSerialization(string input, int sizeLimit = (1 << 14) - 1)
     {
-        var sizeLimit = (1 << 14) - 1; // // Max length of string allowed
+        // sizeLimit is the max length of string allowed
         var buffer = new byte[64 * 1024];
-        var length = MessagePackSerializer.SerializeAsciiString(buffer, 0, input);
+        var length = MessagePackSerializer.SerializeAsciiString(buffer, 0, input, sizeLimit);
         var deserializedString = MessagePack.MessagePackSerializer.Deserialize<string>(buffer);
         if (!string.IsNullOrEmpty(input) && input.Length > sizeLimit)
         {
-            // We truncate the string using `.` in the last three characters which takes 3 bytes of memort
+            // We truncate the string using `.` in the last three characters which takes 3 bytes of memory
 #pragma warning disable CA1846 // Prefer 'AsSpan' over 'Substring'
             var byteCount = Encoding.ASCII.GetByteCount(input.Substring(0, sizeLimit - 3)) + 3;
 #pragma warning restore CA1846 // Prefer 'AsSpan' over 'Substring'
@@ -88,11 +88,11 @@ private void MessagePackSerializer_TestASCIIStringSerialization(string input)
         }
     }
 
-    private void MessagePackSerializer_TestUnicodeStringSerialization(string input)
+    private void MessagePackSerializer_TestUnicodeStringSerialization(string input, int sizeLimit = (1 << 14) - 1)
     {
-        var sizeLimit = (1 << 14) - 1; // // Max length of string allowed
+        // sizeLimit is the max length of string allowed
         var buffer = new byte[64 * 1024];
-        var length = MessagePackSerializer.SerializeUnicodeString(buffer, 0, input);
+        var length = MessagePackSerializer.SerializeUnicodeString(buffer, 0, input, sizeLimit);
 
         var deserializedString = MessagePack.MessagePackSerializer.Deserialize<string>(buffer);
         if (!string.IsNullOrEmpty(input) && input.Length > sizeLimit)
@@ -292,6 +292,9 @@ public void MessagePackSerializer_SerializeAsciiString()
         this.MessagePackSerializer_TestASCIIStringSerialization(new string('Z', (1 << 14) - 1));
         this.MessagePackSerializer_TestASCIIStringSerialization(new string('Z', 1 << 14));
 
+        this.MessagePackSerializer_TestASCIIStringSerialization(new string('Z', (1 << 15) - 1));
+        this.MessagePackSerializer_TestASCIIStringSerialization(new string('Z', 1 << 15));
+
         // Unicode special characters
         // SerializeAsciiString will encode non-ASCII characters with '?'
         Assert.Throws<EqualException>(() => this.MessagePackSerializer_TestASCIIStringSerialization("\u0418"));
@@ -328,6 +331,9 @@ public void MessagePackSerializer_SerializeUnicodeString()
         this.MessagePackSerializer_TestUnicodeStringSerialization(new string('\u0418', (1 << 14) - 1));
         this.MessagePackSerializer_TestUnicodeStringSerialization(new string('\u0418', 1 << 14));
 
+        this.MessagePackSerializer_TestUnicodeStringSerialization(new string('\u0418', (1 << 15) - 1));
+        this.MessagePackSerializer_TestUnicodeStringSerialization(new string('\u0418', 1 << 15));
+
         // Unicode regular and special characters
         this.MessagePackSerializer_TestUnicodeStringSerialization("\u0418TestString");
         this.MessagePackSerializer_TestUnicodeStringSerialization("TestString\u0418");