Skip to content

feat(logs): add Buffering and Batching #4310

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Flash0ver merged 32 commits into from
Jul 29, 2025
Merged

feat(logs): add Buffering and Batching #4310

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
d24d165
feat: add Batch Processor for Logs
Flash0ver Jun 26, 2025
aad0599
test: Batch Processor for Logs
Flash0ver Jun 26, 2025
76fcc1b
docs: Batch Processor for Logs
Flash0ver Jun 26, 2025
2ad33f6
test: fix unavailable API on TargetFramework=net48
Flash0ver Jun 27, 2025
38e1c04
test: run all Logs tests on full framework
Flash0ver Jun 27, 2025
f7a43b8
ref: remove usage of System.Threading.Lock
Flash0ver Jun 27, 2025
e6b0b74
ref: rename members for clarity
Flash0ver Jun 30, 2025
a84b78f
Merge branch 'feat/logs' into feat/logs-buffering
Flash0ver Jun 30, 2025
53c90ea
ref: delete Timer-Abstraction and change to System.Threading.Timer
Flash0ver Jul 1, 2025
6580632
ref: delete .ctor only called from tests
Flash0ver Jul 1, 2025
6e2ee9b
ref: switch Buffer-Processor to be lock-free but discarding
Flash0ver Jul 2, 2025
0774709
test: fix BatchBuffer and Tests
Flash0ver Jul 3, 2025
d9ae794
fix: flushing buffer on Timeout
Flash0ver Jul 8, 2025
7e1f5ea
feat: add Backpressure-ClientReport
Flash0ver Jul 10, 2025
365a2fb
ref: make BatchProcessor more resilient
Flash0ver Jul 11, 2025
c478391
Format code
getsentry-bot Jul 11, 2025
211beea
Merge branch 'feat/logs' into feat/logs-buffering
Flash0ver Jul 11, 2025
c699c2d
test: fix on .NET Framework
Flash0ver Jul 11, 2025
b21b537
fix: BatchBuffer flushed on Shutdown/Dispose
Flash0ver Jul 14, 2025
e8850db
ref: minimize locking
Flash0ver Jul 22, 2025
57f9ccc
Merge branch 'feat/logs' into feat/logs-buffering
Flash0ver Jul 22, 2025
c63bc53
ref: rename BatchProcessor to StructuredLogBatchProcessor
Flash0ver Jul 22, 2025
f28cc6d
ref: rename BatchBuffer to StructuredLogBatchBuffer
Flash0ver Jul 22, 2025
79ce02e
ref: remove internal options
Flash0ver Jul 23, 2025
0702796
test: ref
Flash0ver Jul 23, 2025
4e5f097
perf: update Benchmark result
Flash0ver Jul 23, 2025
1276725
ref: make SentryStructuredLogger. Flush abstract
Flash0ver Jul 24, 2025
3816fab
ref: guard an invariant of the Flush-Scope
Flash0ver Jul 24, 2025
28b6654
ref: remove unused values
Flash0ver Jul 24, 2025
1eef330
docs: improve comments
Flash0ver Jul 24, 2025
d72ca5c
perf: update Benchmark after signature change
Flash0ver Jul 25, 2025
49fefc1
ref: discard logs gracefully when Hub is (being) disposed
Flash0ver Jul 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
test: run all Logs tests on full framework
  • Loading branch information
@Flash0ver
Flash0ver committed Jun 27, 2025
commit 38e1c047e1351a50d508f2c9b42317a4d14c4a7f
4 changes: 2 additions & 2 deletions test/Sentry.Testing/JsonSerializableExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ private static string WriteToJsonString(Action<Utf8JsonWriter> writeAction, bool
public static JsonDocument ToJsonDocument(this ISentryJsonSerializable serializable, IDiagnosticLogger? logger = null) =>
WriteToJsonDocument(writer => writer.WriteSerializableValue(serializable, logger));

public static JsonDocument ToJsonDocument(this object @object, IDiagnosticLogger? logger = null) =>
WriteToJsonDocument(writer => writer.WriteDynamicValue(@object, logger));
public static JsonDocument ToJsonDocument<T>(this T @object, Action<T, Utf8JsonWriter, IDiagnosticLogger?> serialize, IDiagnosticLogger? logger = null) where T : class =>
WriteToJsonDocument(writer => serialize.Invoke(@object, writer, logger));

private static JsonDocument WriteToJsonDocument(Action<Utf8JsonWriter> writeAction)
{
Expand Down
73 changes: 41 additions & 32 deletions test/Sentry.Tests/SentryLogTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,6 @@ public void WriteTo_EnvelopeItem_MaximalSerializedSentryLog()
_output.Entries.Should().BeEmpty();
}

#if (NETCOREAPP3_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER) //System.Buffers.ArrayBufferWriter<T>
[Fact]
public void WriteTo_MessageParameters_AsAttributes()
{
Expand All @@ -268,56 +267,62 @@ public void WriteTo_MessageParameters_AsAttributes()
uint.MaxValue,
long.MinValue,
ulong.MaxValue,
#if NET5_0_OR_GREATER
nint.MinValue,
nuint.MaxValue,
#endif
1f,
2d,
3m,
true,
'c',
"string",
#if (NETCOREAPP2_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER)
KeyValuePair.Create("key", "value"),
#else
new KeyValuePair<string, string>("key", "value"),
#endif
null,
],
};

ArrayBufferWriter<byte> bufferWriter = new();
using Utf8JsonWriter writer = new(bufferWriter);
log.WriteTo(writer, _output);
writer.Flush();
var currentParameterAttributeIndex = -1;
string GetNextParameterAttributeName() => $"sentry.message.parameter.{++currentParameterAttributeIndex}";

var document = JsonDocument.Parse(bufferWriter.WrittenMemory);
var document = log.ToJsonDocument(static (obj, writer, logger) => obj.WriteTo(writer, logger), _output);
var attributes = document.RootElement.GetProperty("attributes");
Assert.Collection(attributes.EnumerateObject().ToArray(),
property => property.AssertAttributeInteger("sentry.message.parameter.0", json => json.GetSByte(), sbyte.MinValue),
property => property.AssertAttributeInteger("sentry.message.parameter.1", json => json.GetByte(), byte.MaxValue),
property => property.AssertAttributeInteger("sentry.message.parameter.2", json => json.GetInt16(), short.MinValue),
property => property.AssertAttributeInteger("sentry.message.parameter.3", json => json.GetUInt16(), ushort.MaxValue),
property => property.AssertAttributeInteger("sentry.message.parameter.4", json => json.GetInt32(), int.MinValue),
property => property.AssertAttributeInteger("sentry.message.parameter.5", json => json.GetUInt32(), uint.MaxValue),
property => property.AssertAttributeInteger("sentry.message.parameter.6", json => json.GetInt64(), long.MinValue),
property => property.AssertAttributeString("sentry.message.parameter.7", json => json.GetString(), ulong.MaxValue.ToString(NumberFormatInfo.InvariantInfo)),
property => property.AssertAttributeInteger("sentry.message.parameter.8", json => json.GetInt64(), nint.MinValue),
property => property.AssertAttributeString("sentry.message.parameter.9", json => json.GetString(), nuint.MaxValue.ToString(NumberFormatInfo.InvariantInfo)),
property => property.AssertAttributeDouble("sentry.message.parameter.10", json => json.GetSingle(), 1f),
property => property.AssertAttributeDouble("sentry.message.parameter.11", json => json.GetDouble(), 2d),
property => property.AssertAttributeString("sentry.message.parameter.12", json => json.GetString(), 3m.ToString(NumberFormatInfo.InvariantInfo)),
property => property.AssertAttributeBoolean("sentry.message.parameter.13", json => json.GetBoolean(), true),
property => property.AssertAttributeString("sentry.message.parameter.14", json => json.GetString(), "c"),
property => property.AssertAttributeString("sentry.message.parameter.15", json => json.GetString(), "string"),
property => property.AssertAttributeString("sentry.message.parameter.16", json => json.GetString(), "[key, value]")
property => property.AssertAttributeInteger(GetNextParameterAttributeName(), json => json.GetSByte(), sbyte.MinValue),
property => property.AssertAttributeInteger(GetNextParameterAttributeName(), json => json.GetByte(), byte.MaxValue),
property => property.AssertAttributeInteger(GetNextParameterAttributeName(), json => json.GetInt16(), short.MinValue),
property => property.AssertAttributeInteger(GetNextParameterAttributeName(), json => json.GetUInt16(), ushort.MaxValue),
property => property.AssertAttributeInteger(GetNextParameterAttributeName(), json => json.GetInt32(), int.MinValue),
property => property.AssertAttributeInteger(GetNextParameterAttributeName(), json => json.GetUInt32(), uint.MaxValue),
property => property.AssertAttributeInteger(GetNextParameterAttributeName(), json => json.GetInt64(), long.MinValue),
property => property.AssertAttributeString(GetNextParameterAttributeName(), json => json.GetString(), ulong.MaxValue.ToString(NumberFormatInfo.InvariantInfo)),
#if NET5_0_OR_GREATER
property => property.AssertAttributeInteger(GetNextParameterAttributeName(), json => json.GetInt64(), nint.MinValue),
property => property.AssertAttributeString(GetNextParameterAttributeName(), json => json.GetString(), nuint.MaxValue.ToString(NumberFormatInfo.InvariantInfo)),
#endif
property => property.AssertAttributeDouble(GetNextParameterAttributeName(), json => json.GetSingle(), 1f),
property => property.AssertAttributeDouble(GetNextParameterAttributeName(), json => json.GetDouble(), 2d),
property => property.AssertAttributeString(GetNextParameterAttributeName(), json => json.GetString(), 3m.ToString(NumberFormatInfo.InvariantInfo)),
property => property.AssertAttributeBoolean(GetNextParameterAttributeName(), json => json.GetBoolean(), true),
property => property.AssertAttributeString(GetNextParameterAttributeName(), json => json.GetString(), "c"),
property => property.AssertAttributeString(GetNextParameterAttributeName(), json => json.GetString(), "string"),
property => property.AssertAttributeString(GetNextParameterAttributeName(), json => json.GetString(), "[key, value]")
);
Assert.Collection(_output.Entries,
entry => entry.Message.Should().Match("*ulong*is not supported*overflow*"),
#if NET5_0_OR_GREATER
entry => entry.Message.Should().Match("*nuint*is not supported*64-bit*"),
#endif
entry => entry.Message.Should().Match("*decimal*is not supported*overflow*"),
entry => entry.Message.Should().Match("*System.Collections.Generic.KeyValuePair`2[System.String,System.String]*is not supported*ToString*"),
entry => entry.Message.Should().Match("*null*is not supported*ignored*")
);
}
#endif

#if (NETCOREAPP3_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER) //System.Buffers.ArrayBufferWriter<T>
[Fact]
public void WriteTo_Attributes_AsJson()
{
Expand All @@ -330,23 +335,24 @@ public void WriteTo_Attributes_AsJson()
log.SetAttribute("uint", uint.MaxValue);
log.SetAttribute("long", long.MinValue);
log.SetAttribute("ulong", ulong.MaxValue);
#if NET5_0_OR_GREATER
log.SetAttribute("nint", nint.MinValue);
log.SetAttribute("nuint", nuint.MaxValue);
#endif
log.SetAttribute("float", 1f);
log.SetAttribute("double", 2d);
log.SetAttribute("decimal", 3m);
log.SetAttribute("bool", true);
log.SetAttribute("char", 'c');
log.SetAttribute("string", "string");
#if (NETCOREAPP2_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER)
log.SetAttribute("object", KeyValuePair.Create("key", "value"));
#else
log.SetAttribute("object", new KeyValuePair<string, string>("key", "value"));
#endif
log.SetAttribute("null", null!);

ArrayBufferWriter<byte> bufferWriter = new();
using Utf8JsonWriter writer = new(bufferWriter);
log.WriteTo(writer, _output);
writer.Flush();

var document = JsonDocument.Parse(bufferWriter.WrittenMemory);
var document = log.ToJsonDocument(static (obj, writer, logger) => obj.WriteTo(writer, logger), _output);
var attributes = document.RootElement.GetProperty("attributes");
Assert.Collection(attributes.EnumerateObject().ToArray(),
property => property.AssertAttributeInteger("sbyte", json => json.GetSByte(), sbyte.MinValue),
Expand All @@ -357,8 +363,10 @@ public void WriteTo_Attributes_AsJson()
property => property.AssertAttributeInteger("uint", json => json.GetUInt32(), uint.MaxValue),
property => property.AssertAttributeInteger("long", json => json.GetInt64(), long.MinValue),
property => property.AssertAttributeString("ulong", json => json.GetString(), ulong.MaxValue.ToString(NumberFormatInfo.InvariantInfo)),
#if NET5_0_OR_GREATER
property => property.AssertAttributeInteger("nint", json => json.GetInt64(), nint.MinValue),
property => property.AssertAttributeString("nuint", json => json.GetString(), nuint.MaxValue.ToString(NumberFormatInfo.InvariantInfo)),
#endif
property => property.AssertAttributeDouble("float", json => json.GetSingle(), 1f),
property => property.AssertAttributeDouble("double", json => json.GetDouble(), 2d),
property => property.AssertAttributeString("decimal", json => json.GetString(), 3m.ToString(NumberFormatInfo.InvariantInfo)),
Expand All @@ -369,13 +377,14 @@ public void WriteTo_Attributes_AsJson()
);
Assert.Collection(_output.Entries,
entry => entry.Message.Should().Match("*ulong*is not supported*overflow*"),
#if NET5_0_OR_GREATER
entry => entry.Message.Should().Match("*nuint*is not supported*64-bit*"),
#endif
entry => entry.Message.Should().Match("*decimal*is not supported*overflow*"),
entry => entry.Message.Should().Match("*System.Collections.Generic.KeyValuePair`2[System.String,System.String]*is not supported*ToString*"),
entry => entry.Message.Should().Match("*null*is not supported*ignored*")
);
}
#endif
}

file static class AssertExtensions
Expand Down