Improve SentryLogBatcher flush logic (#3211)

Author: denrase

CHANGELOG.md

@@ -23,6 +23,7 @@
 - Call options.log for structured logs ([#3187](https://github.com/getsentry/sentry-dart/pull/3187))
 - Remove async usage from `FlutterErrorIntegration` ([#3202](https://github.com/getsentry/sentry-dart/pull/3202))
 - Tag all spans during app start with start type info ([#3190](https://github.com/getsentry/sentry-dart/pull/3190))
+- Improve `SentryLogBatcher` flush logic ([#3211](https://github.com/getsentry/sentry-dart/pull/3187))
 
 ### Dependencies
 

packages/dart/lib/src/sentry_envelope.dart

@@ -1,4 +1,5 @@
 import 'dart:convert';
+import 'dart:typed_data';
 
 import 'client_reports/client_report.dart';
 import 'protocol.dart';
@@ -9,6 +10,7 @@ import 'sentry_item_type.dart';
 import 'sentry_options.dart';
 import 'sentry_trace_context_header.dart';
 import 'utils.dart';
+import 'package:meta/meta.dart';
 
 /// Class representation of `Envelope` file.
 class SentryEnvelope {
@@ -96,6 +98,37 @@ class SentryEnvelope {
     );
   }
 
+  /// Create a [SentryEnvelope] containing raw log data payload.
+  /// This is used by the log batcher to send pre-encoded log batches.
+  @internal
+  factory SentryEnvelope.fromLogsData(
+    List<List<int>> encodedLogs,
+    SdkVersion sdkVersion,
+  ) {
+    // Create the payload in the format expected by Sentry
+    // Format: {"items": [log1, log2, ...]}
+    final builder = BytesBuilder(copy: false);
+    builder.add(utf8.encode('{"items":['));
+    for (int i = 0; i < encodedLogs.length; i++) {
+      if (i > 0) {
+        builder.add(utf8.encode(','));
+      }
+      builder.add(encodedLogs[i]);
+    }
+    builder.add(utf8.encode(']}'));
+
+    return SentryEnvelope(
+      SentryEnvelopeHeader(
+        null,
+        sdkVersion,
+      ),
+      [
+        SentryEnvelopeItem.fromLogsData(
+            builder.takeBytes(), encodedLogs.length),
+      ],
+    );
+  }
+
   /// Stream binary data representation of `Envelope` file encoded.
   Stream<List<int>> envelopeStream(SentryOptions options) async* {
     yield utf8JsonEncoder.convert(header.toJson());

packages/dart/lib/src/sentry_envelope_item.dart

@@ -6,6 +6,7 @@ import 'sentry_attachment/sentry_attachment.dart';
 import 'sentry_envelope_item_header.dart';
 import 'sentry_item_type.dart';
 import 'utils.dart';
+import 'package:meta/meta.dart';
 
 /// Item holding header information and JSON encoded data.
 class SentryEnvelopeItem {
@@ -78,6 +79,21 @@ class SentryEnvelopeItem {
     );
   }
 
+  /// Create a [SentryEnvelopeItem] which holds pre-encoded log data.
+  /// This is used by the log batcher to send pre-encoded log batches.
+  @internal
+  factory SentryEnvelopeItem.fromLogsData(List<int> payload, int logsCount) {
+    return SentryEnvelopeItem(
+      SentryEnvelopeItemHeader(
+        SentryItemType.log,
+        itemCount: logsCount,
+        contentType: 'application/vnd.sentry.items.log+json',
+      ),
+      () => payload,
+      originalObject: null,
+    );
+  }
+
   /// Header with info about type and length of data in bytes.
   final SentryEnvelopeItemHeader header;
 

packages/dart/lib/src/sentry_log_batcher.dart

@@ -1,52 +1,97 @@
 import 'dart:async';
-import 'sentry_envelope.dart';
 import 'sentry_options.dart';
 import 'protocol/sentry_log.dart';
+import 'protocol/sentry_level.dart';
+import 'sentry_envelope.dart';
+import 'utils.dart';
 import 'package:meta/meta.dart';
 
 @internal
 class SentryLogBatcher {
-  SentryLogBatcher(this._options, {Duration? flushTimeout, int? maxBufferSize})
-      : _flushTimeout = flushTimeout ?? Duration(seconds: 5),
-        _maxBufferSize = maxBufferSize ?? 100;
+  SentryLogBatcher(
+    this._options, {
+    Duration? flushTimeout,
+    int? maxBufferSizeBytes,
+  })  : _flushTimeout = flushTimeout ?? Duration(seconds: 5),
+        _maxBufferSizeBytes = maxBufferSizeBytes ??
+            1024 * 1024; // 1MB default per BatchProcessor spec
 
   final SentryOptions _options;
   final Duration _flushTimeout;
-  final int _maxBufferSize;
+  final int _maxBufferSizeBytes;
 
-  final _logBuffer = <SentryLog>[];
+  // Store encoded log data instead of raw logs to avoid re-serialization
+  final List<List<int>> _encodedLogs = [];
+  int _encodedLogsSize = 0;
 
   Timer? _flushTimer;
 
+  /// Adds a log to the buffer.
   void addLog(SentryLog log) {
-    _logBuffer.add(log);
+    try {
+      final encodedLog = utf8JsonEncoder.convert(log.toJson());
 
-    _flushTimer?.cancel();
+      _encodedLogs.add(encodedLog);
+      _encodedLogsSize += encodedLog.length;
 
-    if (_logBuffer.length >= _maxBufferSize) {
-      return flush();
-    } else {
-      _flushTimer = Timer(_flushTimeout, flush);
+      // Flush if size threshold is reached
+      if (_encodedLogsSize >= _maxBufferSizeBytes) {
+        // Buffer size exceeded, flush immediately
+        _performFlushLogs();
+      } else if (_flushTimer == null) {
+        // Start timeout only when first item is added
+        _startTimer();
+      }
+      // Note: We don't restart the timer on subsequent additions per spec
+    } catch (error) {
+      _options.log(
+        SentryLevel.error,
+        'Failed to encode log: $error',
+      );
     }
   }
 
+  /// Flushes the buffer immediately, sending all buffered logs.
   void flush() {
+    _performFlushLogs();
+  }
+
+  void _startTimer() {
+    _flushTimer = Timer(_flushTimeout, () {
+      _options.log(
+        SentryLevel.debug,
+        'SentryLogBatcher: Timer fired, calling performCaptureLogs().',
+      );
+      _performFlushLogs();
+    });
+  }
+
+  void _performFlushLogs() {
+    // Reset timer state first
     _flushTimer?.cancel();
     _flushTimer = null;
 
-    final logs = List<SentryLog>.from(_logBuffer);
-    _logBuffer.clear();
+    // Reset buffer on function exit
+    final logsToSend = List<List<int>>.from(_encodedLogs);
+    _encodedLogs.clear();
+    _encodedLogsSize = 0;
 
-    if (logs.isEmpty) {
+    if (logsToSend.isEmpty) {
+      _options.log(
+        SentryLevel.debug,
+        'SentryLogBatcher: No logs to flush.',
+      );
       return;
     }
 
-    final envelope = SentryEnvelope.fromLogs(
-      logs,
-      _options.sdk,
-    );
-
-    // TODO: Make sure the Android SDK understands the log envelope type.
-    _options.transport.send(envelope);
+    try {
+      final envelope = SentryEnvelope.fromLogsData(logsToSend, _options.sdk);
+      _options.transport.send(envelope);
+    } catch (error) {
+      _options.log(
+        SentryLevel.error,
+        'Failed to send batched logs: $error',
+      );
+    }
   }
 }

packages/dart/test/sentry_envelope_item_test.dart

@@ -130,7 +130,32 @@ void main() {
 
       expect(sut.header.contentType, 'application/vnd.sentry.items.log+json');
       expect(sut.header.type, SentryItemType.log);
+      expect(sut.header.itemCount, 2);
       expect(actualData, expectedData);
     });
+
+    test('fromLogsData', () async {
+      final payload =
+          utf8.encode('{"items":[{"test":"data1"},{"test":"data2"}]');
+      final logsCount = 2;
+
+      final sut = SentryEnvelopeItem.fromLogsData(payload, logsCount);
+
+      expect(sut.header.contentType, 'application/vnd.sentry.items.log+json');
+      expect(sut.header.type, SentryItemType.log);
+      expect(sut.header.itemCount, logsCount);
+
+      final actualData = await sut.dataFactory();
+      expect(actualData, payload);
+    });
+
+    test('fromLogsData null original object', () async {
+      final payload = utf8.encode('{"items":[{"test":"data"}]}');
+      final logsCount = 1;
+
+      final sut = SentryEnvelopeItem.fromLogsData(payload, logsCount);
+
+      expect(sut.originalObject, null);
+    });
   });
 }

packages/dart/test/sentry_envelope_test.dart

@@ -173,6 +173,42 @@ void main() {
       expect(actualItemData, expectedItemData);
     });
 
+    test('fromLogsData', () async {
+      final encodedLogs = [
+        utf8.encode(
+            '{"timestamp":"2023-01-01T00:00:00.000Z","level":"info","body":"test1","attributes":{}}'),
+        utf8.encode(
+            '{"timestamp":"2023-01-01T00:00:01.000Z","level":"info","body":"test2","attributes":{}}'),
+      ];
+
+      final sdkVersion =
+          SdkVersion(name: 'fixture-name', version: 'fixture-version');
+      final sut = SentryEnvelope.fromLogsData(encodedLogs, sdkVersion);
+
+      expect(sut.header.eventId, null);
+      expect(sut.header.sdkVersion, sdkVersion);
+      expect(sut.items.length, 1);
+
+      final expectedEnvelopeItem = SentryEnvelopeItem.fromLogsData(
+        // The envelope should create the final payload with {"items": [...]} wrapper
+        utf8.encode('{"items":[') +
+            encodedLogs[0] +
+            utf8.encode(',') +
+            encodedLogs[1] +
+            utf8.encode(']}'),
+        2, // logsCount
+      );
+
+      expect(sut.items[0].header.contentType,
+          expectedEnvelopeItem.header.contentType);
+      expect(sut.items[0].header.type, expectedEnvelopeItem.header.type);
+      expect(sut.items[0].header.itemCount, 2);
+
+      final actualItem = await sut.items[0].dataFactory();
+      final expectedItem = await expectedEnvelopeItem.dataFactory();
+      expect(actualItem, expectedItem);
+    });
+
     test('max attachment size', () async {
       final attachment = SentryAttachment.fromLoader(
         loader: () => Uint8List.fromList([1, 2, 3, 4]),