Skip to content

Commit 3e70915

Browse files
markushimarkushi
authored
Merge 56a5022 into dcafe87 
2 parents dcafe87 + 56a5022 commit 3e70915

33 files changed

+347
-125
lines changed

‎sentry-android-core/src/main/java/io/sentry/android/core/EnvelopeFileObserverIntegration.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@ public final void register(final @NotNull IHub hub, final @NotNull SentryOptions
4343
options.getEnvelopeReader(),
4444
options.getSerializer(),
4545
logger,
46-
options.getFlushTimeoutMillis());
46+
options.getFlushTimeoutMillis(),
47+
options.getMaxQueueSize());
4748

4849
observer =
4950
new EnvelopeFileObserver(path, outboxSender, logger, options.getFlushTimeoutMillis());

‎sentry-android-core/src/main/java/io/sentry/android/core/SendCachedEnvelopeIntegration.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
package io.sentry.android.core;
22

3+
import io.sentry.DataCategory;
34
import io.sentry.IConnectionStatusProvider;
45
import io.sentry.IHub;
56
import io.sentry.Integration;
67
import io.sentry.SendCachedEnvelopeFireAndForgetIntegration;
78
import io.sentry.SentryLevel;
89
import io.sentry.SentryOptions;
10+
import io.sentry.transport.RateLimiter;
911
import io.sentry.util.LazyEvaluator;
1012
import io.sentry.util.Objects;
1113
import java.io.Closeable;
@@ -28,6 +30,7 @@ final class SendCachedEnvelopeIntegration
2830
private @Nullable IConnectionStatusProvider connectionStatusProvider;
2931
private @Nullable IHub hub;
3032
private @Nullable SentryAndroidOptions options;
33+
private @Nullable SendCachedEnvelopeFireAndForgetIntegration.SendFireAndForget sender;
3134

3235
public SendCachedEnvelopeIntegration(
3336
final @NotNull SendCachedEnvelopeFireAndForgetIntegration.SendFireAndForgetFactory factory,
@@ -54,6 +57,8 @@ public void register(@NotNull IHub hub, @NotNull SentryOptions options) {
5457
connectionStatusProvider = options.getConnectionStatusProvider();
5558
connectionStatusProvider.addConnectionStatusObserver(this);
5659

60+
sender = factory.create(hub, options);
61+
5762
sendCachedEnvelopes(hub, this.options);
5863
}
5964

@@ -71,6 +76,7 @@ public void onConnectionStatusChanged(IConnectionStatusProvider.ConnectionStatus
7176
}
7277
}
7378

79+
@SuppressWarnings({"NullAway"})
7480
private synchronized void sendCachedEnvelopes(
7581
final @NotNull IHub hub, final @NotNull SentryAndroidOptions options) {
7682

@@ -81,8 +87,14 @@ private synchronized void sendCachedEnvelopes(
8187
return;
8288
}
8389

84-
final SendCachedEnvelopeFireAndForgetIntegration.SendFireAndForget sender =
85-
factory.create(hub, options);
90+
// in case there's rate limiting active, skip processing
91+
final @Nullable RateLimiter rateLimiter = hub.getRateLimiter();
92+
if (rateLimiter != null && rateLimiter.isActiveForCategory(DataCategory.All)) {
93+
options
94+
.getLogger()
95+
.log(SentryLevel.INFO, "SendCachedEnvelopeIntegration, rate limiting active.");
96+
return;
97+
}
8698

8799
if (sender == null) {
88100
options.getLogger().log(SentryLevel.ERROR, "SendCachedEnvelopeIntegration factory is null.");

‎sentry-android-core/src/test/java/io/sentry/android/core/InternalSentrySdkTest.kt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import io.sentry.protocol.Mechanism
2323
import io.sentry.protocol.SentryId
2424
import io.sentry.protocol.User
2525
import io.sentry.transport.ITransport
26+
import io.sentry.transport.RateLimiter
2627
import org.junit.runner.RunWith
2728
import org.mockito.kotlin.mock
2829
import org.mockito.kotlin.whenever
@@ -63,6 +64,10 @@ class InternalSentrySdkTest {
6364
override fun flush(timeoutMillis: Long) {
6465
// no-op
6566
}
67+
68+
override fun getRateLimiter(): RateLimiter? {
69+
return null
70+
}
6671
}
6772
}
6873
}

‎sentry-android-core/src/test/java/io/sentry/android/core/SendCachedEnvelopeIntegrationTest.kt‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import io.sentry.ILogger
77
import io.sentry.SendCachedEnvelopeFireAndForgetIntegration.SendFireAndForget
88
import io.sentry.SendCachedEnvelopeFireAndForgetIntegration.SendFireAndForgetFactory
99
import io.sentry.SentryLevel.DEBUG
10+
import io.sentry.transport.RateLimiter
1011
import io.sentry.util.LazyEvaluator
1112
import org.awaitility.kotlin.await
1213
import org.mockito.kotlin.any
@@ -192,4 +193,18 @@ class SendCachedEnvelopeIntegrationTest {
192193
sut.onConnectionStatusChanged(ConnectionStatus.NO_PERMISSION)
193194
verify(fixture.factory, times(3)).create(any(), any())
194195
}
196+
197+
@Test
198+
fun `when rate limiter is active, does not send envelopes`() {
199+
val sut = fixture.getSut(hasStartupCrashMarker = false)
200+
val rateLimiter = mock<RateLimiter> {
201+
whenever(mock.isActiveForCategory(any())).thenReturn(true)
202+
}
203+
whenever(fixture.hub.rateLimiter).thenReturn(rateLimiter)
204+
205+
sut.register(fixture.hub, fixture.options)
206+
207+
// no factory call should be done if there's rate limiting active
208+
verify(fixture.factory, never()).create(any(), any())
209+
}
195210
}

‎sentry-android-core/src/test/java/io/sentry/android/core/SessionTrackingIntegrationTest.kt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import io.sentry.TraceContext
2121
import io.sentry.UserFeedback
2222
import io.sentry.protocol.SentryId
2323
import io.sentry.protocol.SentryTransaction
24+
import io.sentry.transport.RateLimiter
2425
import org.junit.runner.RunWith
2526
import org.mockito.kotlin.mock
2627
import org.robolectric.annotation.Config
@@ -162,5 +163,9 @@ class SessionTrackingIntegrationTest {
162163
): SentryId {
163164
TODO("Not yet implemented")
164165
}
166+
167+
override fun getRateLimiter(): RateLimiter? {
168+
TODO("Not yet implemented")
169+
}
165170
}
166171
}

‎sentry-apache-http-client-5/api/sentry-apache-http-client-5.api‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ public final class io/sentry/transport/apache/ApacheHttpClientTransport : io/sen
22
public fun <init> (Lio/sentry/SentryOptions;Lio/sentry/RequestDetails;Lorg/apache/hc/client5/http/impl/async/CloseableHttpAsyncClient;Lio/sentry/transport/RateLimiter;)V
33
public fun close ()V
44
public fun flush (J)V
5+
public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
56
public fun send (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)V
67
}
78

‎sentry-apache-http-client-5/src/main/java/io/sentry/transport/apache/ApacheHttpClientTransport.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,11 @@ public void flush(long timeoutMillis) {
189189
}
190190
}
191191

192+
@Override
193+
public @NotNull RateLimiter getRateLimiter() {
194+
return rateLimiter;
195+
}
196+
192197
@Override
193198
public void close() throws IOException {
194199
options.getLogger().log(DEBUG, "Shutting down");

‎sentry/api/sentry.api‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ public final class io/sentry/EnvelopeReader : io/sentry/IEnvelopeReader {
239239
}
240240

241241
public final class io/sentry/EnvelopeSender : io/sentry/IEnvelopeSender {
242-
public fun <init> (Lio/sentry/IHub;Lio/sentry/ISerializer;Lio/sentry/ILogger;JLio/sentry/cache/IEnvelopeCache;)V
242+
public fun <init> (Lio/sentry/IHub;Lio/sentry/ISerializer;Lio/sentry/ILogger;JI)V
243243
public synthetic fun processDirectory (Ljava/io/File;)V
244244
public fun processEnvelopeFile (Ljava/lang/String;Lio/sentry/Hint;)V
245245
}
@@ -373,6 +373,7 @@ public final class io/sentry/Hub : io/sentry/IHub {
373373
public fun getBaggage ()Lio/sentry/BaggageHeader;
374374
public fun getLastEventId ()Lio/sentry/protocol/SentryId;
375375
public fun getOptions ()Lio/sentry/SentryOptions;
376+
public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
376377
public fun getSpan ()Lio/sentry/ISpan;
377378
public fun getTraceparent ()Lio/sentry/SentryTraceHeader;
378379
public fun getTransaction ()Lio/sentry/ITransaction;
@@ -422,6 +423,7 @@ public final class io/sentry/HubAdapter : io/sentry/IHub {
422423
public static fun getInstance ()Lio/sentry/HubAdapter;
423424
public fun getLastEventId ()Lio/sentry/protocol/SentryId;
424425
public fun getOptions ()Lio/sentry/SentryOptions;
426+
public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
425427
public fun getSpan ()Lio/sentry/ISpan;
426428
public fun getTraceparent ()Lio/sentry/SentryTraceHeader;
427429
public fun getTransaction ()Lio/sentry/ITransaction;
@@ -515,6 +517,7 @@ public abstract interface class io/sentry/IHub {
515517
public abstract fun getBaggage ()Lio/sentry/BaggageHeader;
516518
public abstract fun getLastEventId ()Lio/sentry/protocol/SentryId;
517519
public abstract fun getOptions ()Lio/sentry/SentryOptions;
520+
public abstract fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
518521
public abstract fun getSpan ()Lio/sentry/ISpan;
519522
public abstract fun getTraceparent ()Lio/sentry/SentryTraceHeader;
520523
public abstract fun getTransaction ()Lio/sentry/ITransaction;
@@ -608,6 +611,7 @@ public abstract interface class io/sentry/ISentryClient {
608611
public abstract fun captureUserFeedback (Lio/sentry/UserFeedback;)V
609612
public abstract fun close ()V
610613
public abstract fun flush (J)V
614+
public abstract fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
611615
public abstract fun isEnabled ()Z
612616
}
613617

@@ -910,6 +914,7 @@ public final class io/sentry/NoOpHub : io/sentry/IHub {
910914
public static fun getInstance ()Lio/sentry/NoOpHub;
911915
public fun getLastEventId ()Lio/sentry/protocol/SentryId;
912916
public fun getOptions ()Lio/sentry/SentryOptions;
917+
public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
913918
public fun getSpan ()Lio/sentry/ISpan;
914919
public fun getTraceparent ()Lio/sentry/SentryTraceHeader;
915920
public fun getTransaction ()Lio/sentry/ITransaction;
@@ -1070,7 +1075,7 @@ public final class io/sentry/OptionsContainer {
10701075
}
10711076

10721077
public final class io/sentry/OutboxSender : io/sentry/IEnvelopeSender {
1073-
public fun <init> (Lio/sentry/IHub;Lio/sentry/IEnvelopeReader;Lio/sentry/ISerializer;Lio/sentry/ILogger;J)V
1078+
public fun <init> (Lio/sentry/IHub;Lio/sentry/IEnvelopeReader;Lio/sentry/ISerializer;Lio/sentry/ILogger;JI)V
10741079
public synthetic fun processDirectory (Ljava/io/File;)V
10751080
public fun processEnvelopeFile (Ljava/lang/String;Lio/sentry/Hint;)V
10761081
}
@@ -1516,6 +1521,7 @@ public final class io/sentry/SentryClient : io/sentry/ISentryClient {
15161521
public fun captureUserFeedback (Lio/sentry/UserFeedback;)V
15171522
public fun close ()V
15181523
public fun flush (J)V
1524+
public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
15191525
public fun isEnabled ()Z
15201526
}
15211527

@@ -2472,7 +2478,6 @@ public final class io/sentry/TypeCheckHint {
24722478
public static final field OKHTTP_RESPONSE Ljava/lang/String;
24732479
public static final field OPEN_FEIGN_REQUEST Ljava/lang/String;
24742480
public static final field OPEN_FEIGN_RESPONSE Ljava/lang/String;
2475-
public static final field SENTRY_CACHED_ENVELOPE_FILE_PATH Ljava/lang/String;
24762481
public static final field SENTRY_DART_SDK_NAME Ljava/lang/String;
24772482
public static final field SENTRY_DOTNET_SDK_NAME Ljava/lang/String;
24782483
public static final field SENTRY_EVENT_DROP_REASON Ljava/lang/String;
@@ -2761,6 +2766,10 @@ public abstract interface class io/sentry/hints/DiskFlushNotification {
27612766
public abstract fun markFlushed ()V
27622767
}
27632768

2769+
public abstract interface class io/sentry/hints/Enqueable {
2770+
public abstract fun markEnqueued ()V
2771+
}
2772+
27642773
public final class io/sentry/hints/EventDropReason : java/lang/Enum {
27652774
public static final field MULTITHREADED_DEDUPLICATION Lio/sentry/hints/EventDropReason;
27662775
public static fun valueOf (Ljava/lang/String;)Lio/sentry/hints/EventDropReason;
@@ -4138,6 +4147,7 @@ public final class io/sentry/transport/AsyncHttpTransport : io/sentry/transport/
41384147
public fun <init> (Lio/sentry/transport/QueuedThreadPoolExecutor;Lio/sentry/SentryOptions;Lio/sentry/transport/RateLimiter;Lio/sentry/transport/ITransportGate;Lio/sentry/transport/HttpConnection;)V
41394148
public fun close ()V
41404149
public fun flush (J)V
4150+
public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
41414151
public fun send (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)V
41424152
}
41434153

@@ -4152,6 +4162,7 @@ public abstract interface class io/sentry/transport/ICurrentDateProvider {
41524162

41534163
public abstract interface class io/sentry/transport/ITransport : java/io/Closeable {
41544164
public abstract fun flush (J)V
4165+
public abstract fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
41554166
public fun send (Lio/sentry/SentryEnvelope;)V
41564167
public abstract fun send (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)V
41574168
}
@@ -4173,6 +4184,7 @@ public final class io/sentry/transport/NoOpTransport : io/sentry/transport/ITran
41734184
public fun close ()V
41744185
public fun flush (J)V
41754186
public static fun getInstance ()Lio/sentry/transport/NoOpTransport;
4187+
public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
41764188
public fun send (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)V
41774189
}
41784190

@@ -4185,7 +4197,7 @@ public final class io/sentry/transport/RateLimiter {
41854197
public fun <init> (Lio/sentry/SentryOptions;)V
41864198
public fun <init> (Lio/sentry/transport/ICurrentDateProvider;Lio/sentry/SentryOptions;)V
41874199
public fun filter (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)Lio/sentry/SentryEnvelope;
4188-
public fun getRateLimitedUntilFor (Ljava/lang/String;)Ljava/util/Date;
4200+
public fun isActiveForCategory (Lio/sentry/DataCategory;)Z
41894201
public fun updateRetryAfterLimits (Ljava/lang/String;Ljava/lang/String;I)V
41904202
}
41914203

@@ -4203,6 +4215,7 @@ public final class io/sentry/transport/StdoutTransport : io/sentry/transport/ITr
42034215
public fun <init> (Lio/sentry/ISerializer;)V
42044216
public fun close ()V
42054217
public fun flush (J)V
4218+
public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
42064219
public fun send (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)V
42074220
}
42084221

‎sentry/src/main/java/io/sentry/DirectoryProcessor.java‎

Lines changed: 57 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,23 +3,37 @@
33
import static io.sentry.SentryLevel.ERROR;
44

55
import io.sentry.hints.Cached;
6+
import io.sentry.hints.Enqueable;
67
import io.sentry.hints.Flushable;
78
import io.sentry.hints.Retryable;
89
import io.sentry.hints.SubmissionResult;
10+
import io.sentry.transport.RateLimiter;
911
import io.sentry.util.HintUtils;
1012
import java.io.File;
13+
import java.util.Queue;
1114
import java.util.concurrent.CountDownLatch;
1215
import java.util.concurrent.TimeUnit;
1316
import org.jetbrains.annotations.NotNull;
17+
import org.jetbrains.annotations.Nullable;
1418

1519
abstract class DirectoryProcessor {
1620

21+
private static final long ENVELOPE_PROCESSING_DELAY = 100L;
22+
private final @NotNull IHub hub;
1723
private final @NotNull ILogger logger;
1824
private final long flushTimeoutMillis;
19-
20-
DirectoryProcessor(final @NotNull ILogger logger, final long flushTimeoutMillis) {
25+
private final Queue<String> processedEnvelopes;
26+
27+
DirectoryProcessor(
28+
final @NotNull IHub hub,
29+
final @NotNull ILogger logger,
30+
final long flushTimeoutMillis,
31+
final int maxQueueSize) {
32+
this.hub = hub;
2133
this.logger = logger;
2234
this.flushTimeoutMillis = flushTimeoutMillis;
35+
this.processedEnvelopes =
36+
SynchronizedQueue.synchronizedQueue(new CircularFifoQueue<>(maxQueueSize));
2337
}
2438

2539
public void processDirectory(final @NotNull File directory) {
@@ -60,14 +74,36 @@ public void processDirectory(final @NotNull File directory) {
6074
continue;
6175
}
6276

63-
logger.log(SentryLevel.DEBUG, "Processing file: %s", file.getAbsolutePath());
77+
final String filePath = file.getAbsolutePath();
78+
// if envelope has already been submitted into the transport queue, we don't process it
79+
// again
80+
if (processedEnvelopes.contains(filePath)) {
81+
logger.log(
82+
SentryLevel.DEBUG,
83+
"File '%s' has already been processed so it will not be processed again.",
84+
filePath);
85+
continue;
86+
}
87+
88+
// in case there's rate limiting active, skip processing
89+
final @Nullable RateLimiter rateLimiter = hub.getRateLimiter();
90+
if (rateLimiter != null && rateLimiter.isActiveForCategory(DataCategory.All)) {
91+
logger.log(SentryLevel.INFO, "DirectoryProcessor, rate limiting active.");
92+
return;
93+
}
94+
95+
logger.log(SentryLevel.DEBUG, "Processing file: %s", filePath);
6496

6597
final SendCachedEnvelopeHint cachedHint =
66-
new SendCachedEnvelopeHint(flushTimeoutMillis, logger);
98+
new SendCachedEnvelopeHint(flushTimeoutMillis, logger, filePath, processedEnvelopes);
6799

68100
final Hint hint = HintUtils.createWithTypeCheckHint(cachedHint);
69-
hint.set(TypeCheckHint.SENTRY_CACHED_ENVELOPE_FILE_PATH, file.getAbsolutePath());
70101
processFile(file, hint);
102+
103+
// a short delay between processing envelopes to avoid bursting our server and hitting
104+
// another rate limit https://develop.sentry.dev/sdk/features/#additional-capabilities
105+
// InterruptedException will be handled by the outer try-catch
106+
Thread.sleep(ENVELOPE_PROCESSING_DELAY);
71107
}
72108
} catch (Throwable e) {
73109
logger.log(SentryLevel.ERROR, e, "Failed processing '%s'", directory.getAbsolutePath());
@@ -79,16 +115,24 @@ public void processDirectory(final @NotNull File directory) {
79115
protected abstract boolean isRelevantFileName(String fileName);
80116

81117
private static final class SendCachedEnvelopeHint
82-
implements Cached, Retryable, SubmissionResult, Flushable {
118+
implements Cached, Retryable, SubmissionResult, Flushable, Enqueable {
83119
boolean retry = false;
84120
boolean succeeded = false;
85121

86122
private final CountDownLatch latch;
87123
private final long flushTimeoutMillis;
88124
private final @NotNull ILogger logger;
89-
90-
public SendCachedEnvelopeHint(final long flushTimeoutMillis, final @NotNull ILogger logger) {
125+
private final @NotNull String filePath;
126+
private final @NotNull Queue<String> processedEnvelopes;
127+
128+
public SendCachedEnvelopeHint(
129+
final long flushTimeoutMillis,
130+
final @NotNull ILogger logger,
131+
final @NotNull String filePath,
132+
final @NotNull Queue<String> processedEnvelopes) {
91133
this.flushTimeoutMillis = flushTimeoutMillis;
134+
this.filePath = filePath;
135+
this.processedEnvelopes = processedEnvelopes;
92136
this.latch = new CountDownLatch(1);
93137
this.logger = logger;
94138
}
@@ -124,5 +168,10 @@ public void setResult(boolean succeeded) {
124168
public boolean isSuccess() {
125169
return succeeded;
126170
}
171+
172+
@Override
173+
public void markEnqueued() {
174+
processedEnvelopes.add(filePath);
175+
}
127176
}
128177
}

0 commit comments

Comments
 (0)