diff --git a/commit-status-publisher-server/src/main/java/jetbrains/buildServer/commitPublisher/Constants.java b/commit-status-publisher-server/src/main/java/jetbrains/buildServer/commitPublisher/Constants.java index 20bdcc60..208d14f1 100644 --- a/commit-status-publisher-server/src/main/java/jetbrains/buildServer/commitPublisher/Constants.java +++ b/commit-status-publisher-server/src/main/java/jetbrains/buildServer/commitPublisher/Constants.java @@ -16,6 +16,7 @@ package jetbrains.buildServer.commitPublisher; +import java.time.Duration; import jetbrains.buildServer.ssh.ServerSshKeyManager; import org.jetbrains.annotations.NotNull; @@ -62,6 +63,7 @@ public class Constants { public static final String GITHUB_OAUTH_PROVIDER_ID = "github_oauth_provider_id"; public static final String GITHUB_CUSTOM_CONTEXT_BUILD_PARAM = "teamcity.commitStatusPublisher.githubContext"; public static final String GITHUB_CONTEXT = "github_context"; + public static final String GITHUB_RATE_LIMIT_RESET_HEADER = "X-RateLimit-Reset"; public static final String BITBUCKET_PUBLISHER_ID = "bitbucketCloudPublisher"; public static final String BITBUCKET_CLOUD_USERNAME = "bitbucketUsername"; @@ -71,6 +73,8 @@ public class Constants { public static final String GITLAB_API_URL = "gitlabApiUrl"; public static final String GITLAB_TOKEN = "secure:gitlabAccessToken"; + public static final Duration RATE_LIMIT_RESET_BUFFER = Duration.ofSeconds(1); + @NotNull public String getVcsRootIdParam() { diff --git a/commit-status-publisher-server/src/main/java/jetbrains/buildServer/commitPublisher/HttpHelper.java b/commit-status-publisher-server/src/main/java/jetbrains/buildServer/commitPublisher/HttpHelper.java index b8e3d639..4aa835fa 100644 --- a/commit-status-publisher-server/src/main/java/jetbrains/buildServer/commitPublisher/HttpHelper.java +++ b/commit-status-publisher-server/src/main/java/jetbrains/buildServer/commitPublisher/HttpHelper.java @@ -19,6 +19,9 @@ import java.io.IOException; import java.net.URISyntaxException; import java.security.KeyStore; +import java.time.Duration; +import java.time.Instant; +import java.time.format.DateTimeFormatter; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -51,6 +54,7 @@ private static void call(@NotNull HttpMethod method, final AtomicReference content = new AtomicReference(); final AtomicReference code = new AtomicReference(0); final AtomicReference text = new AtomicReference(); + final AtomicReference retryDelay = new AtomicReference(); final HTTPRequestBuilder builder; try { @@ -76,6 +80,14 @@ public void consume(@NotNull final HTTPRequestBuilder.Response response) throws content.set(response.getBodyAsString()); code.set(response.getStatusCode()); text.set(response.getStatusText()); + String retryAfter = response.getHeader("Retry-After"); + if (retryAfter != null) { + try { + retryDelay.set(Duration.ofSeconds(Long.parseLong(retryAfter))); + } catch (NumberFormatException ex) { + retryDelay.set(Duration.between(Instant.now(), Instant.from(DateTimeFormatter.RFC_1123_DATE_TIME.parse(retryAfter)))); + } + } } }) .onSuccess(new HTTPRequestBuilder.ResponseConsumer() { @@ -103,7 +115,13 @@ public void consume(@NotNull final HTTPRequestBuilder.Response response) throws } } - if (processor != null) { + Duration duration = retryDelay.get(); + if (duration != null) { + try { + Thread.sleep(duration.plus(Constants.RATE_LIMIT_RESET_BUFFER).toMillis()); + } catch (InterruptedException ignored) {} + call(method, url, username, password, headers, timeout, trustStore, processor, modifier); + } else if (processor != null) { processor.processResponse(new HttpResponse(code.get(), text.get(), content.get())); } } diff --git a/commit-status-publisher-server/src/main/java/jetbrains/buildServer/commitPublisher/github/api/impl/GitHubApiImpl.java b/commit-status-publisher-server/src/main/java/jetbrains/buildServer/commitPublisher/github/api/impl/GitHubApiImpl.java index fa4ea21e..7c990bd7 100644 --- a/commit-status-publisher-server/src/main/java/jetbrains/buildServer/commitPublisher/github/api/impl/GitHubApiImpl.java +++ b/commit-status-publisher-server/src/main/java/jetbrains/buildServer/commitPublisher/github/api/impl/GitHubApiImpl.java @@ -20,6 +20,8 @@ import com.google.gson.GsonBuilder; import com.google.gson.JsonSyntaxException; import java.io.IOException; +import java.time.Duration; +import java.time.Instant; import java.util.*; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; @@ -112,6 +114,7 @@ public CombinedCommitStatus readChangeCombinedStatus(@NotNull final String repoO final AtomicReference status = new AtomicReference<>(); final AtomicReference exceptionRef = new AtomicReference<>(); + final AtomicReference retryDelay = new AtomicReference<>(); IOGuard.allowNetworkCall(() -> { myClient.get(statusUrl, authenticationCredentials(), defaultHeaders(), success -> { @@ -135,14 +138,25 @@ public CombinedCommitStatus readChangeCombinedStatus(@NotNull final String repoO } }, response -> { - logFailedResponse(method, statusUrl, null, response); - exceptionRef.set(new IOException(getErrorMessage(response, null))); + String reset = response.getHeader(Constants.GITHUB_RATE_LIMIT_RESET_HEADER); + if (reset != null) { + retryDelay.set(Duration.between(Instant.now(), Instant.ofEpochSecond(Long.parseLong(reset)))); + } else { + logFailedResponse(method, statusUrl, null, response); + exceptionRef.set(new IOException(getErrorMessage(response, null))); + } }, e -> exceptionRef.set(e)); }); final Exception ex; - if ((ex = exceptionRef.get()) != null) { + final Duration duration = retryDelay.get(); + if (duration != null) { + try { + Thread.sleep(duration.plus(Constants.RATE_LIMIT_RESET_BUFFER).toMillis()); + } catch (InterruptedException ignored) {} + return readChangeCombinedStatus(repoOwner, repoName, hash, perPage, page); + } else if ((ex = exceptionRef.get()) != null) { if (ex instanceof IOException) { throw (IOException) ex; } else { @@ -176,6 +190,7 @@ public void setChangeStatus(@NotNull final String repoOwner, LoggerUtil.logRequest(Constants.GITHUB_PUBLISHER_ID, method, url, entity); final AtomicReference exceptionRef = new AtomicReference<>(); + final AtomicReference retryDelay = new AtomicReference<>(); IOGuard.allowNetworkCall(() -> { myClient.post( url, authenticationCredentials(), defaultHeaders(), @@ -183,14 +198,25 @@ url, authenticationCredentials(), defaultHeaders(), response -> { }, response -> { - logFailedResponse(method, url, entity, response); - exceptionRef.set(new IOException(getErrorMessage(response, MSG_PROXY_OR_PERMISSIONS))); + String reset = response.getHeader(Constants.GITHUB_RATE_LIMIT_RESET_HEADER); + if (reset != null) { + retryDelay.set(Duration.between(Instant.now(), Instant.ofEpochSecond(Long.parseLong(reset)))); + } else { + logFailedResponse(method, url, entity, response); + exceptionRef.set(new IOException(getErrorMessage(response, MSG_PROXY_OR_PERMISSIONS))); + } }, e -> exceptionRef.set(e)); }); final Exception ex; - if ((ex = exceptionRef.get()) != null) { + final Duration duration = retryDelay.get(); + if (duration != null) { + try { + Thread.sleep(duration.plus(Constants.RATE_LIMIT_RESET_BUFFER).toMillis()); + } catch (InterruptedException ignored) {} + setChangeStatus(repoOwner, repoName, hash, status, targetUrl, description, context); + } else if ((ex = exceptionRef.get()) != null) { if (ex instanceof IOException) { throw (IOException) ex; } else { @@ -249,6 +275,7 @@ private T processResponse(@NotNull String uri, @NotNull final Class clazz final AtomicReference exceptionRef = new AtomicReference<>(); final AtomicReference resultRef = new AtomicReference<>(); + final AtomicReference retryDelay = new AtomicReference<>(); IOGuard.allowNetworkCall(() -> { myClient.get(uri, authenticationCredentials(), defaultHeaders(), success -> { @@ -267,8 +294,13 @@ private T processResponse(@NotNull String uri, @NotNull final Class clazz } }, error -> { - logFailedResponse(HttpMethod.GET, uri, null, error, logErrorsDebugOnly); - exceptionRef.set(new IOException(getErrorMessage(error, MSG_PROXY_OR_PERMISSIONS))); + String reset = error.getHeader(Constants.GITHUB_RATE_LIMIT_RESET_HEADER); + if (reset != null) { + retryDelay.set(Duration.between(Instant.now(), Instant.ofEpochSecond(Long.parseLong(reset)))); + } else { + logFailedResponse(HttpMethod.GET, uri, null, error, logErrorsDebugOnly); + exceptionRef.set(new IOException(getErrorMessage(error, MSG_PROXY_OR_PERMISSIONS))); + } }, e -> { exceptionRef.set(e); @@ -277,7 +309,13 @@ private T processResponse(@NotNull String uri, @NotNull final Class clazz }); final Exception ex; - if ((ex = exceptionRef.get()) != null) { + final Duration duration = retryDelay.get(); + if (duration != null) { + try { + Thread.sleep(duration.plus(Constants.RATE_LIMIT_RESET_BUFFER).toMillis()); + } catch (InterruptedException ignored) {} + return processResponse(uri, clazz, logErrorsDebugOnly); + } else if ((ex = exceptionRef.get()) != null) { if (ex instanceof IOException) { throw (IOException)ex; } else if (ex instanceof PublisherException) { @@ -350,6 +388,7 @@ public void postComment(@NotNull final String ownerName, LoggerUtil.logRequest(Constants.GITHUB_PUBLISHER_ID, method, url, entity); final AtomicReference exceptionRef = new AtomicReference<>(); + final AtomicReference retryDelay = new AtomicReference<>(); IOGuard.allowNetworkCall(() -> { myClient.post( url, authenticationCredentials(), defaultHeaders(), @@ -357,14 +396,25 @@ url, authenticationCredentials(), defaultHeaders(), response -> { }, response -> { - logFailedResponse(method, url, entity, response); - exceptionRef.set(new IOException(getErrorMessage(response, null))); + String reset = response.getHeader(Constants.GITHUB_RATE_LIMIT_RESET_HEADER); + if (reset != null) { + retryDelay.set(Duration.between(Instant.now(), Instant.ofEpochSecond(Long.parseLong(reset)))); + } else { + logFailedResponse(method, url, entity, response); + exceptionRef.set(new IOException(getErrorMessage(response, null))); + } }, e -> exceptionRef.set(e)); }); final Exception ex; - if ((ex = exceptionRef.get()) != null) { + final Duration duration = retryDelay.get(); + if (duration != null) { + try { + Thread.sleep(duration.plus(Constants.RATE_LIMIT_RESET_BUFFER).toMillis()); + } catch (InterruptedException ignored) {} + postComment(ownerName, repoName, hash, comment); + } else if ((ex = exceptionRef.get()) != null) { if (ex instanceof IOException) { throw (IOException) ex; } else {