diff --git a/.github/workflows/build-check.yml b/.github/workflows/build-check.yml index e8866bc6..6d957b3d 100644 --- a/.github/workflows/build-check.yml +++ b/.github/workflows/build-check.yml @@ -11,9 +11,7 @@ on: jobs: build: - runs-on: ubuntu-latest - steps: - uses: actions/checkout@v4 - name: Set up JDK 17 diff --git a/CHANGELOG.md b/CHANGELOG.md index a758d08b..d4bb3779 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Changelog All notable changes to this project will be documented in this file. +## [1.0-RC16] + +- Migrated from OkHttpClient to Apache http client (fluent interface) +- Added bill of materials for ranger, for clients who intend to use ranger. + ## [1.0-RC15] - Moved discovery bundle from https://github.com/appform-io/dropwizard-service-discovery. - Updated to dropwizard version 2.1.10 : BOM update. diff --git a/README.md b/README.md index e490c744..92a9ee8c 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,20 @@ Ranger provides two types of discovery out of the box: - Sharded service discovery with service provider node healthchecks We'll take these up, one by one. -###Build instructions +### Bill of materials +You could use the ranger-bom using the following in your dependency management. + +``` + + io.appform.ranger + ranger-bom + ${ranger.version} + pom + import + +``` + +### Build instructions - Clone the source: git clone github.com/appform-io/ranger diff --git a/pom.xml b/pom.xml index b14a9849..fc02586f 100644 --- a/pom.xml +++ b/pom.xml @@ -7,13 +7,17 @@ io.appform.ranger ranger pom - 1.0-RC15 + 1.0-RC16 + Ranger https://github.com/appform-io/ranger - Service Discovery for Java + Service Discovery for Java including dropwizard backed bundles and lightweight hub servers 2015 + ranger-bom + ranger-parent + ranger-core ranger-zookeeper ranger-http @@ -82,95 +86,17 @@ UTF-8 - 31.0.1-jre - 5.5.0 - 1.7.32 - 3.8.0 17 11 - 1.18.22 - 3.0.1u2 - - 5.8.2 - 4.1.1 - 2.0.0 - - 4.9.3 - 3.3.1 - 4.2.0 - - 2.1.10 + 3.8.0 + 3.0.0-M5 + 0.8.8 + 3.2.0 + 3.6.3 + 1.6.13 - - - - org.junit - junit-bom - ${junit.jupiter.version} - pom - import - - - io.dropwizard - dropwizard-bom - pom - import - ${dropwizard.version} - - - - - - - org.projectlombok - lombok - ${lombok.version} - provided - - - com.google.code.findbugs - annotations - ${annotations.version} - - - com.google.guava - guava - ${guava.version} - - - org.slf4j - slf4j-api - ${slf4j.version} - - - io.dropwizard - dropwizard-jackson - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - com.github.rholder - guava-retrying - ${guava-retrying.version} - - - org.awaitility - awaitility - ${awaitility.version} - test - - - @@ -186,12 +112,12 @@ org.apache.maven.plugins maven-surefire-plugin - 3.0.0-M5 + ${maven.surefire.plugin.version} org.jacoco jacoco-maven-plugin - 0.8.8 + ${jacoco.version} @@ -206,13 +132,11 @@ - - org.apache.maven.plugins maven-source-plugin - 3.2.0 + ${maven.source.plugin.version} attach-sources @@ -225,7 +149,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.6.3 + ${maven.javadoc.plugin.version} all,-missing @@ -241,7 +165,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.13 + ${nexus.staging.plguin.version} true ossrh @@ -287,20 +211,5 @@ - - - add-java-open-options-for-jdk16+ - - [16,) - - - - --add-opens java.base/java.net=ALL-UNNAMED - --add-opens java.base/sun.net=ALL-UNNAMED - - - - diff --git a/ranger-bom/pom.xml b/ranger-bom/pom.xml new file mode 100644 index 00000000..cbde2415 --- /dev/null +++ b/ranger-bom/pom.xml @@ -0,0 +1,94 @@ + + + 4.0.0 + + io.appform.ranger + ranger + 1.0-RC16 + + + ranger-bom + pom + Ranger BOM + + Bill of materials to make sure a consistent set of versions is used for Ranger modules. + + + + + + io.appform.ranger + ranger-client + ${project.version} + + + io.appform.ranger + ranger-core + ${project.version} + + + io.appform.ranger + ranger-discovery-bundle + ${project.version} + + + io.appform.ranger + ranger-zookeeper + ${project.version} + + + io.appform.ranger + ranger-zk-client + ${project.version} + + + io.appform.ranger + ranger-zk-server-bundle + ${project.version} + + + io.appform.ranger + ranger-zk-server + ${project.version} + + + io.appform.ranger + ranger-http-model + ${project.version} + + + io.appform.ranger + ranger-http + ${project.version} + + + io.appform.ranger + ranger-http-server-bundle + ${project.version} + + + io.appform.ranger + ranger-http-client + ${project.version} + + + io.appform.ranger + ranger-http-server + ${project.version} + + + io.appform.ranger + ranger-server-common + ${project.version} + + + io.appform.ranger + ranger-server-bundle + ${project.version} + + + + + \ No newline at end of file diff --git a/ranger-client/pom.xml b/ranger-client/pom.xml index 727e9c1b..ac6f64ba 100644 --- a/ranger-client/pom.xml +++ b/ranger-client/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - ranger + ranger-parent io.appform.ranger - 1.0-RC15 + 1.0-RC16 + ../ranger-parent 4.0.0 @@ -15,12 +16,10 @@ io.appform.ranger ranger-core - ${project.version} io.appform.ranger ranger-core - ${project.version} test-jar diff --git a/ranger-core/pom.xml b/ranger-core/pom.xml index 634ab4a9..a9ec2602 100644 --- a/ranger-core/pom.xml +++ b/ranger-core/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - ranger + ranger-parent io.appform.ranger - 1.0-RC15 + 1.0-RC16 + ../ranger-parent 4.0.0 @@ -17,9 +18,9 @@ - org.apache.httpcomponents - httpclient - ${httpclient.version} + org.apache.httpcomponents.client5 + httpclient5-fluent + ${http.client.version} diff --git a/ranger-core/src/main/java/io/appform/ranger/core/healthservice/monitor/sample/PingCheckMonitor.java b/ranger-core/src/main/java/io/appform/ranger/core/healthservice/monitor/sample/PingCheckMonitor.java index 3f86d17b..a33eedfd 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/healthservice/monitor/sample/PingCheckMonitor.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/healthservice/monitor/sample/PingCheckMonitor.java @@ -21,14 +21,11 @@ import io.appform.ranger.core.healthservice.monitor.RollingWindowHealthQueue; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.apache.http.HttpHost; -import org.apache.http.HttpRequest; -import org.apache.http.HttpStatus; -import org.apache.http.conn.routing.HttpRoute; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; - +import org.apache.hc.client5.http.HttpRoute; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; +import org.apache.hc.core5.http.*; import java.util.concurrent.*; /** @@ -39,7 +36,7 @@ @SuppressWarnings("unused") public class PingCheckMonitor extends IsolatedHealthMonitor { - private final HttpRequest httpRequest; + private final ClassicHttpRequest httpRequest; private final String host; private final ExecutorService executorService; private final Integer pingTimeoutInMilliseconds; @@ -58,7 +55,7 @@ public class PingCheckMonitor extends IsolatedHealthMonitor { */ public PingCheckMonitor( TimeEntity timeEntity, - HttpRequest httpRequest, + ClassicHttpRequest httpRequest, Integer pingTimeoutInMilliseconds, Integer pingWindowSize, Integer maxFailures, @@ -113,14 +110,13 @@ private HealthcheckStatus getRollingWindowHealthcheckStatus(HealthcheckStatus he private boolean healthPing() { try { log.debug("executing http HttpRequest: {}, host:{}, port:{}", httpRequest, host, port); - val response = httpClient.execute(new HttpHost(host, port), httpRequest); - if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - log.error("Error while executing Ping Test. HttpRequest: {}, host:{}, port:{}, reason:{}", httpRequest, host, port, response.getStatusLine().getReasonPhrase()); - response.close(); - return false; - } - response.close(); - return true; + return httpClient.execute(new HttpHost(host, port), httpRequest, response -> { + if(response.getCode() != HttpStatus.SC_OK) { + log.error("Error while executing Ping Test. HttpRequest: {}, host:{}, port:{}, reason:{}", httpRequest, host, port, response.getReasonPhrase()); + return false; + } + return true; + }); } catch (Exception e) { log.error("Exception while executing HttpRequest: ", e); return false; diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json index f19b937b..d670a199 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 644166.1778513143 + "mean_ops" : 851898.5556174923 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json index 272db000..231d176f 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 502644.4941310657 + "mean_ops" : 690993.4364166519 } \ No newline at end of file diff --git a/ranger-discovery-bundle/pom.xml b/ranger-discovery-bundle/pom.xml index a0d4245d..30f232e3 100644 --- a/ranger-discovery-bundle/pom.xml +++ b/ranger-discovery-bundle/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - ranger + ranger-parent io.appform.ranger - 1.0-RC15 + 1.0-RC16 + ../ranger-parent 4.0.0 @@ -53,12 +54,10 @@ io.appform.ranger ranger-zk-client - ${project.version} io.appform.ranger ranger-server-common - ${project.version} com.alibaba diff --git a/ranger-http-client/pom.xml b/ranger-http-client/pom.xml index ff13feea..ab407c02 100644 --- a/ranger-http-client/pom.xml +++ b/ranger-http-client/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - ranger + ranger-parent io.appform.ranger - 1.0-RC15 + 1.0-RC16 + ../ranger-parent 4.0.0 @@ -15,12 +16,10 @@ io.appform.ranger ranger-http - ${project.version} io.appform.ranger ranger-client - ${project.version} org.wiremock diff --git a/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java b/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java index 5dd8d92a..fed4720b 100644 --- a/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java @@ -15,6 +15,7 @@ */ package io.appform.ranger.client.http; +import com.google.common.base.Preconditions; import io.appform.ranger.client.AbstractRangerHubClient; import io.appform.ranger.core.finder.nodeselector.RandomServiceNodeSelector; import io.appform.ranger.core.finderhub.ServiceDataSource; @@ -25,10 +26,19 @@ import io.appform.ranger.http.serde.HTTPResponseDataDeserializer; import io.appform.ranger.http.servicefinderhub.HttpServiceDataSource; import io.appform.ranger.http.servicefinderhub.HttpServiceFinderHubBuilder; +import io.appform.ranger.http.utils.HttpClientUtils; import lombok.Builder; import lombok.Getter; +import lombok.SneakyThrows; import lombok.experimental.SuperBuilder; import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.hc.client5.http.config.ConnectionConfig; +import org.apache.hc.client5.http.fluent.Executor; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; +import org.apache.hc.core5.util.TimeValue; @Slf4j @Getter @@ -39,10 +49,30 @@ public abstract class AbstractRangerHttpHubClient nodeSelector = new RandomServiceNodeSelector<>(); + private CloseableHttpClient httpClient; + private Executor httpExecutor; + + @Override + public void start() { + Preconditions.checkNotNull(clientConfig, "Http Client Config can't be null"); + this.httpClient = HttpClientUtils.getCloseableClient(clientConfig); + this.httpExecutor = Executor.newInstance(httpClient); + super.start(); + } + + @Override + @SneakyThrows + public void stop() { + log.info("Stopping the http client"); + if (null != httpClient) { + httpClient.close(); + } + super.stop(); + } @Override protected ServiceDataSource getDefaultDataSource() { - return new HttpServiceDataSource<>(clientConfig, getMapper()); + return new HttpServiceDataSource<>(clientConfig, getMapper(), httpExecutor); } @Override diff --git a/ranger-http-client/src/main/java/io/appform/ranger/client/http/ShardedRangerHttpHubClient.java b/ranger-http-client/src/main/java/io/appform/ranger/client/http/ShardedRangerHttpHubClient.java index ecc69e68..ed96d031 100644 --- a/ranger-http-client/src/main/java/io/appform/ranger/client/http/ShardedRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/io/appform/ranger/client/http/ShardedRangerHttpHubClient.java @@ -42,6 +42,7 @@ protected ServiceFinderFactory> getFinderFactory() .shardSelector(shardSelector) .nodeSelector(this.getNodeSelector()) .mapper(getMapper()) + .httpExecutor(getHttpExecutor()) .build(); } diff --git a/ranger-http-client/src/main/java/io/appform/ranger/client/http/SimpleRangerHttpClient.java b/ranger-http-client/src/main/java/io/appform/ranger/client/http/SimpleRangerHttpClient.java index 021f027c..a18aab00 100644 --- a/ranger-http-client/src/main/java/io/appform/ranger/client/http/SimpleRangerHttpClient.java +++ b/ranger-http-client/src/main/java/io/appform/ranger/client/http/SimpleRangerHttpClient.java @@ -25,10 +25,17 @@ import io.appform.ranger.http.HttpServiceFinderBuilders; import io.appform.ranger.http.config.HttpClientConfig; import io.appform.ranger.http.serde.HTTPResponseDataDeserializer; +import io.appform.ranger.http.utils.HttpClientUtils; import lombok.Builder; import lombok.Getter; +import lombok.SneakyThrows; import lombok.experimental.SuperBuilder; import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.hc.client5.http.fluent.Executor; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; @Slf4j @SuperBuilder @@ -45,6 +52,7 @@ public class SimpleRangerHttpClient extends AbstractRangerClient serviceFinder; + private CloseableHttpClient httpClient; @Override public void start() { @@ -52,7 +60,9 @@ public void start() { Preconditions.checkNotNull(mapper, "Mapper can't be null"); Preconditions.checkNotNull(namespace, "namespace can't be null"); Preconditions.checkNotNull(deserializer, "deserializer can't be null"); + Preconditions.checkNotNull(clientConfig, "Http Client Config can't be null"); + this.httpClient = HttpClientUtils.getCloseableClient(clientConfig); this.serviceFinder = HttpServiceFinderBuilders.httpUnshardedServiceFinderBuilider() .withClientConfig(clientConfig) .withServiceName(serviceName) @@ -61,17 +71,24 @@ public void start() { .withNodeRefreshIntervalMs(nodeRefreshIntervalMs) .withDeserializer(deserializer) .withShardSelector(shardSelector) + .withHttpExecutor(Executor.newInstance(httpClient)) .build(); this.serviceFinder.start(); log.info("Started the service finder"); } @Override + @SneakyThrows public void stop() { log.info("Stopping the service finder"); - if(null != serviceFinder){ + if (null != serviceFinder) { this.serviceFinder.stop(); } + + log.info("Stopping the http client"); + if (null != httpClient) { + httpClient.close(); + } } } diff --git a/ranger-http-client/src/main/java/io/appform/ranger/client/http/UnshardedRangerHttpHubClient.java b/ranger-http-client/src/main/java/io/appform/ranger/client/http/UnshardedRangerHttpHubClient.java index 2bd3f83e..e6173f29 100644 --- a/ranger-http-client/src/main/java/io/appform/ranger/client/http/UnshardedRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/io/appform/ranger/client/http/UnshardedRangerHttpHubClient.java @@ -42,6 +42,7 @@ protected ServiceFinderFactory> getFinderFactory( .shardSelector(shardSelector) .nodeSelector(this.getNodeSelector()) .mapper(getMapper()) + .httpExecutor(getHttpExecutor()) .build(); } diff --git a/ranger-http-client/src/test/java/io/appform/ranger/client/http/BaseRangerHttpClientTest.java b/ranger-http-client/src/test/java/io/appform/ranger/client/http/BaseRangerHttpClientTest.java index bdab6924..a61f4658 100644 --- a/ranger-http-client/src/test/java/io/appform/ranger/client/http/BaseRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/io/appform/ranger/client/http/BaseRangerHttpClientTest.java @@ -81,7 +81,6 @@ public void prepareHttpMocks() throws Exception { .host("127.0.0.1") .port(wireMockExtension.getPort()) .connectionTimeoutMs(30_000) - .operationTimeoutMs(30_000) .build(); log.debug("Started http subsystem"); } diff --git a/ranger-http-model/pom.xml b/ranger-http-model/pom.xml index d3937423..f3eb1096 100644 --- a/ranger-http-model/pom.xml +++ b/ranger-http-model/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - ranger + ranger-parent io.appform.ranger - 1.0-RC15 + 1.0-RC16 + ../ranger-parent 4.0.0 diff --git a/ranger-http-server-bundle/pom.xml b/ranger-http-server-bundle/pom.xml index 740183ff..d5618471 100644 --- a/ranger-http-server-bundle/pom.xml +++ b/ranger-http-server-bundle/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - ranger + ranger-parent io.appform.ranger - 1.0-RC15 + 1.0-RC16 + ../ranger-parent 4.0.0 @@ -19,17 +20,14 @@ io.appform.ranger ranger-http-client - ${project.version} io.appform.ranger ranger-server-bundle - ${project.version} io.appform.ranger ranger-server-common - ${project.version} io.dropwizard diff --git a/ranger-http-server/pom.xml b/ranger-http-server/pom.xml index 4989fdb9..da0d7ef4 100644 --- a/ranger-http-server/pom.xml +++ b/ranger-http-server/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - ranger + ranger-parent io.appform.ranger - 1.0-RC15 + 1.0-RC16 + ../ranger-parent 4.0.0 @@ -15,7 +16,6 @@ io.appform.ranger ranger-http-server-bundle - ${project.version} io.dropwizard diff --git a/ranger-http/pom.xml b/ranger-http/pom.xml index ab190b79..34e8435d 100644 --- a/ranger-http/pom.xml +++ b/ranger-http/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - ranger + ranger-parent io.appform.ranger - 1.0-RC15 + 1.0-RC16 + ../ranger-parent 4.0.0 @@ -15,16 +16,14 @@ io.appform.ranger ranger-core - ${project.version} io.appform.ranger ranger-http-model - ${project.version} - com.squareup.okhttp3 - okhttp + org.apache.httpcomponents.client5 + httpclient5-fluent ${http.client.version} @@ -42,7 +41,6 @@ io.appform.ranger ranger-core - ${project.version} test-jar diff --git a/ranger-http/src/main/java/io/appform/ranger/http/common/HttpNodeDataStoreConnector.java b/ranger-http/src/main/java/io/appform/ranger/http/common/HttpNodeDataStoreConnector.java index f9c51528..7efb6f5a 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/common/HttpNodeDataStoreConnector.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/common/HttpNodeDataStoreConnector.java @@ -18,39 +18,20 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.appform.ranger.core.model.NodeDataStoreConnector; import io.appform.ranger.http.config.HttpClientConfig; +import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; -import okhttp3.ConnectionPool; -import okhttp3.OkHttpClient; - -import java.util.concurrent.TimeUnit; +import org.apache.hc.client5.http.fluent.Executor; /** * */ @Slf4j +@AllArgsConstructor public class HttpNodeDataStoreConnector implements NodeDataStoreConnector { protected final HttpClientConfig config; protected final ObjectMapper mapper; - protected final OkHttpClient httpClient; - - public HttpNodeDataStoreConnector( - final HttpClientConfig config, - final ObjectMapper mapper) { - this.httpClient = new OkHttpClient.Builder() - .callTimeout(config.getOperationTimeoutMs() == 0 - ? 3000 - : config.getOperationTimeoutMs(), TimeUnit.MILLISECONDS) - .connectTimeout(config.getConnectionTimeoutMs() == 0 - ? 3000 - : config.getConnectionTimeoutMs(), TimeUnit.MILLISECONDS) - .followRedirects(true) - .connectionPool(new ConnectionPool(1, 30, TimeUnit.SECONDS)) - .build(); - this.config = config; - this.mapper = mapper; - } - + protected final Executor httpExecutor; @Override public void start() { diff --git a/ranger-http/src/main/java/io/appform/ranger/http/config/HttpClientConfig.java b/ranger-http/src/main/java/io/appform/ranger/http/config/HttpClientConfig.java index b24d762f..edb66098 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/config/HttpClientConfig.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/config/HttpClientConfig.java @@ -15,6 +15,7 @@ */ package io.appform.ranger.http.config; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Builder; import lombok.Value; import lombok.extern.jackson.Jacksonized; @@ -25,11 +26,23 @@ @Value @Builder @Jacksonized +@JsonIgnoreProperties(ignoreUnknown = true) public class HttpClientConfig { String host; int port; boolean secure; - long connectionTimeoutMs; - long operationTimeoutMs; - long refreshIntervalMillis; + @Builder.Default + int maxConnPerRoute = 10; + @Builder.Default + int maxConnTotal = 20; + @Builder.Default + int operationTimeout = 10000; + @Builder.Default + long connectionTimeoutMs = 10000; + @Builder.Default + long validateAfterInactivityMs = 10000; + @Builder.Default + long ttlMs = 60000; + @Builder.Default + long idleConnEvictMs = 60000; } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java index 1eaba0ab..1a56eac2 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java @@ -16,6 +16,7 @@ package io.appform.ranger.http.servicefinder; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; import com.google.common.base.Preconditions; import io.appform.ranger.core.model.NodeDataSource; import io.appform.ranger.core.model.Service; @@ -24,15 +25,18 @@ import io.appform.ranger.http.common.HttpNodeDataStoreConnector; import io.appform.ranger.http.config.HttpClientConfig; import io.appform.ranger.http.serde.HTTPResponseDataDeserializer; -import java.util.Optional; +import io.appform.ranger.http.utils.HttpClientUtils; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import lombok.val; -import okhttp3.HttpUrl; -import okhttp3.Request; +import org.apache.hc.client5.http.fluent.Executor; +import org.apache.hc.client5.http.fluent.Request; +import org.apache.hc.core5.net.URIBuilder; +import org.apache.hc.core5.util.Timeout; -import java.io.IOException; -import java.util.Collections; import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; /** * @@ -45,62 +49,46 @@ public class HttpNodeDataSource> ex public HttpNodeDataSource( Service service, final HttpClientConfig config, - ObjectMapper mapper) { - super(config, mapper); + ObjectMapper mapper, + Executor executor) { + super(config, mapper, executor); this.service = service; } @Override + @SneakyThrows public Optional>> refresh(D deserializer) { Preconditions.checkNotNull(config, "client config has not been set for node data"); Preconditions.checkNotNull(mapper, "mapper has not been set for node data"); val url = String.format("/ranger/nodes/v1/%s/%s", service.getNamespace(), service.getServiceName()); log.debug("Refreshing the node list from url {}", url); - val httpUrl = new HttpUrl.Builder() - .scheme(config.isSecure() + + val httpUrl = new URIBuilder() + .setScheme(config.isSecure() ? "https" : "http") - .host(config.getHost()) - .port(config.getPort() == 0 + .setHost(config.getHost()) + .setPort(config.getPort() == 0 ? defaultPort() : config.getPort()) - .encodedPath(url) - .build(); - val request = new Request.Builder() - .url(httpUrl) - .get() + .setPath(url) .build(); - try (val response = httpClient.newCall(request).execute()) { - if (response.isSuccessful()) { - try (val body = response.body()) { - if (null == body) { - log.warn("HTTP call to {} returned empty body", httpUrl); - } else { - val bytes = body.bytes(); - val serviceNodesResponse = deserializer.deserialize(bytes); - if(serviceNodesResponse.valid()){ - return Optional.of(FinderUtils.filterValidNodes( - service, - serviceNodesResponse.getData(), - healthcheckZombieCheckThresholdTime(service))); - } else{ - log.warn("Http call to {} returned a failure response with response {}", httpUrl, serviceNodesResponse); - } - } - } + return HttpClientUtils.executeRequest(httpExecutor, Request.get(httpUrl), (Function>>>) responseBytes -> { + val serviceNodesResponse = deserializer.deserialize(responseBytes); + if (serviceNodesResponse.valid()) { + return Optional.of(FinderUtils.filterValidNodes( + service, + serviceNodesResponse.getData(), + healthcheckZombieCheckThresholdTime(service))); } else { - log.warn("HTTP call to {} returned: {}", httpUrl, response.code()); + log.warn("Http call to {} returned a failure response with response {}", httpUrl, serviceNodesResponse); + return Optional.empty(); } - } catch (IOException e) { - log.error("Error getting service data from the http endPoint: ", e); - } - return Optional.empty(); - } - - @Override - public boolean isActive() { - return true; + }, (Function>>>) exception -> { + log.error("HTTP call to {} returned with an exception. Executing the error handler", httpUrl, exception); + return Optional.empty(); + }); } } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java index 26ab07d2..612d7714 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java @@ -22,6 +22,7 @@ import io.appform.ranger.core.model.Service; import io.appform.ranger.http.config.HttpClientConfig; import io.appform.ranger.http.serde.HTTPResponseDataDeserializer; +import org.apache.hc.client5.http.fluent.Executor; /** * @@ -30,6 +31,7 @@ public class HttpShardedServiceFinderBuilder extends SimpleShardedServiceFind private HttpClientConfig clientConfig; private ObjectMapper mapper; + private Executor httpExecutor; public HttpShardedServiceFinderBuilder withClientConfig(final HttpClientConfig clientConfig) { this.clientConfig = clientConfig; @@ -41,6 +43,11 @@ public HttpShardedServiceFinderBuilder withObjectMapper(final ObjectMapper ma return this; } + public HttpShardedServiceFinderBuilder withHttpExecutor(final Executor httpExecutor){ + this.httpExecutor = httpExecutor; + return this; + } + @Override public SimpleShardedServiceFinder build() { return buildFinder(); @@ -48,7 +55,7 @@ public SimpleShardedServiceFinder build() { @Override protected NodeDataSource> dataSource(Service service) { - return new HttpNodeDataSource<>(service, clientConfig, mapper); + return new HttpNodeDataSource<>(service, clientConfig, mapper, httpExecutor); } } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java index efdf1f2f..4349c0b1 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java @@ -22,12 +22,14 @@ import io.appform.ranger.core.model.Service; import io.appform.ranger.http.config.HttpClientConfig; import io.appform.ranger.http.serde.HTTPResponseDataDeserializer; +import org.apache.hc.client5.http.fluent.Executor; public class HttpUnshardedServiceFinderBuilider extends SimpleUnshardedServiceFinderBuilder, HTTPResponseDataDeserializer> { private HttpClientConfig clientConfig; private ObjectMapper mapper; + private Executor httpExecutor; public HttpUnshardedServiceFinderBuilider withClientConfig(final HttpClientConfig clientConfig) { this.clientConfig = clientConfig; @@ -39,6 +41,11 @@ public HttpUnshardedServiceFinderBuilider withObjectMapper(final ObjectMapper return this; } + public HttpUnshardedServiceFinderBuilider withHttpExecutor(final Executor executor){ + this.httpExecutor = executor; + return this; + } + @Override public SimpleUnshardedServiceFinder build() { return buildFinder(); @@ -46,7 +53,7 @@ public SimpleUnshardedServiceFinder build() { @Override protected NodeDataSource> dataSource(Service service) { - return new HttpNodeDataSource<>(service, clientConfig, mapper); + return new HttpNodeDataSource<>(service, clientConfig, mapper, httpExecutor); } } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java index 978eca5d..1dd77052 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java @@ -16,77 +16,70 @@ package io.appform.ranger.http.servicefinderhub; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; import com.google.common.base.Preconditions; import io.appform.ranger.core.finderhub.ServiceDataSource; import io.appform.ranger.core.model.Service; import io.appform.ranger.http.common.HttpNodeDataStoreConnector; import io.appform.ranger.http.config.HttpClientConfig; import io.appform.ranger.http.model.ServiceDataSourceResponse; +import io.appform.ranger.http.utils.HttpClientUtils; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import lombok.val; -import okhttp3.HttpUrl; -import okhttp3.Request; +import org.apache.hc.client5.http.fluent.Executor; +import org.apache.hc.client5.http.fluent.Request; +import org.apache.hc.core5.net.URIBuilder; +import org.apache.hc.core5.util.Timeout; -import java.io.IOException; import java.util.Collection; -import java.util.Collections; +import java.util.Set; +import java.util.concurrent.TimeUnit; @Slf4j public class HttpServiceDataSource extends HttpNodeDataStoreConnector implements ServiceDataSource { - public HttpServiceDataSource(HttpClientConfig config, ObjectMapper mapper) { - super(config, mapper); + public HttpServiceDataSource(HttpClientConfig config, ObjectMapper mapper, Executor httpExecutor) { + super(config, mapper, httpExecutor); } @Override + @SneakyThrows public Collection services() { Preconditions.checkNotNull(config, "client config has not been set for node data"); Preconditions.checkNotNull(mapper, "mapper has not been set for node data"); - val httpUrl = new HttpUrl.Builder() - .scheme(config.isSecure() + val httpUrl = new URIBuilder() + .setScheme(config.isSecure() ? "https" : "http") - .host(config.getHost()) - .port(config.getPort() == 0 + .setHost(config.getHost()) + .setPort(config.getPort() == 0 ? defaultPort() : config.getPort()) - .encodedPath("/ranger/services/v1") - .build(); - val request = new Request.Builder() - .url(httpUrl) - .get() + .setPath("/ranger/services/v1") .build(); - try (val response = httpClient.newCall(request).execute()) { - if (response.isSuccessful()) { - try (val body = response.body()) { - if (null == body) { - log.warn("HTTP call to {} returned empty body", httpUrl); - } - else { - val bytes = body.bytes(); - val serviceDataSourceResponse = mapper.readValue(bytes, ServiceDataSourceResponse.class); - if (serviceDataSourceResponse.valid()) { - return serviceDataSourceResponse.getData(); - } - else { - log.warn("Http call to {} returned a failure response with data {}", - httpUrl, - serviceDataSourceResponse); - } - } - } - } - else { - log.warn("HTTP call to {} returned code: {}", httpUrl, response.code()); + return HttpClientUtils.executeRequest(httpExecutor, Request.get(httpUrl), (Function>) responseBytes -> { + val serviceDataSourceResponse = getServiceDataSourceResponse(responseBytes); + + if (serviceDataSourceResponse.valid()) { + return serviceDataSourceResponse.getData(); + } else { + log.warn("Http call to {} returned an invalid response with data {}", + httpUrl, + serviceDataSourceResponse); + return Set.of(); } - } - catch (IOException e) { - log.info("Error parsing the response from server for : {} with exception {}", httpUrl, e); - } - log.error("No data returned from server: " + httpUrl); - return Collections.emptySet(); + }, (Function>) exception -> { + log.error("Error getting list of services from the server with httpUrl {} with exception. Executing the error handler ", httpUrl, exception); + return Set.of(); + }); + } + + @SneakyThrows + private ServiceDataSourceResponse getServiceDataSourceResponse(final byte[] responseBytes) { + return mapper.readValue(responseBytes, ServiceDataSourceResponse.class); } } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java index 129568ba..e3d39ec2 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java @@ -27,6 +27,7 @@ import io.appform.ranger.http.servicefinder.HttpShardedServiceFinderBuilder; import lombok.Builder; import lombok.val; +import org.apache.hc.client5.http.fluent.Executor; public class HttpShardedServiceFinderFactory implements ServiceFinderFactory> { @@ -36,6 +37,7 @@ public class HttpShardedServiceFinderFactory implements ServiceFinderFactory private final ShardSelector> shardSelector; private final ServiceNodeSelector nodeSelector; private final int nodeRefreshIntervalMs; + private final Executor httpExecutor; @Builder public HttpShardedServiceFinderFactory( @@ -44,7 +46,8 @@ public HttpShardedServiceFinderFactory( HTTPResponseDataDeserializer deserializer, ShardSelector> shardSelector, ServiceNodeSelector nodeSelector, - int nodeRefreshIntervalMs) + int nodeRefreshIntervalMs, + Executor httpExecutor) { this.clientConfig = httpClientConfig; this.mapper = mapper; @@ -52,6 +55,7 @@ public HttpShardedServiceFinderFactory( this.shardSelector = shardSelector; this.nodeSelector = nodeSelector; this.nodeRefreshIntervalMs = nodeRefreshIntervalMs; + this.httpExecutor = httpExecutor; } @Override @@ -65,6 +69,7 @@ public ServiceFinder> buildFinder(Service service) .withNodeRefreshIntervalMs(nodeRefreshIntervalMs) .withShardSelector(shardSelector) .withNodeSelector(nodeSelector) + .withHttpExecutor(httpExecutor) .build(); serviceFinder.start(); return serviceFinder; diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java index e85b7719..f0ca6ee6 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java @@ -27,6 +27,7 @@ import io.appform.ranger.http.servicefinder.HttpUnshardedServiceFinderBuilider; import lombok.Builder; import lombok.val; +import org.apache.hc.client5.http.fluent.Executor; public class HttpUnshardedServiceFinderFactory implements ServiceFinderFactory> { @@ -36,6 +37,7 @@ public class HttpUnshardedServiceFinderFactory implements ServiceFinderFactor private final ShardSelector> shardSelector; private final ServiceNodeSelector nodeSelector; private final int nodeRefreshIntervalMs; + private final Executor httpExecutor; @Builder public HttpUnshardedServiceFinderFactory( @@ -44,7 +46,8 @@ public HttpUnshardedServiceFinderFactory( HTTPResponseDataDeserializer deserializer, ShardSelector> shardSelector, ServiceNodeSelector nodeSelector, - int nodeRefreshIntervalMs) + int nodeRefreshIntervalMs, + Executor httpExecutor) { this.clientConfig = httpClientConfig; this.mapper = mapper; @@ -52,6 +55,7 @@ public HttpUnshardedServiceFinderFactory( this.shardSelector = shardSelector; this.nodeSelector = nodeSelector; this.nodeRefreshIntervalMs = nodeRefreshIntervalMs; + this.httpExecutor = httpExecutor; } @Override @@ -65,6 +69,7 @@ public ServiceFinder> buildFinder(Service service .withNodeRefreshIntervalMs(nodeRefreshIntervalMs) .withShardSelector(shardSelector) .withNodeSelector(nodeSelector) + .withHttpExecutor(httpExecutor) .build(); serviceFinder.start(); return serviceFinder; diff --git a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java index 47ea2c98..ee487878 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; import com.google.common.base.Preconditions; import io.appform.ranger.core.model.NodeDataSink; import io.appform.ranger.core.model.Service; @@ -26,26 +27,32 @@ import io.appform.ranger.http.config.HttpClientConfig; import io.appform.ranger.http.model.ServiceRegistrationResponse; import io.appform.ranger.http.serde.HttpRequestDataSerializer; +import io.appform.ranger.http.utils.HttpClientUtils; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import lombok.val; -import okhttp3.HttpUrl; -import okhttp3.Request; -import okhttp3.RequestBody; +import org.apache.hc.client5.http.fluent.Executor; +import org.apache.hc.client5.http.fluent.Request; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.io.entity.ByteArrayEntity; +import org.apache.hc.core5.net.URIBuilder; +import org.apache.hc.core5.util.Timeout; -import java.io.IOException; import java.util.Optional; +import java.util.concurrent.TimeUnit; @Slf4j public class HttpNodeDataSink> extends HttpNodeDataStoreConnector implements NodeDataSink { private final Service service; - public HttpNodeDataSink(Service service, HttpClientConfig config, ObjectMapper mapper) { - super(config, mapper); + public HttpNodeDataSink(Service service, HttpClientConfig config, ObjectMapper mapper, Executor executor) { + super(config, mapper, executor); this.service = service; } @Override + @SneakyThrows public void updateState(S serializer, ServiceNode serviceNode) { Preconditions.checkNotNull(config, "client config has not been set for node data"); Preconditions.checkNotNull(mapper, "mapper has not been set for node data"); @@ -53,48 +60,40 @@ public void updateState(S serializer, ServiceNode serviceNode) { val url = String.format("/ranger/nodes/v1/add/%s/%s", service.getNamespace(), service.getServiceName()); log.debug("Updating state at the url {}", url); - val httpUrl = new HttpUrl.Builder() - .scheme(config.isSecure() + val httpUrl = new URIBuilder() + .setScheme(config.isSecure() ? "https" : "http") - .host(config.getHost()) - .port(config.getPort() == 0 + .setHost(config.getHost()) + .setPort(config.getPort() == 0 ? defaultPort() : config.getPort()) - .encodedPath(url) + .setPath(url) .build(); - val requestBody = RequestBody.create(serializer.serialize(serviceNode)); - val serviceRegistrationResponse = registerService(httpUrl, requestBody).orElse(null); - if(null == serviceRegistrationResponse || !serviceRegistrationResponse.valid()){ - log.warn("Http call to {} returned a failure response {}", httpUrl, serviceRegistrationResponse); - Exceptions.illegalState("Error updating state on the server for node data: " + httpUrl); - } - } - private Optional> registerService(HttpUrl httpUrl, RequestBody requestBody){ - val request = new Request.Builder() - .url(httpUrl) - .post(requestBody) - .build(); - try (val response = httpClient.newCall(request).execute()) { - if (response.isSuccessful()) { - try (val body = response.body()) { - if (null == body) { - log.warn("HTTP call to {} returned empty body", httpUrl); - } - else { - return Optional.of(mapper.readValue(body.bytes(), - new TypeReference>() {})); - } - } + val request = Request.post(httpUrl) + .body(new ByteArrayEntity(serializer.serialize(serviceNode), ContentType.APPLICATION_JSON)); + + HttpClientUtils.executeRequest(httpExecutor, request, (Function>>) responseBytes -> { + val serviceRegistrationResponse = getServiceRegistrationResponse(responseBytes); + + if (null == serviceRegistrationResponse || !serviceRegistrationResponse.valid()) { + log.warn("Http call to {} returned a failure response {}", httpUrl, serviceRegistrationResponse); + Exceptions.illegalState("Error updating state on the server for node data: " + httpUrl); } - else { - log.warn("HTTP call to {} has returned: {}", httpUrl, response.code()); + + return Optional.of(serviceRegistrationResponse); + }, new Function>>() { + @Override + @SneakyThrows + public Optional> apply(Exception exception) { + throw exception; } - } - catch (IOException e) { - log.error("Error updating state on the server with httpUrl {} with exception {} ", httpUrl, e); - } - return Optional.empty(); + }); + } + + @SneakyThrows + private ServiceRegistrationResponse getServiceRegistrationResponse(final byte[] responseBytes) { + return mapper.readValue(responseBytes, new TypeReference<>() {}); } } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilder.java b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilder.java index 98d5a404..10b627ee 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilder.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilder.java @@ -23,12 +23,14 @@ import io.appform.ranger.http.config.HttpClientConfig; import io.appform.ranger.http.serde.HttpRequestDataSerializer; import lombok.extern.slf4j.Slf4j; +import org.apache.hc.client5.http.fluent.Executor; @Slf4j public class HttpShardedServiceProviderBuilder extends BaseServiceProviderBuilder, HttpRequestDataSerializer> { private HttpClientConfig clientConfig; private ObjectMapper mapper; + private Executor httpExecutor; public HttpShardedServiceProviderBuilder withClientConfiguration(final HttpClientConfig clientConfig) { this.clientConfig = clientConfig; @@ -40,6 +42,11 @@ public HttpShardedServiceProviderBuilder withObjectMapper(final ObjectMapper return this; } + public HttpShardedServiceProviderBuilder withHttpExecutor(final Executor executor){ + this.httpExecutor = executor; + return this; + } + @Override public ServiceProvider> build() { return super.buildProvider(); @@ -47,6 +54,6 @@ public ServiceProvider> build() { @Override protected NodeDataSink> dataSink(Service service) { - return new HttpNodeDataSink<>(service, clientConfig, mapper); + return new HttpNodeDataSink<>(service, clientConfig, mapper, httpExecutor); } } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/utils/HttpClientUtils.java b/ranger-http/src/main/java/io/appform/ranger/http/utils/HttpClientUtils.java new file mode 100644 index 00000000..d7bf58b8 --- /dev/null +++ b/ranger-http/src/main/java/io/appform/ranger/http/utils/HttpClientUtils.java @@ -0,0 +1,77 @@ +package io.appform.ranger.http.utils; + +import io.appform.ranger.http.config.HttpClientConfig; +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.hc.client5.http.ClientProtocolException; +import org.apache.hc.client5.http.HttpResponseException; +import org.apache.hc.client5.http.config.ConnectionConfig; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.fluent.Executor; +import org.apache.hc.client5.http.fluent.Request; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.io.SocketConfig; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.util.TimeValue; + +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +@UtilityClass +@Slf4j +public class HttpClientUtils { + + public static CloseableHttpClient getCloseableClient(final HttpClientConfig clientConfig) { + return HttpClientBuilder.create() + .setConnectionManager(PoolingHttpClientConnectionManagerBuilder.create() + .useSystemProperties() + .setMaxConnPerRoute(clientConfig.getMaxConnPerRoute()) + .setMaxConnTotal(clientConfig.getMaxConnTotal()) + .setDefaultSocketConfig(SocketConfig.custom() + .setTcpNoDelay(true) + .setSoTimeout(clientConfig.getOperationTimeout(), TimeUnit.MILLISECONDS) + .build()) + .setDefaultConnectionConfig(ConnectionConfig.custom() + .setConnectTimeout(clientConfig.getConnectionTimeoutMs(), TimeUnit.MILLISECONDS) + .setSocketTimeout(clientConfig.getOperationTimeout(), TimeUnit.MILLISECONDS) + .setValidateAfterInactivity(TimeValue.ofMilliseconds(clientConfig.getValidateAfterInactivityMs())) + .setTimeToLive(clientConfig.getTtlMs(), TimeUnit.MILLISECONDS) + .build()) + .build()) + .useSystemProperties() + .evictExpiredConnections() + .evictIdleConnections(TimeValue.ofMilliseconds(clientConfig.getIdleConnEvictMs())) + .setDefaultRequestConfig(RequestConfig.custom() + .setResponseTimeout(clientConfig.getOperationTimeout(), TimeUnit.MILLISECONDS) + .build()) + .build(); + } + + public static T executeRequest(final Executor httpExecutor, + final Request request, + final Function successHandler, + final Function failureHandler) { + try { + return successHandler.apply(httpExecutor.execute(request).handleResponse(httpResponse -> { + val code = httpResponse.getCode(); + + if (code >= HttpStatus.SC_REDIRECTION) { + throw new HttpResponseException(code, httpResponse.getReasonPhrase()); + } + + val entity = httpResponse.getEntity(); + if (null == entity) { + throw new ClientProtocolException("Response contains no content"); + } + + return EntityUtils.toByteArray(entity); + })); + } catch (Exception e) { + return failureHandler.apply(e); + } + } +} diff --git a/ranger-http/src/test/java/io/appform/ranger/http/common/HttpNodeDataStoreConnectorTest.java b/ranger-http/src/test/java/io/appform/ranger/http/common/HttpNodeDataStoreConnectorTest.java index 07e76767..7f81bf25 100644 --- a/ranger-http/src/test/java/io/appform/ranger/http/common/HttpNodeDataStoreConnectorTest.java +++ b/ranger-http/src/test/java/io/appform/ranger/http/common/HttpNodeDataStoreConnectorTest.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.appform.ranger.http.config.HttpClientConfig; import lombok.val; +import org.apache.hc.client5.http.fluent.Executor; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -30,7 +31,7 @@ void testHttpNodeDataStoreConnector(){ .host("localhost-1") .port(80) .build(); - val httpNodeDataStoreConnector = new HttpNodeDataStoreConnector<>(httpClientConfig, objectMapper); + val httpNodeDataStoreConnector = new HttpNodeDataStoreConnector<>(httpClientConfig, objectMapper, Executor.newInstance()); Assertions.assertNotNull(httpNodeDataStoreConnector); Assertions.assertTrue(httpNodeDataStoreConnector.isActive()); } diff --git a/ranger-http/src/test/java/io/appform/ranger/http/config/HttpClientConfigTest.java b/ranger-http/src/test/java/io/appform/ranger/http/config/HttpClientConfigTest.java index aef7a34c..06c2bb35 100644 --- a/ranger-http/src/test/java/io/appform/ranger/http/config/HttpClientConfigTest.java +++ b/ranger-http/src/test/java/io/appform/ranger/http/config/HttpClientConfigTest.java @@ -29,6 +29,5 @@ void testHttpClientConfig(){ Assertions.assertEquals("localhost-1", resource.getHost()); Assertions.assertEquals(80, resource.getPort()); Assertions.assertEquals(10, resource.getConnectionTimeoutMs()); - Assertions.assertEquals(10, resource.getOperationTimeoutMs()); } } diff --git a/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index df51b998..f8688436 100644 --- a/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -27,6 +27,7 @@ import io.appform.ranger.http.model.ServiceNodesResponse; import lombok.Data; import lombok.val; +import org.apache.hc.client5.http.fluent.Executor; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -70,7 +71,6 @@ void testFinder(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { .host("127.0.0.1") .port(wireMockRuntimeInfo.getHttpPort()) .connectionTimeoutMs(30_000) - .operationTimeoutMs(30_000) .build(); val finder = new HttpShardedServiceFinderBuilder() @@ -80,7 +80,8 @@ void testFinder(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { .withObjectMapper(MAPPER) .withDeserializer(data -> { try { - return MAPPER.readValue(data, new TypeReference>() {}); + return MAPPER.readValue(data, new TypeReference<>() { + }); } catch (IOException e) { throw new IllegalArgumentException(e); @@ -88,6 +89,7 @@ void testFinder(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { }) .withShardSelector((criteria, registry) -> registry.nodeList()) .withNodeRefreshIntervalMs(1000) + .withHttpExecutor(Executor.newInstance()) .build(); finder.start(); RangerTestUtils.sleepUntilFinderStarts(finder); diff --git a/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java b/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java index 1996364f..5ed3aae1 100644 --- a/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java +++ b/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java @@ -23,6 +23,7 @@ import io.appform.ranger.http.config.HttpClientConfig; import io.appform.ranger.http.model.ServiceDataSourceResponse; import lombok.val; +import org.apache.hc.client5.http.fluent.Executor; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -52,9 +53,8 @@ void testServiceDataSource(WireMockRuntimeInfo wireMockRuntimeInfo) throws IOExc .host("127.0.0.1") .port(wireMockRuntimeInfo.getHttpPort()) .connectionTimeoutMs(30_000) - .operationTimeoutMs(30_000) .build(); - val httpServiceDataSource = new HttpServiceDataSource<>(clientConfig, MAPPER); + val httpServiceDataSource = new HttpServiceDataSource<>(clientConfig, MAPPER, Executor.newInstance()); val services = httpServiceDataSource.services(); Assertions.assertNotNull(services); Assertions.assertFalse(services.isEmpty()); diff --git a/ranger-http/src/test/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java b/ranger-http/src/test/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java index 39cd8cb2..35fc9972 100644 --- a/ranger-http/src/test/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java +++ b/ranger-http/src/test/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java @@ -27,6 +27,7 @@ import lombok.Builder; import lombok.Data; import lombok.val; +import org.apache.hc.client5.http.fluent.Executor; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -69,8 +70,6 @@ void testProvider(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { val clientConfig = HttpClientConfig.builder() .host("127.0.0.1") .port(wireMockRuntimeInfo.getHttpPort()) - .connectionTimeoutMs(30_000) - .operationTimeoutMs(30_000) .build(); val serviceProvider = new HttpShardedServiceProviderBuilder() .withNamespace("testns") @@ -83,6 +82,7 @@ void testProvider(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { .withClientConfiguration(clientConfig) .withNodeData(farmNodeData) .withSerializer(node -> requestBytes) + .withHttpExecutor(Executor.newInstance()) .build(); serviceProvider.start(); Assertions.assertNotNull(serviceProvider); diff --git a/ranger-http/src/test/resources/fixtures/httpClientConfig.json b/ranger-http/src/test/resources/fixtures/httpClientConfig.json index 3855f309..569e44f5 100644 --- a/ranger-http/src/test/resources/fixtures/httpClientConfig.json +++ b/ranger-http/src/test/resources/fixtures/httpClientConfig.json @@ -3,6 +3,5 @@ "port" : 80, "secure" : false, "connectionTimeoutMs" : 10, - "operationTimeoutMs" : 10, "refreshIntervalMillis" : 1000 } \ No newline at end of file diff --git a/ranger-parent/pom.xml b/ranger-parent/pom.xml new file mode 100644 index 00000000..391c6d79 --- /dev/null +++ b/ranger-parent/pom.xml @@ -0,0 +1,145 @@ + + + 4.0.0 + + io.appform.ranger + ranger-bom + 1.0-RC16 + ../ranger-bom + + + ranger-parent + pom + Ranger Parent Pom + + + Ranger Parent POM from which all the other module poms inherit. + + + + + 31.0.1-jre + 5.5.0 + 1.7.32 + + 1.18.22 + 3.0.1u2 + + 5.8.2 + 4.1.1 + 2.0.0 + + 5.2.2 + 3.3.1 + 4.2.0 + + 2.1.10 + + + + + + io.appform.ranger + ranger-bom + ${project.version} + pom + import + + + org.junit + junit-bom + ${junit.jupiter.version} + pom + import + + + io.dropwizard + dropwizard-bom + pom + import + ${dropwizard.version} + + + io.appform.ranger + ranger-core + test-jar + ${project.version} + + + io.appform.ranger + ranger-client + test-jar + ${project.version} + + + + + + + org.projectlombok + lombok + ${lombok.version} + provided + + + com.google.code.findbugs + annotations + ${annotations.version} + + + com.google.guava + guava + ${guava.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + io.dropwizard + dropwizard-jackson + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + com.github.rholder + guava-retrying + ${guava-retrying.version} + + + org.awaitility + awaitility + ${awaitility.version} + test + + + + + + + add-java-open-options-for-jdk16+ + + [16,) + + + + --add-opens java.base/java.net=ALL-UNNAMED + --add-opens java.base/sun.net=ALL-UNNAMED + + + + + + \ No newline at end of file diff --git a/ranger-server-bundle/pom.xml b/ranger-server-bundle/pom.xml index ba3c9510..f4820062 100644 --- a/ranger-server-bundle/pom.xml +++ b/ranger-server-bundle/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - ranger + ranger-parent io.appform.ranger - 1.0-RC15 + 1.0-RC16 + ../ranger-parent 4.0.0 @@ -15,12 +16,10 @@ io.appform.ranger ranger-client - ${project.version} io.appform.ranger ranger-http-model - ${project.version} io.dropwizard @@ -36,7 +35,6 @@ io.appform.ranger ranger-client - ${project.version} test-jar diff --git a/ranger-server-common/pom.xml b/ranger-server-common/pom.xml index ab777715..a28992e9 100644 --- a/ranger-server-common/pom.xml +++ b/ranger-server-common/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - ranger + ranger-parent io.appform.ranger - 1.0-RC15 + 1.0-RC16 + ../ranger-parent 4.0.0 diff --git a/ranger-zk-client/pom.xml b/ranger-zk-client/pom.xml index 8b6443d3..61c4c88f 100644 --- a/ranger-zk-client/pom.xml +++ b/ranger-zk-client/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - ranger + ranger-parent io.appform.ranger - 1.0-RC15 + 1.0-RC16 + ../ranger-parent 4.0.0 @@ -15,12 +16,10 @@ io.appform.ranger ranger-zookeeper - ${project.version} io.appform.ranger ranger-client - ${project.version} org.apache.curator diff --git a/ranger-zk-server-bundle/pom.xml b/ranger-zk-server-bundle/pom.xml index cf76d7d5..6340fddf 100644 --- a/ranger-zk-server-bundle/pom.xml +++ b/ranger-zk-server-bundle/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - ranger + ranger-parent io.appform.ranger - 1.0-RC15 + 1.0-RC16 + ../ranger-parent 4.0.0 @@ -15,17 +16,14 @@ io.appform.ranger ranger-zk-client - ${project.version} io.appform.ranger ranger-server-bundle - ${project.version} io.appform.ranger ranger-server-common - ${project.version} io.dropwizard diff --git a/ranger-zk-server/pom.xml b/ranger-zk-server/pom.xml index b4a7f298..04754e10 100644 --- a/ranger-zk-server/pom.xml +++ b/ranger-zk-server/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - ranger + ranger-parent io.appform.ranger - 1.0-RC15 + 1.0-RC16 + ../ranger-parent 4.0.0 diff --git a/ranger-zookeeper/pom.xml b/ranger-zookeeper/pom.xml index e9356012..80331a98 100644 --- a/ranger-zookeeper/pom.xml +++ b/ranger-zookeeper/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - ranger + ranger-parent io.appform.ranger - 1.0-RC15 + 1.0-RC16 + ../ranger-parent 4.0.0 @@ -15,7 +16,6 @@ io.appform.ranger ranger-core - ${project.version} diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/healthservice/monitor/sample/PingCheckMonitorTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/healthservice/monitor/sample/PingCheckMonitorTest.java index 573418a5..71fbe06c 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/healthservice/monitor/sample/PingCheckMonitorTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/healthservice/monitor/sample/PingCheckMonitorTest.java @@ -19,7 +19,7 @@ import io.appform.ranger.core.healthservice.TimeEntity; import io.appform.ranger.core.healthservice.monitor.sample.PingCheckMonitor; import lombok.val; -import org.apache.http.client.methods.HttpGet; +import org.apache.hc.client5.http.classic.methods.HttpGet; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test;