Skip to content

Enable decompression of response within LowLevelRestClient #55413

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
May 15, 2020
Merged

Enable decompression of response within LowLevelRestClient #55413

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7f760d5
Reverted initial changes for supporting decompression within the rhlc
Hakky54 Apr 17, 2020
1fc1e4d
Added support for decompression at llrc and added integration test
Hakky54 Apr 17, 2020
15d1201
Merge branch 'master' into feature-enable-decompression-of-response-w…
Hakky54 Apr 17, 2020
9816d84
Refactored a bit
Hakky54 Apr 17, 2020
9024274
Applied feedback
Hakky54 May 15, 2020
15b49dd
Merge branch 'master' into feature-enable-decompression-of-response-w…
Hakky54 May 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 2 additions & 11 deletions client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,7 @@

package org.elasticsearch.client;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.client.entity.GzipDecompressingEntity;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.ActionListener;
Expand Down Expand Up @@ -1874,18 +1872,11 @@ protected final ElasticsearchStatusException parseResponseException(ResponseExce
return elasticsearchException;
}

protected final <Resp> Resp parseEntity(final HttpEntity httpEntity,
protected final <Resp> Resp parseEntity(final HttpEntity entity,
final CheckedFunction<XContentParser, Resp, IOException> entityParser) throws IOException {
if (httpEntity == null) {
if (entity == null) {
throw new IllegalStateException("Response body expected but not returned");
}

final HttpEntity entity = Optional.ofNullable(httpEntity.getContentEncoding())
.map(Header::getValue)
.filter("gzip"::equalsIgnoreCase)
.map(gzipHeaderValue -> (HttpEntity) new GzipDecompressingEntity(httpEntity))
.orElse(httpEntity);

if (entity.getContentType() == null) {
throw new IllegalStateException("Elasticsearch didn't return the [Content-Type] header, unable to parse response body");
}
Expand Down
57 changes: 0 additions & 57 deletions client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
import org.apache.http.message.BasicStatusLine;
import org.apache.http.nio.entity.NByteArrayEntity;
import org.apache.http.nio.entity.NStringEntity;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
Expand Down Expand Up @@ -118,12 +117,10 @@
import org.hamcrest.Matchers;
import org.junit.Before;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.net.SocketTimeoutException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
Expand All @@ -137,7 +134,6 @@
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.GZIPOutputStream;

import static org.elasticsearch.client.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider.registeredMetricName;
import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
Expand Down Expand Up @@ -326,59 +322,6 @@ public void testParseEntity() throws IOException {
}
}

public void testParseCompressedEntity() throws IOException {
CheckedFunction<XContentParser, String, IOException> entityParser = parser -> {
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
assertTrue(parser.nextToken().isValue());
String value = parser.text();
assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
return value;
};

HttpEntity jsonEntity = createGzipEncodedEntity("{\"field\":\"value\"}", ContentType.APPLICATION_JSON);
assertEquals("value", restHighLevelClient.parseEntity(jsonEntity, entityParser));
HttpEntity yamlEntity = createGzipEncodedEntity("---\nfield: value\n", ContentType.create("application/yaml"));
assertEquals("value", restHighLevelClient.parseEntity(yamlEntity, entityParser));
HttpEntity smileEntity = createGzipEncodedEntity(SmileXContent.contentBuilder(), ContentType.create("application/smile"));
assertEquals("value", restHighLevelClient.parseEntity(smileEntity, entityParser));
HttpEntity cborEntity = createGzipEncodedEntity(CborXContent.contentBuilder(), ContentType.create("application/cbor"));
assertEquals("value", restHighLevelClient.parseEntity(cborEntity, entityParser));
}

private HttpEntity createGzipEncodedEntity(String content, ContentType contentType) throws IOException {
byte[] gzipEncodedContent = compressContentWithGzip(content.getBytes(StandardCharsets.UTF_8));
NByteArrayEntity httpEntity = new NByteArrayEntity(gzipEncodedContent, contentType);
httpEntity.setContentEncoding("gzip");

return httpEntity;
}

private HttpEntity createGzipEncodedEntity(XContentBuilder xContentBuilder, ContentType contentType) throws IOException {
try (XContentBuilder builder = xContentBuilder) {
builder.startObject();
builder.field("field", "value");
builder.endObject();

BytesRef bytesRef = BytesReference.bytes(xContentBuilder).toBytesRef();
byte[] gzipEncodedContent = compressContentWithGzip(bytesRef.bytes);
NByteArrayEntity httpEntity = new NByteArrayEntity(gzipEncodedContent, contentType);
httpEntity.setContentEncoding("gzip");

return httpEntity;
}
}

private static byte[] compressContentWithGzip(byte[] content) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream(content.length);
GZIPOutputStream gzip = new GZIPOutputStream(bos);
gzip.write(content);
gzip.close();
bos.close();

return bos.toByteArray();
}

private static HttpEntity createBinaryEntity(XContentBuilder xContentBuilder, ContentType contentType) throws IOException {
try (XContentBuilder builder = xContentBuilder) {
builder.startObject();
Expand Down
10 changes: 10 additions & 0 deletions client/rest/src/main/java/org/elasticsearch/client/RestClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.http.HttpResponse;
import org.apache.http.client.AuthCache;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpOptions;
Expand Down Expand Up @@ -67,6 +68,7 @@
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
Expand Down Expand Up @@ -272,6 +274,14 @@ private Response performRequest(final NodeTuple<Iterator<Node>> nodeTuple,
private ResponseOrResponseException convertResponse(InternalRequest request, Node node, HttpResponse httpResponse) throws IOException {
RequestLogger.logResponse(logger, request.httpRequest, node.getHost(), httpResponse);
int statusCode = httpResponse.getStatusLine().getStatusCode();

Optional.ofNullable(httpResponse.getEntity())
.map(HttpEntity::getContentEncoding)
.map(Header::getValue)
.filter("gzip"::equalsIgnoreCase)
.map(gzipHeaderValue -> new GzipDecompressingEntity(httpResponse.getEntity()))
.ifPresent(httpResponse::setEntity);

Response response = new Response(request.httpRequest.getRequestLine(), node.getHost(), httpResponse);
if (isSuccessfulResponse(statusCode) || request.ignoreErrorCodes.contains(response.getStatusLine().getStatusCode())) {
onResponse(node);
Expand Down
32 changes: 26 additions & 6 deletions qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpCompressionIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,32 +19,51 @@
package org.elasticsearch.http;

import org.apache.http.HttpHeaders;
import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.test.rest.ESRestTestCase;

import java.io.IOException;

import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;

public class HttpCompressionIT extends ESRestTestCase {
private static final String GZIP_ENCODING = "gzip";

private static final String GZIP_ENCODING = "gzip";
private static final String SAMPLE_DOCUMENT = "{\n" +
" \"name\": {\n" +
" \"first name\": \"Steve\",\n" +
" \"last name\": \"Jobs\"\n" +
" }\n" +
"}";


public void testCompressesResponseIfRequested() throws IOException {
Request request = new Request("GET", "/");
RequestOptions.Builder options = request.getOptions().toBuilder();
options.addHeader(HttpHeaders.ACCEPT_ENCODING, GZIP_ENCODING);
request.setOptions(options);
Request request = new Request("POST", "/company/_doc/2");
request.setJsonEntity(SAMPLE_DOCUMENT);
Response response = client().performRequest(request);
assertEquals(201, response.getStatusLine().getStatusCode());
assertNull(response.getHeader(HttpHeaders.CONTENT_ENCODING));
assertThat(response.getEntity(), is(not(instanceOf(GzipDecompressingEntity.class))));

request = new Request("GET", "/company/_doc/2");
RequestOptions requestOptions = RequestOptions.DEFAULT.toBuilder()
.addHeader(HttpHeaders.ACCEPT_ENCODING, GZIP_ENCODING)
.build();

request.setOptions(requestOptions);
response = client().performRequest(request);
assertEquals(200, response.getStatusLine().getStatusCode());
assertEquals(GZIP_ENCODING, response.getHeader(HttpHeaders.CONTENT_ENCODING));
assertThat(response.getEntity(), instanceOf(GzipDecompressingEntity.class));

String body = EntityUtils.toString(response.getEntity());
assertThat(body, containsString(SAMPLE_DOCUMENT));
}

public void testUncompressedResponseByDefault() throws IOException {
Expand All @@ -57,6 +76,7 @@ public void testUncompressedResponseByDefault() throws IOException {
response = client().performRequest(request);
assertEquals(201, response.getStatusLine().getStatusCode());
assertNull(response.getHeader(HttpHeaders.CONTENT_ENCODING));
assertThat(response.getEntity(), is(not(instanceOf(GzipDecompressingEntity.class))));
}

}