From 80743f092f3112327655044793d5084986a62995 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Mon, 29 Jul 2019 16:21:48 -0400 Subject: [PATCH 1/6] Add version created to the enrich policy directly --- .../xpack/core/enrich/EnrichPolicy.java | 40 ++++++++++++++++--- .../enrich/action/PutEnrichPolicyAction.java | 6 +++ .../rest-api-spec/test/enrich/10_basic.yml | 2 + .../xpack/enrich/EnrichStore.java | 4 ++ .../xpack/enrich/EnrichMultiNodeIT.java | 8 +++- .../xpack/enrich/EnrichRestartIT.java | 20 ++++++++-- 6 files changed, 70 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java index ece8c6b002316..a1f7fadcd74c8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.enrich; +import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; @@ -12,6 +13,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -39,6 +41,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { private static final ParseField INDICES = new ParseField("indices"); private static final ParseField ENRICH_KEY = new ParseField("enrich_key"); private static final ParseField ENRICH_VALUES = new ParseField("enrich_values"); + private static final ParseField VERSION_CREATED = new ParseField("version_created"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("policy", @@ -52,10 +55,11 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { ); static { - declareParserOptions(PARSER); + declareCommonConstructorParsingOptions(PARSER); + PARSER.declareField(EnrichPolicy::setVersionCreated, ((p, c) -> Version.fromString(p.text())), VERSION_CREATED, ValueType.STRING); } - private static void declareParserOptions(ConstructingObjectParser parser) { + private static void declareCommonConstructorParsingOptions(ConstructingObjectParser parser) { parser.declareString(ConstructingObjectParser.constructorArg(), TYPE); parser.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { XContentBuilder contentBuilder = XContentBuilder.builder(p.contentType().xContent()); @@ -76,6 +80,7 @@ public static EnrichPolicy fromXContent(XContentParser parser) throws IOExceptio private final List indices; private final String enrichKey; private final List enrichValues; + private Version versionCreated; public EnrichPolicy(StreamInput in) throws IOException { this( @@ -85,6 +90,9 @@ public EnrichPolicy(StreamInput in) throws IOException { in.readString(), in.readStringList() ); + if (in.readBoolean()) { + setVersionCreated(Version.readVersion(in)); + } } public EnrichPolicy(String type, @@ -97,6 +105,7 @@ public EnrichPolicy(String type, this.indices = indices; this.enrichKey = enrichKey; this.enrichValues = enrichValues; + this.versionCreated = null; } public String getType() { @@ -119,6 +128,14 @@ public List getEnrichValues() { return enrichValues; } + public Version getVersionCreated() { + return versionCreated; + } + + public void setVersionCreated(Version versionCreated) { + this.versionCreated = versionCreated; + } + public static String getBaseName(String policyName) { return ENRICH_INDEX_NAME_BASE + policyName; } @@ -130,6 +147,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeStringCollection(indices); out.writeString(enrichKey); out.writeStringCollection(enrichValues); + if (versionCreated != null) { + out.writeBoolean(true); + Version.writeVersion(versionCreated, out); + } else { + out.writeBoolean(false); + } } @Override @@ -141,6 +164,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.array(INDICES.getPreferredName(), indices.toArray(new String[0])); builder.field(ENRICH_KEY.getPreferredName(), enrichKey); builder.array(ENRICH_VALUES.getPreferredName(), enrichValues.toArray(new String[0])); + if (versionCreated != null) { + builder.field(VERSION_CREATED.getPreferredName(), versionCreated.toString()); + } return builder; } @@ -153,7 +179,8 @@ public boolean equals(Object o) { Objects.equals(query, policy.query) && indices.equals(policy.indices) && enrichKey.equals(policy.enrichKey) && - enrichValues.equals(policy.enrichValues); + enrichValues.equals(policy.enrichValues) && + Objects.equals(versionCreated, policy.versionCreated); } @Override @@ -163,7 +190,8 @@ public int hashCode() { query, indices, enrichKey, - enrichValues + enrichValues, + versionCreated ); } @@ -235,7 +263,9 @@ public static class NamedPolicy implements Writeable, ToXContent { static { PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME); - declareParserOptions(PARSER); + declareCommonConstructorParsingOptions(PARSER); + PARSER.declareField((policy, version) -> policy.policy.setVersionCreated(version), ((p, c) -> Version.fromString(p.text())), + VERSION_CREATED, ValueType.STRING); } private final String name; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/PutEnrichPolicyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/PutEnrichPolicyAction.java index 10c18932be28a..df285175400af 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/PutEnrichPolicyAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/PutEnrichPolicyAction.java @@ -63,6 +63,12 @@ public EnrichPolicy getPolicy() { @Override public ActionRequestValidationException validate() { + if (policy.getVersionCreated() != null) { + ActionRequestValidationException e = new ActionRequestValidationException(); + e.addValidationError("Cannot set [version_created] field on enrich policy [" + name + "]. Found [" + + policy.getVersionCreated() + "]"); + return e; + } return null; } diff --git a/x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml b/x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml index 42d0020049a7d..d6f62ad38ac49 100644 --- a/x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml +++ b/x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml @@ -23,6 +23,7 @@ - match: { indices: ["bar*"] } - match: { enrich_key: baz } - match: { enrich_values: ["a", "b"] } + - is_true: version_created - do: enrich.list_policy: {} @@ -32,6 +33,7 @@ - match: { policies.0.indices: ["bar*"] } - match: { policies.0.enrich_key: baz } - match: { policies.0.enrich_values: ["a", "b"] } + - is_true: policies.0.version_created - do: enrich.delete_policy: diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichStore.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichStore.java index 36b399b3d3d07..ff1c229344651 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichStore.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichStore.java @@ -7,6 +7,7 @@ import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.metadata.MetaData; @@ -45,6 +46,9 @@ public static void putPolicy(String name, EnrichPolicy policy, ClusterService cl } // TODO: add policy validation + if (policy.getVersionCreated() == null) { + policy.setVersionCreated(Version.CURRENT); + } updateClusterState(clusterService, handler, current -> { final Map policies = getPolicies(current); if (policies.get(name) != null) { diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichMultiNodeIT.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichMultiNodeIT.java index ff8262d6ceb04..63e1541e99162 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichMultiNodeIT.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichMultiNodeIT.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.enrich; import org.apache.lucene.search.TotalHits; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; @@ -69,7 +70,12 @@ public void testEnrichAPIs() { EnrichPolicy result = client().execute(GetEnrichPolicyAction.INSTANCE, new GetEnrichPolicyAction.Request(policyName)).actionGet().getPolicy(); - assertThat(result, equalTo(enrichPolicy)); + assertThat(result.getType(), equalTo(enrichPolicy.getType())); + assertThat(result.getQuery(), equalTo(enrichPolicy.getQuery())); + assertThat(result.getIndices(), equalTo(enrichPolicy.getIndices())); + assertThat(result.getEnrichKey(), equalTo(enrichPolicy.getEnrichKey())); + assertThat(result.getEnrichValues(), equalTo(enrichPolicy.getEnrichValues())); + assertThat(result.getVersionCreated(), equalTo(Version.CURRENT)); String enrichIndexPrefix = EnrichPolicy.getBaseName(policyName) + "*"; refresh(enrichIndexPrefix); SearchResponse searchResponse = client().search(new SearchRequest(enrichIndexPrefix)).actionGet(); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java index 2cdf5ed3afd8d..50b73bbe58661 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.enrich; +import org.elasticsearch.Version; import org.elasticsearch.index.reindex.ReindexPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -35,11 +36,13 @@ public void testRestart() throws Exception { final int numPolicies = randomIntBetween(2, 4); internalCluster().startNode(); - EnrichPolicy enrichPolicy = - new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, List.of(SOURCE_INDEX_NAME), KEY_FIELD, List.of(DECORATE_FIELDS)); + EnrichPolicy enrichPolicy = newPolicy(); for (int i = 0; i < numPolicies; i++) { String policyName = POLICY_NAME + i; - PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(policyName, enrichPolicy); + PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(policyName, newPolicy()); + if (request.validate() != null) { + throw request.validate(); + } client().execute(PutEnrichPolicyAction.INSTANCE, request).actionGet(); } @@ -49,6 +52,10 @@ public void testRestart() throws Exception { verifyPolicies(numPolicies, enrichPolicy); } + private EnrichPolicy newPolicy() { + return new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, List.of(SOURCE_INDEX_NAME), KEY_FIELD, List.of(DECORATE_FIELDS)); + } + private static void verifyPolicies(int numPolicies, EnrichPolicy enrichPolicy) { ListEnrichPolicyAction.Response response = client().execute(ListEnrichPolicyAction.INSTANCE, new ListEnrichPolicyAction.Request()).actionGet(); @@ -59,7 +66,12 @@ private static void verifyPolicies(int numPolicies, EnrichPolicy enrichPolicy) { .filter(namedPolicy -> namedPolicy.getName().equals(policyName)) .findFirst(); assertThat(result.isPresent(), is(true)); - assertThat(result.get().getPolicy(), equalTo(enrichPolicy)); + assertThat(result.get().getPolicy().getType(), equalTo(enrichPolicy.getType())); + assertThat(result.get().getPolicy().getQuery(), equalTo(enrichPolicy.getQuery())); + assertThat(result.get().getPolicy().getIndices(), equalTo(enrichPolicy.getIndices())); + assertThat(result.get().getPolicy().getEnrichKey(), equalTo(enrichPolicy.getEnrichKey())); + assertThat(result.get().getPolicy().getEnrichValues(), equalTo(enrichPolicy.getEnrichValues())); + assertThat(result.get().getPolicy().getVersionCreated(), equalTo(Version.CURRENT)); } } From e5f18ba4c7d923f88480a8109e9ebbd76f91e6d3 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Mon, 5 Aug 2019 15:13:45 -0400 Subject: [PATCH 2/6] Make version created on EnrichPolicy final Add it as a constructor argument. Only serialize it to XContent if include_version is set --- .../xpack/core/enrich/EnrichPolicy.java | 39 ++++++++++++------- .../xpack/enrich/EnrichStore.java | 14 ++++++- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java index a1f7fadcd74c8..fdfcc715c18ac 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java @@ -50,13 +50,13 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { (QuerySource) args[1], (List) args[2], (String) args[3], - (List) args[4] + (List) args[4], + (Version) args[5] ) ); static { declareCommonConstructorParsingOptions(PARSER); - PARSER.declareField(EnrichPolicy::setVersionCreated, ((p, c) -> Version.fromString(p.text())), VERSION_CREATED, ValueType.STRING); } private static void declareCommonConstructorParsingOptions(ConstructingObjectParser parser) { @@ -69,6 +69,8 @@ private static void declareCommonConstructorParsingOptions(ConstructingObjec parser.declareStringArray(ConstructingObjectParser.constructorArg(), INDICES); parser.declareString(ConstructingObjectParser.constructorArg(), ENRICH_KEY); parser.declareStringArray(ConstructingObjectParser.constructorArg(), ENRICH_VALUES); + parser.declareField(ConstructingObjectParser.optionalConstructorArg(), ((p, c) -> Version.fromString(p.text())), + VERSION_CREATED, ValueType.STRING); } public static EnrichPolicy fromXContent(XContentParser parser) throws IOException { @@ -80,7 +82,7 @@ public static EnrichPolicy fromXContent(XContentParser parser) throws IOExceptio private final List indices; private final String enrichKey; private final List enrichValues; - private Version versionCreated; + private final Version versionCreated; public EnrichPolicy(StreamInput in) throws IOException { this( @@ -88,11 +90,9 @@ public EnrichPolicy(StreamInput in) throws IOException { in.readOptionalWriteable(QuerySource::new), in.readStringList(), in.readString(), - in.readStringList() + in.readStringList(), + in.readBoolean() ? Version.readVersion(in) : null ); - if (in.readBoolean()) { - setVersionCreated(Version.readVersion(in)); - } } public EnrichPolicy(String type, @@ -108,6 +108,20 @@ public EnrichPolicy(String type, this.versionCreated = null; } + public EnrichPolicy(String type, + QuerySource query, + List indices, + String enrichKey, + List enrichValues, + Version versionCreated) { + this.type = type; + this.query = query; + this.indices = indices; + this.enrichKey = enrichKey; + this.enrichValues = enrichValues; + this.versionCreated = versionCreated; + } + public String getType() { return type; } @@ -132,10 +146,6 @@ public Version getVersionCreated() { return versionCreated; } - public void setVersionCreated(Version versionCreated) { - this.versionCreated = versionCreated; - } - public static String getBaseName(String policyName) { return ENRICH_INDEX_NAME_BASE + policyName; } @@ -164,7 +174,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.array(INDICES.getPreferredName(), indices.toArray(new String[0])); builder.field(ENRICH_KEY.getPreferredName(), enrichKey); builder.array(ENRICH_VALUES.getPreferredName(), enrichValues.toArray(new String[0])); - if (versionCreated != null) { + if (params.paramAsBoolean("include_version", false) && versionCreated != null) { builder.field(VERSION_CREATED.getPreferredName(), versionCreated.toString()); } return builder; @@ -257,15 +267,14 @@ public static class NamedPolicy implements Writeable, ToXContent { (QuerySource) args[2], (List) args[3], (String) args[4], - (List) args[5]) + (List) args[5], + (Version) args[6]) ) ); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME); declareCommonConstructorParsingOptions(PARSER); - PARSER.declareField((policy, version) -> policy.policy.setVersionCreated(version), ((p, c) -> Version.fromString(p.text())), - VERSION_CREATED, ValueType.STRING); } private final String name; diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichStore.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichStore.java index ff1c229344651..a417d83580328 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichStore.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichStore.java @@ -46,15 +46,25 @@ public static void putPolicy(String name, EnrichPolicy policy, ClusterService cl } // TODO: add policy validation + final EnrichPolicy finalPolicy; if (policy.getVersionCreated() == null) { - policy.setVersionCreated(Version.CURRENT); + finalPolicy = new EnrichPolicy( + policy.getType(), + policy.getQuery(), + policy.getIndices(), + policy.getEnrichKey(), + policy.getEnrichValues(), + Version.CURRENT + ); + } else { + finalPolicy = policy; } updateClusterState(clusterService, handler, current -> { final Map policies = getPolicies(current); if (policies.get(name) != null) { throw new ResourceAlreadyExistsException("policy [{}] already exists", name); } - policies.put(name, policy); + policies.put(name, finalPolicy); return policies; }); } From 4aa67ca548e5d20cceb182a81682e1821735a5f5 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Mon, 5 Aug 2019 16:31:01 -0400 Subject: [PATCH 3/6] Fix tests --- .../resources/rest-api-spec/test/enrich/10_basic.yml | 2 -- .../xpack/enrich/EnrichPolicyUpdateTests.java | 7 ++++++- .../elasticsearch/xpack/enrich/EnrichRestartIT.java | 2 -- .../elasticsearch/xpack/enrich/EnrichStoreTests.java | 12 ++++++++++-- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml b/x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml index d6f62ad38ac49..42d0020049a7d 100644 --- a/x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml +++ b/x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml @@ -23,7 +23,6 @@ - match: { indices: ["bar*"] } - match: { enrich_key: baz } - match: { enrich_values: ["a", "b"] } - - is_true: version_created - do: enrich.list_policy: {} @@ -33,7 +32,6 @@ - match: { policies.0.indices: ["bar*"] } - match: { policies.0.enrich_key: baz } - match: { policies.0.enrich_values: ["a", "b"] } - - is_true: policies.0.version_created - do: enrich.delete_policy: diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyUpdateTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyUpdateTests.java index c834048899d75..62788a9bac178 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyUpdateTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyUpdateTests.java @@ -39,7 +39,12 @@ public void testUpdatePolicyOnly() { new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, List.of("index"), "key1", List.of("field1")); PutEnrichPolicyAction.Request putPolicyRequest = new PutEnrichPolicyAction.Request("my_policy", instance1); assertAcked(client().execute(PutEnrichPolicyAction.INSTANCE, putPolicyRequest).actionGet()); - assertThat(enrichProcessorFactory.policies.get("my_policy"), equalTo(instance1)); + EnrichPolicy actual = enrichProcessorFactory.policies.get("my_policy"); + assertThat(actual.getType(), equalTo(instance1.getType())); + assertThat(actual.getEnrichKey(), equalTo(instance1.getEnrichKey())); + assertThat(actual.getEnrichValues(), equalTo(instance1.getEnrichValues())); + assertThat(actual.getIndices(), equalTo(instance1.getIndices())); + assertThat(actual.getQuery(), equalTo(instance1.getQuery())); String pipelineConfig = "{\"processors\":[{\"enrich\": {\"policy_name\": \"my_policy\", \"enrich_values\": []}}]}"; PutPipelineRequest putPipelineRequest = new PutPipelineRequest("1", new BytesArray(pipelineConfig), XContentType.JSON); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java index 50b73bbe58661..1df870f1e0170 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.enrich; -import org.elasticsearch.Version; import org.elasticsearch.index.reindex.ReindexPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -71,7 +70,6 @@ private static void verifyPolicies(int numPolicies, EnrichPolicy enrichPolicy) { assertThat(result.get().getPolicy().getIndices(), equalTo(enrichPolicy.getIndices())); assertThat(result.get().getPolicy().getEnrichKey(), equalTo(enrichPolicy.getEnrichKey())); assertThat(result.get().getPolicy().getEnrichValues(), equalTo(enrichPolicy.getEnrichValues())); - assertThat(result.get().getPolicy().getVersionCreated(), equalTo(Version.CURRENT)); } } diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichStoreTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichStoreTests.java index 9a398b2bceb0b..7682f5fb0ffbe 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichStoreTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichStoreTests.java @@ -38,11 +38,19 @@ public void testCrud() throws Exception { assertThat(error.get(), nullValue()); EnrichPolicy result = EnrichStore.getPolicy(name, clusterService.state()); - assertThat(result, equalTo(policy)); + assertThat(result.getType(), equalTo(policy.getType())); + assertThat(result.getEnrichKey(), equalTo(policy.getEnrichKey())); + assertThat(result.getEnrichValues(), equalTo(policy.getEnrichValues())); + assertThat(result.getIndices(), equalTo(policy.getIndices())); + assertThat(result.getQuery(), equalTo(policy.getQuery())); Map listPolicies = EnrichStore.getPolicies(clusterService.state()); assertThat(listPolicies.size(), equalTo(1)); - assertThat(listPolicies.get(name), equalTo(policy)); + assertThat(listPolicies.get(name).getType(), equalTo(policy.getType())); + assertThat(listPolicies.get(name).getEnrichKey(), equalTo(policy.getEnrichKey())); + assertThat(listPolicies.get(name).getEnrichValues(), equalTo(policy.getEnrichValues())); + assertThat(listPolicies.get(name).getIndices(), equalTo(policy.getIndices())); + assertThat(listPolicies.get(name).getQuery(), equalTo(policy.getQuery())); deleteEnrichPolicy(name, clusterService); result = EnrichStore.getPolicy(name, clusterService.state()); From d10935f7346da9e839c42a4a75983646bb16f427 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Wed, 7 Aug 2019 17:01:02 -0400 Subject: [PATCH 4/6] Set version_created to CURRENT if not given --- .../xpack/core/enrich/EnrichPolicy.java | 20 +++++-------------- .../enrich/action/PutEnrichPolicyAction.java | 11 +++++----- .../xpack/enrich/EnrichMultiNodeIT.java | 8 +------- .../xpack/enrich/EnrichPolicyTests.java | 4 +++- .../xpack/enrich/EnrichPolicyUpdateTests.java | 7 +------ .../xpack/enrich/EnrichRestartIT.java | 14 ++++--------- .../xpack/enrich/EnrichStoreTests.java | 12 ++--------- 7 files changed, 21 insertions(+), 55 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java index fdfcc715c18ac..d488a44dcfe01 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java @@ -91,7 +91,7 @@ public EnrichPolicy(StreamInput in) throws IOException { in.readStringList(), in.readString(), in.readStringList(), - in.readBoolean() ? Version.readVersion(in) : null + Version.readVersion(in) ); } @@ -100,12 +100,7 @@ public EnrichPolicy(String type, List indices, String enrichKey, List enrichValues) { - this.type = type; - this.query= query; - this.indices = indices; - this.enrichKey = enrichKey; - this.enrichValues = enrichValues; - this.versionCreated = null; + this(type, query, indices, enrichKey, enrichValues, Version.CURRENT); } public EnrichPolicy(String type, @@ -119,7 +114,7 @@ public EnrichPolicy(String type, this.indices = indices; this.enrichKey = enrichKey; this.enrichValues = enrichValues; - this.versionCreated = versionCreated; + this.versionCreated = versionCreated != null ? versionCreated : Version.CURRENT; } public String getType() { @@ -157,12 +152,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeStringCollection(indices); out.writeString(enrichKey); out.writeStringCollection(enrichValues); - if (versionCreated != null) { - out.writeBoolean(true); - Version.writeVersion(versionCreated, out); - } else { - out.writeBoolean(false); - } + Version.writeVersion(versionCreated, out); } @Override @@ -190,7 +180,7 @@ public boolean equals(Object o) { indices.equals(policy.indices) && enrichKey.equals(policy.enrichKey) && enrichValues.equals(policy.enrichValues) && - Objects.equals(versionCreated, policy.versionCreated); + versionCreated.equals(policy.versionCreated); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/PutEnrichPolicyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/PutEnrichPolicyAction.java index df285175400af..6c85402fb29e3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/PutEnrichPolicyAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/PutEnrichPolicyAction.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.enrich.action; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -37,6 +38,10 @@ public static class Request extends MasterNodeRequest namedPolicy.getName().equals(policyName)) .findFirst(); assertThat(result.isPresent(), is(true)); - assertThat(result.get().getPolicy().getType(), equalTo(enrichPolicy.getType())); - assertThat(result.get().getPolicy().getQuery(), equalTo(enrichPolicy.getQuery())); - assertThat(result.get().getPolicy().getIndices(), equalTo(enrichPolicy.getIndices())); - assertThat(result.get().getPolicy().getEnrichKey(), equalTo(enrichPolicy.getEnrichKey())); - assertThat(result.get().getPolicy().getEnrichValues(), equalTo(enrichPolicy.getEnrichValues())); + assertThat(result.get().getPolicy(), equalTo(enrichPolicy)); } } diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichStoreTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichStoreTests.java index 7682f5fb0ffbe..9a398b2bceb0b 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichStoreTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichStoreTests.java @@ -38,19 +38,11 @@ public void testCrud() throws Exception { assertThat(error.get(), nullValue()); EnrichPolicy result = EnrichStore.getPolicy(name, clusterService.state()); - assertThat(result.getType(), equalTo(policy.getType())); - assertThat(result.getEnrichKey(), equalTo(policy.getEnrichKey())); - assertThat(result.getEnrichValues(), equalTo(policy.getEnrichValues())); - assertThat(result.getIndices(), equalTo(policy.getIndices())); - assertThat(result.getQuery(), equalTo(policy.getQuery())); + assertThat(result, equalTo(policy)); Map listPolicies = EnrichStore.getPolicies(clusterService.state()); assertThat(listPolicies.size(), equalTo(1)); - assertThat(listPolicies.get(name).getType(), equalTo(policy.getType())); - assertThat(listPolicies.get(name).getEnrichKey(), equalTo(policy.getEnrichKey())); - assertThat(listPolicies.get(name).getEnrichValues(), equalTo(policy.getEnrichValues())); - assertThat(listPolicies.get(name).getIndices(), equalTo(policy.getIndices())); - assertThat(listPolicies.get(name).getQuery(), equalTo(policy.getQuery())); + assertThat(listPolicies.get(name), equalTo(policy)); deleteEnrichPolicy(name, clusterService); result = EnrichStore.getPolicy(name, clusterService.state()); From 19769726df1c792a0ba2c3898799af560012e937 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Wed, 7 Aug 2019 17:13:55 -0400 Subject: [PATCH 5/6] Cleanup --- .../org/elasticsearch/xpack/enrich/EnrichPolicyTests.java | 4 +--- .../java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java | 4 ---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java index 1ce430ba7c7ee..b481004506556 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.enrich; -import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ToXContent; @@ -62,8 +61,7 @@ public static EnrichPolicy randomEnrichPolicy(XContentType xContentType) { randomBoolean() ? querySource : null, Arrays.asList(generateRandomStringArray(8, 4, false, false)), randomAlphaOfLength(4), - Arrays.asList(generateRandomStringArray(8, 4, false, false)), - Version.CURRENT + Arrays.asList(generateRandomStringArray(8, 4, false, false)) ); } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java index ed901db8defde..2cdf5ed3afd8d 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java @@ -49,10 +49,6 @@ public void testRestart() throws Exception { verifyPolicies(numPolicies, enrichPolicy); } - private EnrichPolicy newPolicy() { - return new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, List.of(SOURCE_INDEX_NAME), KEY_FIELD, List.of(DECORATE_FIELDS)); - } - private static void verifyPolicies(int numPolicies, EnrichPolicy enrichPolicy) { ListEnrichPolicyAction.Response response = client().execute(ListEnrichPolicyAction.INSTANCE, new ListEnrichPolicyAction.Request()).actionGet(); From 86bd726b3246e9fd288b7a07d5389f87c22e9ea6 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Fri, 23 Aug 2019 11:41:53 -0400 Subject: [PATCH 6/6] Change current_version to elasticsearch_version --- .../xpack/core/enrich/EnrichPolicy.java | 24 +++++++++---------- .../enrich/action/PutEnrichPolicyAction.java | 4 ++-- .../xpack/enrich/EnrichStore.java | 6 ++--- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java index 0e31406752168..92102001304ae 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java @@ -41,7 +41,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { private static final ParseField INDICES = new ParseField("indices"); private static final ParseField MATCH_FIELD = new ParseField("match_field"); private static final ParseField ENRICH_FIELDS = new ParseField("enrich_fields"); - private static final ParseField VERSION_CREATED = new ParseField("version_created"); + private static final ParseField ELASTICSEARCH_VERSION = new ParseField("elasticsearch_version"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("policy", @@ -70,7 +70,7 @@ private static void declareCommonConstructorParsingOptions(ConstructingObjec parser.declareString(ConstructingObjectParser.constructorArg(), MATCH_FIELD); parser.declareStringArray(ConstructingObjectParser.constructorArg(), ENRICH_FIELDS); parser.declareField(ConstructingObjectParser.optionalConstructorArg(), ((p, c) -> Version.fromString(p.text())), - VERSION_CREATED, ValueType.STRING); + ELASTICSEARCH_VERSION, ValueType.STRING); } public static EnrichPolicy fromXContent(XContentParser parser) throws IOException { @@ -82,7 +82,7 @@ public static EnrichPolicy fromXContent(XContentParser parser) throws IOExceptio private final List indices; private final String matchField; private final List enrichFields; - private final Version versionCreated; + private final Version elasticsearchVersion; public EnrichPolicy(StreamInput in) throws IOException { this( @@ -108,13 +108,13 @@ public EnrichPolicy(String type, List indices, String matchField, List enrichFields, - Version versionCreated) { + Version elasticsearchVersion) { this.type = type; this.query = query; this.indices = indices; this.matchField = matchField; this.enrichFields = enrichFields; - this.versionCreated = versionCreated != null ? versionCreated : Version.CURRENT; + this.elasticsearchVersion = elasticsearchVersion != null ? elasticsearchVersion : Version.CURRENT; } public String getType() { @@ -137,8 +137,8 @@ public List getEnrichFields() { return enrichFields; } - public Version getVersionCreated() { - return versionCreated; + public Version getElasticsearchVersion() { + return elasticsearchVersion; } public static String getBaseName(String policyName) { @@ -152,7 +152,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeStringCollection(indices); out.writeString(matchField); out.writeStringCollection(enrichFields); - Version.writeVersion(versionCreated, out); + Version.writeVersion(elasticsearchVersion, out); } @Override @@ -164,8 +164,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.array(INDICES.getPreferredName(), indices.toArray(new String[0])); builder.field(MATCH_FIELD.getPreferredName(), matchField); builder.array(ENRICH_FIELDS.getPreferredName(), enrichFields.toArray(new String[0])); - if (params.paramAsBoolean("include_version", false) && versionCreated != null) { - builder.field(VERSION_CREATED.getPreferredName(), versionCreated.toString()); + if (params.paramAsBoolean("include_version", false) && elasticsearchVersion != null) { + builder.field(ELASTICSEARCH_VERSION.getPreferredName(), elasticsearchVersion.toString()); } return builder; } @@ -180,7 +180,7 @@ public boolean equals(Object o) { indices.equals(policy.indices) && matchField.equals(policy.matchField) && enrichFields.equals(policy.enrichFields) && - versionCreated.equals(policy.versionCreated); + elasticsearchVersion.equals(policy.elasticsearchVersion); } @Override @@ -191,7 +191,7 @@ public int hashCode() { indices, matchField, enrichFields, - versionCreated + elasticsearchVersion ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/PutEnrichPolicyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/PutEnrichPolicyAction.java index 6c85402fb29e3..d30ba4c383e6e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/PutEnrichPolicyAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/PutEnrichPolicyAction.java @@ -38,9 +38,9 @@ public static class Request extends MasterNodeRequest