Skip to content

Add the cluster version to enrich policies when they are created. #44595

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

Add the cluster version to enrich policies when they are created. #44595

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3fc1dac
Rename EnrichPolicy to EnrichPolicyDefinition
jbaiera Jul 16, 2019
31d3f96
Add new EnrichPolicy definition
jbaiera Jul 16, 2019
25b7288
Switch everything back over to using the new EnrichPolicy
jbaiera Jul 18, 2019
6f6711b
Merge branch 'enrich' into enrich-add-policy-metadata
jbaiera Jul 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
268 changes: 44 additions & 224 deletions x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,142 +5,93 @@
*/
package org.elasticsearch.xpack.core.enrich;

import java.io.IOException;
import java.util.Objects;

import org.elasticsearch.Version;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;

import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Objects;

/**
* Represents an enrich policy including its configuration.
*/
public final class EnrichPolicy implements Writeable, ToXContentFragment {
public class EnrichPolicy implements Writeable, ToXContentFragment {

public static final String ENRICH_INDEX_NAME_BASE = ".enrich-";

public static final String EXACT_MATCH_TYPE = "exact_match";
public static final String[] SUPPORTED_POLICY_TYPES = new String[]{EXACT_MATCH_TYPE};

private static final ParseField TYPE = new ParseField("type");
private static final ParseField QUERY = new ParseField("query");
private static final ParseField INDICES = new ParseField("indices");
private static final ParseField ENRICH_KEY = new ParseField("enrich_key");
private static final ParseField ENRICH_VALUES = new ParseField("enrich_values");
private static final ParseField NAME = new ParseField("name");
private static final ParseField VERSION_CREATED = new ParseField("version_created");
private static final ParseField DEFINITION = new ParseField("definition");

@SuppressWarnings("unchecked")
private static final ConstructingObjectParser<EnrichPolicy, Void> PARSER = new ConstructingObjectParser<>("policy",
args -> new EnrichPolicy(
(String) args[0],
(QuerySource) args[1],
(List<String>) args[2],
(String) args[3],
(List<String>) args[4]
(Version) args[1],
(EnrichPolicyDefinition) args[2]
)
);

static {
declareParserOptions(PARSER);
}

private static void declareParserOptions(ConstructingObjectParser<?, ?> parser) {
parser.declareString(ConstructingObjectParser.constructorArg(), TYPE);
parser.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
XContentBuilder contentBuilder = XContentBuilder.builder(p.contentType().xContent());
contentBuilder.generator().copyCurrentStructure(p);
return new QuerySource(BytesReference.bytes(contentBuilder), contentBuilder.contentType());
}, QUERY);
parser.declareStringArray(ConstructingObjectParser.constructorArg(), INDICES);
parser.declareString(ConstructingObjectParser.constructorArg(), ENRICH_KEY);
parser.declareStringArray(ConstructingObjectParser.constructorArg(), ENRICH_VALUES);
PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME);
PARSER.declareField(ConstructingObjectParser.constructorArg(), (p, c) -> Version.fromString(p.text()), VERSION_CREATED,
Copy link
Contributor

@jakelandis jakelandis Jul 23, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can newer ES versions create a Version object (via the fromString) that is not explicitly defined. e.g. what happens if version 6.8.0 is the version serialized can this read it ?

Copy link
Member Author

@jbaiera jbaiera Jul 24, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes - The fromString() method condenses the version string given down into the integral version id. If the version id isn't defined in the known list of versions, it is still created, but the version of lucene that it associates with the returned version is a guess. I don't foresee this being an issue, but I could be off base here.

ObjectParser.ValueType.STRING);
PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> EnrichPolicyDefinition.fromXContent(p), DEFINITION);
}

public static EnrichPolicy fromXContent(XContentParser parser) throws IOException {
return PARSER.parse(parser, null);
}

private final String type;
private final QuerySource query;
private final List<String> indices;
private final String enrichKey;
private final List<String> enrichValues;
private final String name;
private final Version versionCreated;
private final EnrichPolicyDefinition definition;

public EnrichPolicy(StreamInput in) throws IOException {
this(
in.readString(),
in.readOptionalWriteable(QuerySource::new),
in.readStringList(),
in.readString(),
in.readStringList()
Version.readVersion(in),
new EnrichPolicyDefinition(in)
);
}

public EnrichPolicy(String type,
QuerySource query,
List<String> indices,
String enrichKey,
List<String> enrichValues) {
this.type = type;
this.query= query;
this.indices = indices;
this.enrichKey = enrichKey;
this.enrichValues = enrichValues;
public EnrichPolicy(String name, Version versionCreated, EnrichPolicyDefinition definition) {
this.name = name;
this.versionCreated = versionCreated;
this.definition = definition;
}

public String getType() {
return type;
public String getName() {
return name;
}

public QuerySource getQuery() {
return query;
public Version getVersionCreated() {
return versionCreated;
}

public List<String> getIndices() {
return indices;
}

public String getEnrichKey() {
return enrichKey;
}

public List<String> getEnrichValues() {
return enrichValues;
}

public static String getBaseName(String policyName) {
return ENRICH_INDEX_NAME_BASE + policyName;
public EnrichPolicyDefinition getDefinition() {
return definition;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(type);
out.writeOptionalWriteable(query);
out.writeStringCollection(indices);
out.writeString(enrichKey);
out.writeStringCollection(enrichValues);
out.writeString(name);
Version.writeVersion(versionCreated, out);
definition.writeTo(out);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(TYPE.getPreferredName(), type);
if (query != null) {
builder.field(QUERY.getPreferredName(), query.getQueryAsMap());
}
builder.array(INDICES.getPreferredName(), indices.toArray(new String[0]));
builder.field(ENRICH_KEY.getPreferredName(), enrichKey);
builder.array(ENRICH_VALUES.getPreferredName(), enrichValues.toArray(new String[0]));
builder.field(NAME.getPreferredName(), name);
builder.field(VERSION_CREATED.getPreferredName());
versionCreated.toXContent(builder, params);
builder.startObject(DEFINITION.getPreferredName());
Copy link
Contributor

@jakelandis jakelandis Jul 23, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

does the definition need be a sub object ?

definition.toXContent(builder, params);
builder.endObject();
return builder;
}

Expand All @@ -149,149 +100,18 @@ public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
EnrichPolicy policy = (EnrichPolicy) o;
return type.equals(policy.type) &&
Objects.equals(query, policy.query) &&
indices.equals(policy.indices) &&
enrichKey.equals(policy.enrichKey) &&
enrichValues.equals(policy.enrichValues);
return name.equals(policy.name) &&
Objects.equals(versionCreated, policy.versionCreated) &&
Objects.equals(definition, policy.definition);
}

@Override
public int hashCode() {
return Objects.hash(
type,
query,
indices,
enrichKey,
enrichValues
);
return Objects.hash(name, versionCreated, definition);
}

@Override
public String toString() {
return Strings.toString(this);
}

public static class QuerySource implements Writeable {

private final BytesReference query;
private final XContentType contentType;

QuerySource(StreamInput in) throws IOException {
this(in.readBytesReference(), in.readEnum(XContentType.class));
}

public QuerySource(BytesReference query, XContentType contentType) {
this.query = query;
this.contentType = contentType;
}

public BytesReference getQuery() {
return query;
}

public Map<String, Object> getQueryAsMap() {
return XContentHelper.convertToMap(query, true, contentType).v2();
}

public XContentType getContentType() {
return contentType;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeBytesReference(query);
out.writeEnum(contentType);
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
QuerySource that = (QuerySource) o;
return query.equals(that.query) &&
contentType == that.contentType;
}

@Override
public int hashCode() {
return Objects.hash(query, contentType);
}
}

public static class NamedPolicy implements Writeable, ToXContent {

static final ParseField NAME = new ParseField("name");
@SuppressWarnings("unchecked")
static final ConstructingObjectParser<NamedPolicy, Void> PARSER = new ConstructingObjectParser<>("named_policy",
args -> new NamedPolicy(
(String) args[0],
new EnrichPolicy((String) args[1],
(QuerySource) args[2],
(List<String>) args[3],
(String) args[4],
(List<String>) args[5])
)
);

static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME);
declareParserOptions(PARSER);
}

private final String name;
private final EnrichPolicy policy;

public NamedPolicy(String name, EnrichPolicy policy) {
this.name = name;
this.policy = policy;
}

public NamedPolicy(StreamInput in) throws IOException {
name = in.readString();
policy = new EnrichPolicy(in);
}

public String getName() {
return name;
}

public EnrichPolicy getPolicy() {
return policy;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(name);
policy.writeTo(out);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
{
builder.field(NAME.getPreferredName(), name);
policy.toXContent(builder, params);
}
builder.endObject();
return builder;
}

public static NamedPolicy fromXContent(XContentParser parser) throws IOException {
return PARSER.parse(parser, null);
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
NamedPolicy that = (NamedPolicy) o;
return name.equals(that.name) &&
policy.equals(that.policy);
}

@Override
public int hashCode() {
return Objects.hash(name, policy);
}
}
}
Loading