From 4f4271dd47e86d8b508b405f1ccf97be985601ab Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 14 Apr 2020 13:05:01 -0400 Subject: [PATCH 1/6] Fix BWC issues for x_pack/usage Fixes several BWC issues in x_pack/usage discovered after introduction of the bwc test. Relates to #54847 --- .../action/AnalyticsUsageTransportAction.java | 17 +---- .../TransportAnalyticsStatsActionTests.java | 3 +- .../xpack/core/XPackClientPlugin.java | 8 ++- .../elasticsearch/xpack/core/XPackField.java | 3 +- .../core/action/XPackUsageFeatureAction.java | 1 + .../xpack/core/action/XPackUsageResponse.java | 9 ++- .../analytics/AnalyticsFeatureSetUsage.java | 52 ++++++--------- .../action/AnalyticsStatsAction.java | 23 ++++++- .../core/enrich/EnrichFeatureSetUsage.java | 24 +++++++ .../flattened/FlattenedFeatureSetUsage.java | 63 +++++++++++++++++++ .../xpack/enrich/EnrichPlugin.java | 3 + .../action/EnrichUsageTransportAction.java | 62 ++++++++++++++++++ .../mixed_cluster/100_analytics_usage.yml | 60 ++++++++++++++++++ 13 files changed, 276 insertions(+), 52 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichFeatureSetUsage.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java create mode 100644 x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichUsageTransportAction.java create mode 100644 x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/100_analytics_usage.yml diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsUsageTransportAction.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsUsageTransportAction.java index bb6da79efbe84..de46da42f893f 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsUsageTransportAction.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsUsageTransportAction.java @@ -21,12 +21,9 @@ import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage; -import org.elasticsearch.xpack.core.analytics.EnumCounters; import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction; import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; public class AnalyticsUsageTransportAction extends XPackUsageFeatureTransportAction { private final XPackLicenseState licenseState; @@ -50,20 +47,12 @@ protected void masterOperation(Task task, XPackUsageRequest request, ClusterStat AnalyticsStatsAction.Request statsRequest = new AnalyticsStatsAction.Request(); statsRequest.setParentTask(clusterService.localNode().getId(), task.getId()); client.execute(AnalyticsStatsAction.INSTANCE, statsRequest, ActionListener.wrap(r -> - listener.onResponse(new XPackUsageFeatureResponse(usageFeatureResponse(true, true, r))), + listener.onResponse(new XPackUsageFeatureResponse(new AnalyticsFeatureSetUsage(true, true, r))), listener::onFailure)); } else { - AnalyticsFeatureSetUsage usage = new AnalyticsFeatureSetUsage(false, true, Collections.emptyMap()); + AnalyticsFeatureSetUsage usage = new AnalyticsFeatureSetUsage(false, true, + new AnalyticsStatsAction.Response(state.getClusterName(), Collections.emptyList(), Collections.emptyList())); listener.onResponse(new XPackUsageFeatureResponse(usage)); } } - - static AnalyticsFeatureSetUsage usageFeatureResponse(boolean available, boolean enabled, AnalyticsStatsAction.Response r) { - List> countersPerNode = r.getNodes() - .stream() - .map(AnalyticsStatsAction.NodeResponse::getStats) - .collect(Collectors.toList()); - EnumCounters mergedCounters = EnumCounters.merge(AnalyticsStatsAction.Item.class, countersPerNode); - return new AnalyticsFeatureSetUsage(available, enabled, mergedCounters.toMap()); - } } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java index 49047d41de190..822f284b299ef 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java @@ -34,7 +34,6 @@ import static java.util.Collections.emptyList; import static java.util.stream.Collectors.toList; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.xpack.analytics.action.AnalyticsUsageTransportAction.usageFeatureResponse; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -83,7 +82,7 @@ private ObjectPath run(AnalyticsUsage... nodeUsages) throws IOException { AnalyticsStatsAction.Response response = new AnalyticsStatsAction.Response( new ClusterName("cluster_name"), nodeResponses, emptyList()); - AnalyticsFeatureSetUsage usage = usageFeatureResponse(true, true, response); + AnalyticsFeatureSetUsage usage = new AnalyticsFeatureSetUsage(true, true, response); try (XContentBuilder builder = jsonBuilder()) { usage.toXContent(builder, ToXContent.EMPTY_PARAMS); return ObjectPath.createFromXContent(JsonXContent.jsonXContent, BytesReference.bytes(builder)); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 41e1d5950a9f3..1180018bcfdc2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -36,7 +36,9 @@ import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; +import org.elasticsearch.xpack.core.enrich.EnrichFeatureSetUsage; import org.elasticsearch.xpack.core.eql.EqlFeatureSetUsage; +import org.elasticsearch.xpack.core.flattened.FlattenedFeatureSetUsage; import org.elasticsearch.xpack.core.frozen.FrozenIndicesFeatureSetUsage; import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction; import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage; @@ -484,6 +486,8 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(Task.Status.class, TransformField.TASK_NAME, TransformState::new), new NamedWriteableRegistry.Entry(PersistentTaskState.class, TransformField.TASK_NAME, TransformState::new), new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(), TimeSyncConfig::new), + // Flattened for backward compatibility with 7.x + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.FLATTENED, FlattenedFeatureSetUsage::new), // Vectors new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.VECTORS, VectorsFeatureSetUsage::new), // Voting Only Node @@ -493,7 +497,9 @@ public List getNamedWriteables() { // Spatial new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SPATIAL, SpatialFeatureSetUsage::new), // Analytics - new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ANALYTICS, AnalyticsFeatureSetUsage::new) + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ANALYTICS, AnalyticsFeatureSetUsage::new), + // Enrich + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ENRICH, EnrichFeatureSetUsage::new) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index 1dbda1d320ee4..6a5986f5497d5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -41,7 +41,8 @@ public final class XPackField { public static final String CCR = "ccr"; /** Name constant for the transform feature. */ public static final String TRANSFORM = "transform"; - /** Name constant for flattened fields. */ + /** Name constant for flattened fields. -- For Backward Compatibility with 7.x */ + public static final String FLATTENED = "flattened"; /** Name constant for the vectors feature. */ public static final String VECTORS = "vectors"; /** Name constant for the voting-only-node feature. */ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java index c696fdeaa3e29..69178ad526ef1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java @@ -40,6 +40,7 @@ public class XPackUsageFeatureAction extends ActionType ALL = Arrays.asList( SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, EQL, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java index 29be3866e39ba..2d5f3fceda9dd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java @@ -23,7 +23,14 @@ public XPackUsageResponse(StreamInput in) throws IOException { int size = in.readVInt(); usages = new ArrayList<>(size); for (int i = 0; i < size; i++) { - usages.add(in.readNamedWriteable(XPackFeatureSet.Usage.class)); + String name = in.readString(); + if (name.equals("flattened")) { + // work-around for removed flattened + new XPackFeatureSet.Usage(in) { }; + in.readInt(); + } else { + usages.add(in.readNamedWriteable(XPackFeatureSet.Usage.class, name)); + } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/AnalyticsFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/AnalyticsFeatureSetUsage.java index 30b9119a5dd29..a7521b95f56da 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/AnalyticsFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/AnalyticsFeatureSetUsage.java @@ -6,47 +6,47 @@ package org.elasticsearch.xpack.core.analytics; -import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.XPackFeatureSet; import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction; import java.io.IOException; -import java.util.Collections; -import java.util.Map; import java.util.Objects; public class AnalyticsFeatureSetUsage extends XPackFeatureSet.Usage { - private final Map stats; + private final AnalyticsStatsAction.Response response; - public AnalyticsFeatureSetUsage(boolean available, boolean enabled, Map stats) { + public AnalyticsFeatureSetUsage(boolean available, boolean enabled, AnalyticsStatsAction.Response response) { super(XPackField.ANALYTICS, available, enabled); - this.stats = stats; + this.response = response; } public AnalyticsFeatureSetUsage(StreamInput input) throws IOException { super(input); - if (input.getVersion().onOrAfter(Version.V_7_8_0)) { - stats = input.readMap(); - } else { - stats = Collections.emptyMap(); - } + this.response = new AnalyticsStatsAction.Response(input); } @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_7_8_0)) { - out.writeMap(stats); + public int hashCode() { + return Objects.hash(available, enabled, response); + } + + @Override + protected void innerXContent(XContentBuilder builder, Params params) throws IOException { + super.innerXContent(builder, params); + if (response != null) { + response.toXContent(builder, params); } } @Override - public int hashCode() { - return Objects.hash(available, enabled, stats); + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + response.writeTo(out); } @Override @@ -58,20 +58,8 @@ public boolean equals(Object obj) { return false; } AnalyticsFeatureSetUsage other = (AnalyticsFeatureSetUsage) obj; - return Objects.equals(available, other.available) && - Objects.equals(enabled, other.enabled) && - Objects.equals(stats, other.stats); - } - - @Override - protected void innerXContent(XContentBuilder builder, Params params) throws IOException { - super.innerXContent(builder, params); - if (enabled) { - builder.startObject("stats"); - for (Map.Entry entry : stats.entrySet()) { - builder.field(entry.getKey() + "_usage", entry.getValue()); - } - builder.endObject(); - } + return Objects.equals(available, other.available) + && Objects.equals(enabled, other.enabled) + && Objects.equals(response, other.response); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java index edd3025ec2fa9..f0822e17d04cf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java @@ -23,7 +23,9 @@ import java.io.IOException; import java.util.List; +import java.util.Locale; import java.util.Objects; +import java.util.stream.Collectors; public class AnalyticsStatsAction extends ActionType { public static final AnalyticsStatsAction INSTANCE = new AnalyticsStatsAction(); @@ -89,7 +91,7 @@ public NodeRequest(Request request) { } } - public static class Response extends BaseNodesResponse implements Writeable { + public static class Response extends BaseNodesResponse implements Writeable, ToXContentObject { public Response(StreamInput in) throws IOException { super(in); } @@ -107,6 +109,25 @@ protected List readNodesFrom(StreamInput in) throws IOException { protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { out.writeList(nodes); } + + public EnumCounters getStats() { + List> countersPerNode = getNodes() + .stream() + .map(AnalyticsStatsAction.NodeResponse::getStats) + .collect(Collectors.toList()); + return EnumCounters.merge(Item.class, countersPerNode); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + EnumCounters stats = getStats(); + builder.startObject("stats"); + for (Item item : Item.values()) { + builder.field(item.name().toLowerCase(Locale.ROOT) + "_usage", stats.get(item)); + } + builder.endObject(); + return builder; + } } public static class NodeResponse extends BaseNodeResponse { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichFeatureSetUsage.java new file mode 100644 index 0000000000000..9d7aaa6641b1a --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichFeatureSetUsage.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.enrich; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; + +import java.io.IOException; + +public class EnrichFeatureSetUsage extends XPackFeatureSet.Usage { + + public EnrichFeatureSetUsage(boolean available, boolean enabled) { + super(XPackField.ENRICH, available, enabled); + } + + public EnrichFeatureSetUsage(StreamInput input) throws IOException { + super(input); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java new file mode 100644 index 0000000000000..55e49dcbaef3c --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.flattened; + +import org.elasticsearch.Version; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; + +import java.io.IOException; +import java.util.Objects; + +// For backward compatibility with 7.x +public class FlattenedFeatureSetUsage extends XPackFeatureSet.Usage { + private final int fieldCount; + + public FlattenedFeatureSetUsage(StreamInput input) throws IOException { + super(input); + this.fieldCount = input.getVersion().onOrAfter(Version.V_7_6_0) ? input.readInt() : 0; + } + + public FlattenedFeatureSetUsage(boolean available, boolean enabled, int fieldCount) { + super(XPackField.FLATTENED, available, enabled); + this.fieldCount = fieldCount; + } + + int fieldCount() { + return fieldCount; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_7_6_0)) { + out.writeInt(fieldCount); + } + } + + @Override + protected void innerXContent(XContentBuilder builder, Params params) throws IOException { + super.innerXContent(builder, params); + builder.field("field_count", fieldCount); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FlattenedFeatureSetUsage that = (FlattenedFeatureSetUsage) o; + return available == that.available && enabled == that.enabled && fieldCount == that.fieldCount; + } + + @Override + public int hashCode() { + return Objects.hash(available, enabled, fieldCount); + } +} diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java index be804788c0038..721bc57039380 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java @@ -37,6 +37,7 @@ import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; import org.elasticsearch.xpack.core.enrich.action.DeleteEnrichPolicyAction; import org.elasticsearch.xpack.core.enrich.action.EnrichStatsAction; import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyAction; @@ -46,6 +47,7 @@ import org.elasticsearch.xpack.enrich.action.EnrichCoordinatorStatsAction; import org.elasticsearch.xpack.enrich.action.EnrichInfoTransportAction; import org.elasticsearch.xpack.enrich.action.EnrichShardMultiSearchAction; +import org.elasticsearch.xpack.enrich.action.EnrichUsageTransportAction; import org.elasticsearch.xpack.enrich.action.TransportDeleteEnrichPolicyAction; import org.elasticsearch.xpack.enrich.action.TransportEnrichStatsAction; import org.elasticsearch.xpack.enrich.action.TransportExecuteEnrichPolicyAction; @@ -151,6 +153,7 @@ protected XPackLicenseState getLicenseState() { return List.of( new ActionHandler<>(XPackInfoFeatureAction.ENRICH, EnrichInfoTransportAction.class), + new ActionHandler<>(XPackUsageFeatureAction.ENRICH, EnrichUsageTransportAction.class), new ActionHandler<>(GetEnrichPolicyAction.INSTANCE, TransportGetEnrichPolicyAction.class), new ActionHandler<>(DeleteEnrichPolicyAction.INSTANCE, TransportDeleteEnrichPolicyAction.class), new ActionHandler<>(PutEnrichPolicyAction.INSTANCE, TransportPutEnrichPolicyAction.class), diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichUsageTransportAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichUsageTransportAction.java new file mode 100644 index 0000000000000..2b91e4ba7446d --- /dev/null +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichUsageTransportAction.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.enrich.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.protocol.xpack.XPackUsageRequest; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; +import org.elasticsearch.xpack.core.enrich.EnrichFeatureSetUsage; + +public class EnrichUsageTransportAction extends XPackUsageFeatureTransportAction { + private final XPackLicenseState licenseState; + private final boolean enabled; + + @Inject + public EnrichUsageTransportAction( + Settings settings, + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + XPackLicenseState licenseState + ) { + super( + XPackUsageFeatureAction.ENRICH.name(), + transportService, + clusterService, + threadPool, + actionFilters, + indexNameExpressionResolver + ); + this.licenseState = licenseState; + this.enabled = XPackSettings.ENRICH_ENABLED_SETTING.get(settings); + } + + @Override + protected void masterOperation( + Task task, + XPackUsageRequest request, + ClusterState state, + ActionListener listener + ) { + boolean available = licenseState.isEnrichAllowed(); + listener.onResponse(new XPackUsageFeatureResponse(new EnrichFeatureSetUsage(available, enabled))); + } +} diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/100_analytics_usage.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/100_analytics_usage.yml new file mode 100644 index 0000000000000..32190c6a599c9 --- /dev/null +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/100_analytics_usage.yml @@ -0,0 +1,60 @@ +--- +setup: + - skip: + version: " - 7.4.99" + reason: "analytics wasn't available before 7.5" + features: allowed_warnings + - do: + bulk: + index: analytics_usage + refresh: true + body: + - '{"index": {}}' + - '{"timestamp": "2017-01-01T05:00:00Z", "s": 1, "v1": 3.1415, "v2": 2.1415, "str": "a"}' + - '{"index": {}}' + - '{"timestamp": "2017-01-01T05:00:00Z", "s": 2, "v1": 1.0, "v2": 2.0, "str": "a"}' + - '{"index": {}}' + - '{"timestamp": "2017-01-01T05:00:00Z", "s": 3, "v1": 2.71828, "v2": 3.71828, "str": "b"}' + +--- +"Basic test for usage stats on analytics indices": + - skip: + version: " - 7.7.99" + reason: "stats are not working in earlier versions" + - do: + allowed_warnings: + - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.' + xpack.usage: {} + - match: { analytics.available: true } + - match: { analytics.enabled: true } + - is_true: analytics.stats + - set: {analytics.stats.cumulative_cardinality_usage: cumulative_cardinality_usage} + + # use cumulative_cardinality agg + - do: + search: + index: "analytics_usage" + body: + size: 0 + aggs: + histo: + date_histogram: + field: "timestamp" + calendar_interval: "day" + aggs: + distinct_s: + cardinality: + field: "s" + total_users: + cumulative_cardinality: + buckets_path: "distinct_s" + + - length: { aggregations.histo.buckets: 1 } + + - do: + allowed_warnings: + - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.' + xpack.usage: {} + - match: { analytics.available: true } + - match: { analytics.enabled: true } + - gt: { analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage } From 7fda54413484d2dcb23bf4adc60fdb4c06bea028 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 14 Apr 2020 17:57:31 -0400 Subject: [PATCH 2/6] Fix AnalyticsInfoTransportActionTests --- .../action/AnalyticsInfoTransportActionTests.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java index 467465e7bb23e..3d6440e862cd4 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -42,6 +43,7 @@ public class AnalyticsInfoTransportActionTests extends ESTestCase { private Task task; private ClusterService clusterService; private ClusterName clusterName; + private ClusterState clusterState; @Before public void init() { @@ -52,7 +54,9 @@ public void init() { DiscoveryNode discoveryNode = mock(DiscoveryNode.class); when(discoveryNode.getId()).thenReturn(randomAlphaOfLength(10)); when(clusterService.localNode()).thenReturn(discoveryNode); - clusterName = mock(ClusterName.class); + clusterName = new ClusterName(randomAlphaOfLength(10)); + clusterState = mock(ClusterState.class); + when(clusterState.getClusterName()).thenReturn(clusterName); } public void testAvailable() throws Exception { @@ -65,7 +69,7 @@ public void testAvailable() throws Exception { AnalyticsUsageTransportAction usageAction = new AnalyticsUsageTransportAction(mock(TransportService.class), clusterService, null, mock(ActionFilters.class), null, licenseState, client); PlainActionFuture future = new PlainActionFuture<>(); - usageAction.masterOperation(task, null, null, future); + usageAction.masterOperation(task, null, clusterState, future); XPackFeatureSet.Usage usage = future.get().getUsage(); assertThat(usage.available(), is(available)); @@ -90,7 +94,7 @@ public void testEnabled() throws Exception { AnalyticsUsageTransportAction usageAction = new AnalyticsUsageTransportAction(mock(TransportService.class), clusterService, null, mock(ActionFilters.class), null, licenseState, client); PlainActionFuture future = new PlainActionFuture<>(); - usageAction.masterOperation(task, null, null, future); + usageAction.masterOperation(task, null, clusterState, future); XPackFeatureSet.Usage usage = future.get().getUsage(); assertTrue(usage.enabled()); From 131350103fc9b8efc3776a4254d82dead8fa9e10 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 15 Apr 2020 07:10:47 -0400 Subject: [PATCH 3/6] Remove special handling for flattened --- .../xpack/core/action/XPackUsageResponse.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java index 2d5f3fceda9dd..29be3866e39ba 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java @@ -23,14 +23,7 @@ public XPackUsageResponse(StreamInput in) throws IOException { int size = in.readVInt(); usages = new ArrayList<>(size); for (int i = 0; i < size; i++) { - String name = in.readString(); - if (name.equals("flattened")) { - // work-around for removed flattened - new XPackFeatureSet.Usage(in) { }; - in.readInt(); - } else { - usages.add(in.readNamedWriteable(XPackFeatureSet.Usage.class, name)); - } + usages.add(in.readNamedWriteable(XPackFeatureSet.Usage.class)); } } From b874e66db499fbf25adfeb67968bf3f5cf73ddda Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 16 Apr 2020 08:18:22 -0400 Subject: [PATCH 4/6] Address review comments --- .../main/java/org/elasticsearch/xpack/core/XPackField.java | 6 +++++- .../xpack/core/flattened/FlattenedFeatureSetUsage.java | 5 ++++- .../test/mixed_cluster/100_analytics_usage.yml | 4 ++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index 6a5986f5497d5..01b0c3d36d50a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -41,7 +41,11 @@ public final class XPackField { public static final String CCR = "ccr"; /** Name constant for the transform feature. */ public static final String TRANSFORM = "transform"; - /** Name constant for flattened fields. -- For Backward Compatibility with 7.x */ + /** Name constant for flattened fields. + * + * @deprecated used for Backward Compatibility with 7.x only + */ + @Deprecated public static final String FLATTENED = "flattened"; /** Name constant for the vectors feature. */ public static final String VECTORS = "vectors"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java index 55e49dcbaef3c..6715f342795e1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java @@ -16,7 +16,10 @@ import java.io.IOException; import java.util.Objects; -// For backward compatibility with 7.x +/** + * @deprecated used for backward compatibility with 7.x only + */ +@Deprecated public class FlattenedFeatureSetUsage extends XPackFeatureSet.Usage { private final int fieldCount; diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/100_analytics_usage.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/100_analytics_usage.yml index 32190c6a599c9..f0aafeccf5ca4 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/100_analytics_usage.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/100_analytics_usage.yml @@ -21,6 +21,10 @@ setup: - skip: version: " - 7.7.99" reason: "stats are not working in earlier versions" + - do: + xpack.info: {} + - match: { features.analytics.available: true } + - match: { features.analytics.enabled: true } - do: allowed_warnings: - '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.' From 560f49694eafa051e2cd15e51bd0fd490dde68d7 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 16 Apr 2020 08:36:17 -0400 Subject: [PATCH 5/6] Add a test to remind us to remove FlattenedFeatureSetUsageTests in 9.x --- .../FlattenedFeatureSetUsageTests.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsageTests.java diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsageTests.java new file mode 100644 index 0000000000000..fade04a32ba92 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsageTests.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.flattened; + +import org.elasticsearch.Version; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; + +public class FlattenedFeatureSetUsageTests extends AbstractWireSerializingTestCase { + + public void testFlattenedIsRemovedInNextMajor() { + assertTrue("FlattenedFeatureSetUsageTests is no longer needed and should be removed in 9.x", + Version.CURRENT.major <= Version.V_8_0_0.major); + } + + @Override + protected FlattenedFeatureSetUsage createTestInstance() { + return new FlattenedFeatureSetUsage(randomBoolean(), randomBoolean(), randomIntBetween(0, 1000)); + } + + @Override + protected FlattenedFeatureSetUsage mutateInstance(FlattenedFeatureSetUsage instance) throws IOException { + + boolean available = instance.available(); + boolean enabled = instance.enabled(); + int fieldCount = instance.fieldCount(); + + switch (between(0, 2)) { + case 0: + available = !available; + break; + case 1: + enabled = !enabled; + break; + case 2: + fieldCount = randomValueOtherThan(instance.fieldCount(), () -> randomIntBetween(0, 1000)); + break; + } + + return new FlattenedFeatureSetUsage(available, enabled, fieldCount); + } + + @Override + protected Writeable.Reader instanceReader() { + return FlattenedFeatureSetUsage::new; + } + +} From d36e8f9c26ffe07cbe3d6c3abd0ccc80d4240ab7 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 16 Apr 2020 08:58:00 -0400 Subject: [PATCH 6/6] Add version to Enrich and Flattened Feature Set Usages --- .../xpack/core/enrich/EnrichFeatureSetUsage.java | 6 ++++++ .../xpack/core/flattened/FlattenedFeatureSetUsage.java | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichFeatureSetUsage.java index 9d7aaa6641b1a..c38901e057aaf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichFeatureSetUsage.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.enrich; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.core.XPackFeatureSet; import org.elasticsearch.xpack.core.XPackField; @@ -21,4 +22,9 @@ public EnrichFeatureSetUsage(boolean available, boolean enabled) { public EnrichFeatureSetUsage(StreamInput input) throws IOException { super(input); } + + @Override + public Version getMinimalSupportedVersion() { + return Version.V_7_5_0; + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java index 6715f342795e1..9c000faa2cb0b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java @@ -37,6 +37,11 @@ int fieldCount() { return fieldCount; } + @Override + public Version getMinimalSupportedVersion() { + return Version.V_7_3_0; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out);