Support "on-by-config" features in usage tracking

test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java

@@ -54,9 +54,13 @@ private XContentTestUtils() {
 
     public static Map<String, Object> convertToMap(ToXContent part) throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder();
-        builder.startObject();
-        part.toXContent(builder, EMPTY_PARAMS);
-        builder.endObject();
+        if (part.isFragment()) {
+            builder.startObject();
+            part.toXContent(builder, EMPTY_PARAMS);
+            builder.endObject();
+        } else {
+            part.toXContent(builder, EMPTY_PARAMS);
+        }
         return XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2();
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetFeatureUsageResponse.java

@@ -6,6 +6,7 @@
 
 package org.elasticsearch.license;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -19,31 +20,42 @@
 import java.time.ZonedDateTime;
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 public class GetFeatureUsageResponse extends ActionResponse implements ToXContentObject {
 
     public static class FeatureUsageInfo implements Writeable {
         public final String name;
         public final ZonedDateTime lastUsedTime;
         public final String licenseLevel;
+        public final Set<String> identifiers;
 
-        public FeatureUsageInfo(String name, ZonedDateTime lastUsedTime, String licenseLevel) {
+        public FeatureUsageInfo(String name, ZonedDateTime lastUsedTime, String licenseLevel, Set<String> identifiers) {
             this.name = name;
             this.lastUsedTime = lastUsedTime;
             this.licenseLevel = licenseLevel;
+            this.identifiers = identifiers;
         }
 
         public FeatureUsageInfo(StreamInput in) throws IOException {
             this.name = in.readString();
             this.lastUsedTime = ZonedDateTime.ofInstant(Instant.ofEpochSecond(in.readLong()), ZoneOffset.UTC);
             this.licenseLevel = in.readString();
+            if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+                this.identifiers = in.readSet(StreamInput::readString);
+            } else {
+                this.identifiers = Set.of();
+            }
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(name);
             out.writeLong(lastUsedTime.toEpochSecond());
             out.writeString(licenseLevel);
+            if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+                out.writeStringCollection(this.identifiers);
+            }
         }
     }
 
@@ -75,6 +87,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field("name", feature.name);
             builder.field("last_used", feature.lastUsedTime.toString());
             builder.field("license_level", feature.licenseLevel);
+            builder.field("ids", feature.identifiers);
             builder.endObject();
         }
         builder.endArray();

x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetFeatureUsageAction.java

@@ -18,9 +18,10 @@
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
+import java.util.Set;
 
 public class TransportGetFeatureUsageAction extends HandledTransportAction<GetFeatureUsageRequest, GetFeatureUsageResponse> {
 
@@ -39,14 +40,13 @@ public TransportGetFeatureUsageAction(TransportService transportService, ActionF
 
     @Override
     protected void doExecute(Task task, GetFeatureUsageRequest request, ActionListener<GetFeatureUsageResponse> listener) {
-        Map<XPackLicenseState.Feature, Long> featureUsage = licenseState.getLastUsed();
+        Collection<XPackLicenseState.FeatureUsage> featureUsage = licenseState.getFeatureUsage();
         List<GetFeatureUsageResponse.FeatureUsageInfo> usageInfos = new ArrayList<>();
-        for (var entry : featureUsage.entrySet()) {
-            XPackLicenseState.Feature feature = entry.getKey();
-            String name = feature.name().toLowerCase(Locale.ROOT);
-            ZonedDateTime lastUsedTime = Instant.ofEpochMilli(entry.getValue()).atZone(ZoneOffset.UTC);
-            String licenseLevel = feature.minimumOperationMode.name().toLowerCase(Locale.ROOT);
-            usageInfos.add(new GetFeatureUsageResponse.FeatureUsageInfo(name, lastUsedTime, licenseLevel));
+        for (var usage : featureUsage) {
+            String name = usage.feature.name().toLowerCase(Locale.ROOT);
+            ZonedDateTime lastUsedTime = Instant.ofEpochMilli(usage.lastUsed).atZone(ZoneOffset.UTC);
+            String licenseLevel = usage.feature.minimumOperationMode.name().toLowerCase(Locale.ROOT);
+            usageInfos.add(new GetFeatureUsageResponse.FeatureUsageInfo(name, lastUsedTime, licenseLevel, Set.copyOf(usage.identifiers)));
         }
         listener.onResponse(new GetFeatureUsageResponse(usageInfos));
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java

@@ -15,8 +15,11 @@
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.monitoring.MonitoringField;
 
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -109,7 +112,7 @@ public enum Feature {
         }
     }
 
-    // temporarily non tracked feeatures which need rework in how they are checked
+    // temporarily non tracked features which need rework in how they are checked
     // so they are not tracked as always used
     private static final Set<Feature> NON_TRACKED_FEATURES = Set.of(
         Feature.SECURITY_IP_FILTERING,
@@ -411,6 +414,7 @@ private static class Status {
     private final boolean isSecurityExplicitlyEnabled;
     private final Map<Feature, LongAccumulator> lastUsed;
     private final LongSupplier epochMillisProvider;
+    private final Map<Feature, Set<String>> alwaysOnFeatures;
 
     // Since Status is the only field that can be updated, we do not need to synchronize access to
     // XPackLicenseState. However, if status is read multiple times in a method, it can change in between
@@ -427,22 +431,29 @@ public XPackLicenseState(Settings settings, LongSupplier epochMillisProvider) {
         // care to actually keep track of
         Map<Feature, LongAccumulator> lastUsed = new EnumMap<>(Feature.class);
         for (Feature feature : Feature.values()) {
-            if (feature.minimumOperationMode.compareTo(OperationMode.BASIC) > 0 && NON_TRACKED_FEATURES.contains(feature) == false) {
+            if (isTrackableFeature(feature)) {
                 lastUsed.put(feature, new LongAccumulator(Long::max, 0));
             }
         }
         this.lastUsed = lastUsed;
         this.epochMillisProvider = epochMillisProvider;
+        this.alwaysOnFeatures = new EnumMap<>(Feature.class);
+    }
+
+    protected static boolean isTrackableFeature(Feature feature) {
+        return feature.minimumOperationMode.compareTo(OperationMode.BASIC) > 0 && NON_TRACKED_FEATURES.contains(feature) == false;
     }
 
     private XPackLicenseState(List<LicenseStateListener> listeners, boolean isSecurityEnabled, boolean isSecurityExplicitlyEnabled,
-                              Status status, Map<Feature, LongAccumulator> lastUsed, LongSupplier epochMillisProvider) {
+                              Status status, Map<Feature, LongAccumulator> lastUsed, LongSupplier epochMillisProvider,
+                              Map<Feature, Set<String>> alwaysOnFeatures) {
         this.listeners = listeners;
         this.isSecurityEnabled = isSecurityEnabled;
         this.isSecurityExplicitlyEnabled = isSecurityExplicitlyEnabled;
         this.status = status;
         this.lastUsed = lastUsed;
         this.epochMillisProvider = epochMillisProvider;
+        this.alwaysOnFeatures = alwaysOnFeatures;
     }
 
     private static boolean isSecurityExplicitlyEnabled(Settings settings) {
@@ -514,6 +525,20 @@ public boolean checkFeature(Feature feature) {
         return allowed;
     }
 
+    /**
+     * Marks a licensed feature as <em>on by configuration</em>.
+     * By default {@link #getFeatureUsage()} method returns the last time {@link #checkFeature(Feature)} was called.
+     * One this method is called, the specified feature will be marked as "always in use",and {@link #getFeatureUsage()} method will return
+     * the current time ("now"), instead of the last time the feature was checked.
+     */
+    public synchronized boolean setFeatureActive(Feature feature, String... identifiers) {
+        boolean allowed = checkFeature(feature);
+        if (allowed && isTrackableFeature(feature)) {
+            alwaysOnFeatures.computeIfAbsent(feature, ignore -> new HashSet<>()).addAll(Arrays.asList(identifiers));
+        }
+        return allowed;
+    }
+
     /**
      * Checks whether the given feature is allowed by the current license.
      * <p>
@@ -528,10 +553,18 @@ public boolean isAllowed(Feature feature) {
      *
      * Note that if a feature has not been used, it will not appear in the map.
      */
-    public Map<Feature, Long> getLastUsed() {
-        return lastUsed.entrySet().stream()
+    public Collection<FeatureUsage> getFeatureUsage() {
+        final Map<Feature, FeatureUsage> used = lastUsed.entrySet().stream()
             .filter(e -> e.getValue().get() != 0) // feature was never used
-            .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().get()));
+            .map(e -> new FeatureUsage(e.getKey(), e.getValue().get(), Set.of()))
+            .collect(Collectors.toMap(u -> u.feature, Function.identity()));
+        if (alwaysOnFeatures.isEmpty() == false) {
+            final long now = epochMillisProvider.getAsLong();
+            alwaysOnFeatures.entrySet().stream()
+                .map(e -> new FeatureUsage(e.getKey(), now, e.getValue()))
+                .forEach(u -> used.put(u.feature, u));
+        }
+        return used.values();
     }
 
     public static boolean isMachineLearningAllowedForOperationMode(final OperationMode operationMode) {
@@ -606,8 +639,14 @@ public static boolean isAllowedByOperationMode(
      * is needed for multiple interactions with the license state.
      */
     public XPackLicenseState copyCurrentLicenseState() {
-        return executeAgainstStatus(status ->
-            new XPackLicenseState(listeners, isSecurityEnabled, isSecurityExplicitlyEnabled, status, lastUsed, epochMillisProvider));
+        return executeAgainstStatus(status -> new XPackLicenseState(
+            listeners,
+            isSecurityEnabled,
+            isSecurityExplicitlyEnabled,
+            status,
+            lastUsed,
+            epochMillisProvider,
+            alwaysOnFeatures));
     }
 
     /**
@@ -629,12 +668,49 @@ public boolean isAllowedByLicense(OperationMode minimumMode, boolean needActive)
 
     /**
      * A convenient method to test whether a feature is by license status.
-     * @see #isAllowedByLicense(OperationMode, boolean)
      *
-     * @param minimumMode  The minimum license to meet or exceed
+     * @param minimumMode The minimum license to meet or exceed
+     * @see #isAllowedByLicense(OperationMode, boolean)
      */
     public boolean isAllowedByLicense(OperationMode minimumMode) {
         return isAllowedByLicense(minimumMode, true);
     }
 
+    public static class FeatureUsage {
+        public final Feature feature;
+        public final long lastUsed;
+        public final Set<String> identifiers;
+
+        public FeatureUsage(Feature feature, long lastUsed, Set<String> identifiers) {
+            this.feature = feature;
+            this.lastUsed = lastUsed;
+            this.identifiers = Objects.requireNonNull(identifiers);
+        }
+
+        @Override
+        public String toString() {
+            final StringBuilder sb = new StringBuilder("FeatureUsage{");
+            sb.append("feature=").append(feature);
+            sb.append(", lastUsed=").append(lastUsed);
+            sb.append(", identifiers=").append(identifiers);
+            sb.append('}');
+            return sb.toString();
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            FeatureUsage that = (FeatureUsage) o;
+            return this.lastUsed == that.lastUsed &&
+                this.feature == that.feature &&
+                this.identifiers.equals(that.identifiers);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(feature, lastUsed, identifiers);
+        }
+    }
+
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/license/GetFeatureUsageResponseTests.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.license;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.XContentTestUtils;
+
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.temporal.ChronoUnit;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.iterableWithSize;
+
+public class GetFeatureUsageResponseTests extends ESTestCase {
+
+    public void testSerializationCurrentVersion() throws Exception {
+        final GetFeatureUsageResponse response = randomResponse();
+        final Version version = Version.CURRENT;
+        final GetFeatureUsageResponse read = copyResponse(response, version);
+        assertThat(read.getFeatures(), hasSize(response.getFeatures().size()));
+        for (int i = 0; i < response.getFeatures().size(); i++) {
+            final GetFeatureUsageResponse.FeatureUsageInfo origFeature = response.getFeatures().get(i);
+            final GetFeatureUsageResponse.FeatureUsageInfo readFeature = read.getFeatures().get(i);
+            assertThat(readFeature.name, equalTo(origFeature.name));
+            assertThat(readFeature.licenseLevel, equalTo(origFeature.licenseLevel));
+            assertThat(readFeature.lastUsedTime, equalTo(origFeature.lastUsedTime));
+            assertThat(readFeature.identifiers, equalTo(origFeature.identifiers));
+        }
+    }
+
+    public void testSerializationVersion7_10() throws Exception {
+        final GetFeatureUsageResponse response = randomResponse();
+        final Version version = Version.V_7_10_0;
+        final GetFeatureUsageResponse read = copyResponse(response, version);
+        assertThat(read.getFeatures(), hasSize(response.getFeatures().size()));
+        for (int i = 0; i < response.getFeatures().size(); i++) {
+            final GetFeatureUsageResponse.FeatureUsageInfo origFeature = response.getFeatures().get(i);
+            final GetFeatureUsageResponse.FeatureUsageInfo readFeature = read.getFeatures().get(i);
+            assertThat(readFeature.name, equalTo(origFeature.name));
+            assertThat(readFeature.licenseLevel, equalTo(origFeature.licenseLevel));
+            assertThat(readFeature.lastUsedTime, equalTo(origFeature.lastUsedTime));
+            assertThat(readFeature.identifiers, emptyIterable());
+        }
+    }
+
+    public void testToXContent() throws Exception {
+        final GetFeatureUsageResponse response = randomResponse();
+        var map = XContentTestUtils.convertToMap(response);
+
+        assertThat(map.get("features"), instanceOf(List.class));
+        final List<?> features = (List<?>) map.get("features");
+        assertThat(features, iterableWithSize(response.getFeatures().size()));
+
+        for (int i = 0; i < features.size(); i++) {
+            assertThat(features.get(i), instanceOf(Map.class));
+            var read = (Map<String, ?>) features.get(i);
+            final GetFeatureUsageResponse.FeatureUsageInfo orig = response.getFeatures().get(i);
+            assertThat(read.get("name"), equalTo(orig.name));
+            assertThat(read.get("license_level"), equalTo(orig.licenseLevel));
+            assertThat(read.get("last_used"), equalTo(orig.lastUsedTime.toString()));
+            assertThat(Set.copyOf((Collection<?>) read.get("ids")), equalTo(orig.identifiers));
+        }
+    }
+
+    protected GetFeatureUsageResponse randomResponse() {
+        final String featureName = randomAlphaOfLengthBetween(8, 24);
+        final Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
+        final ZonedDateTime lastUsed = ZonedDateTime.ofInstant(now.minusSeconds(randomIntBetween(1, 10_000)), ZoneOffset.UTC);
+        final String licenseLevel = randomAlphaOfLengthBetween(4, 12);
+        final Set<String> identifiers = Set.copyOf(randomList(8, () -> randomAlphaOfLengthBetween(2, 8)));
+        final List<GetFeatureUsageResponse.FeatureUsageInfo> usage = randomList(1, 5,
+            () -> new GetFeatureUsageResponse.FeatureUsageInfo(featureName, lastUsed, licenseLevel, identifiers));
+        return new GetFeatureUsageResponse(usage);
+    }
+
+    protected GetFeatureUsageResponse copyResponse(GetFeatureUsageResponse response, Version version) throws java.io.IOException {
+        return copyWriteable(response, new NamedWriteableRegistry(List.of()), GetFeatureUsageResponse::new, version);
+    }
+}