Adding Beats x-pack plugin + index templates

x-pack/build.gradle

@@ -50,6 +50,7 @@ subprojects {
   ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-deprecation:${version}": xpackModule('deprecation')]
   ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-graph:${version}": xpackModule('graph')]
   ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-logstash:${version}": xpackModule('logstash')]
+  ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-beats:${version}": xpackModule('beats')]
   ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-ml:${version}": xpackModule('ml')]
   ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-monitoring:${version}": xpackModule('monitoring')]
   ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-security:${version}": xpackModule('security')]

x-pack/plugin/beats/build.gradle

@@ -0,0 +1,22 @@
+evaluationDependsOn(xpackModule('core'))
+
+apply plugin: 'elasticsearch.esplugin'
+esplugin {
+    name 'x-pack-beats'
+    description 'Elasticsearch Expanded Pack Plugin - Beats'
+    classname 'org.elasticsearch.xpack.beats.Beats'
+    extendedPlugins = ['x-pack-core']
+}
+archivesBaseName = 'x-pack-beats'
+
+dependencies {
+    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+
+}
+
+run {
+    plugin xpackModule('core')
+}
+
+integTest.enabled = false

x-pack/plugin/beats/src/main/java/org/elasticsearch/xpack/beats/Beats.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.beats;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
+import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.template.TemplateUtils;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.function.UnaryOperator;
+import java.util.regex.Pattern;
+
+/**
+ * This class activates/deactivates the beats modules depending if we're running a node client or transport client
+ */
+public class Beats extends Plugin implements ActionPlugin {
+
+    private static final String BEATS_AGENTS_TEMPLATE_NAME = "beats-agents-template";
+    private static final String BEATS_ADMIN_TEMPLATE_NAME = "beats-admin-template";
+    private static final String TEMPLATE_VERSION_PATTERN =
+            Pattern.quote("${beats.template.version}");
+
+    private final boolean enabled;
+    private final boolean transportClientMode;
+
+    public Beats(Settings settings) {
+        this.enabled = XPackSettings.BEATS_ENABLED.get(settings);
+        this.transportClientMode = XPackPlugin.transportClientMode(settings);
+    }
+
+    boolean isEnabled() {
+      return enabled;
+    }
+
+    boolean isTransportClient() {
+      return transportClientMode;
+    }
+
+    public Collection<Module> createGuiceModules() {
+        List<Module> modules = new ArrayList<>();
+        modules.add(b -> {
+            XPackPlugin.bindFeatureSet(b, BeatsFeatureSet.class);
+        });
+        return modules;
+    }
+
+    public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
+        return templates -> {
+            TemplateUtils.loadTemplateIntoMap("/" + BEATS_AGENTS_TEMPLATE_NAME + ".json", templates, BEATS_AGENTS_TEMPLATE_NAME,
+                    Version.CURRENT.toString(), TEMPLATE_VERSION_PATTERN, Loggers.getLogger(Beats.class));
+            TemplateUtils.loadTemplateIntoMap("/" + BEATS_ADMIN_TEMPLATE_NAME + ".json", templates, BEATS_ADMIN_TEMPLATE_NAME,
+                Version.CURRENT.toString(), TEMPLATE_VERSION_PATTERN, Loggers.getLogger(Beats.class));
+            return templates;
+        };
+    }
+}

x-pack/plugin/beats/src/main/java/org/elasticsearch/xpack/beats/BeatsFeatureSet.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.beats;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage;
+
+
+import java.io.IOException;
+import java.util.Map;
+
+public class BeatsFeatureSet implements XPackFeatureSet {
+
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public BeatsFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState) {
+        this.enabled = XPackSettings.BEATS_ENABLED.get(settings);
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    public String name() {
+        return XPackField.BEATS;
+    }
+
+    @Override
+    public String description() {
+        return "Beats management component for X-Pack";
+    }
+
+    @Override
+    public boolean available() {
+        return licenseState != null && licenseState.isBeatsAllowed();
+    }
+
+    @Override
+    public boolean enabled() {
+        return enabled;
+    }
+
+    @Override
+    public Map<String, Object> nativeCodeInfo() {
+      return null;
+    }
+
+    @Override
+    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
+        listener.onResponse(new BeatsFeatureSetUsage(available(), enabled()));
+    }
+
+}

x-pack/plugin/beats/src/main/plugin-metadata/plugin-security.policy

@@ -0,0 +1,25 @@
+grant {
+  // needed for multiple server implementations used in tests
+  permission java.net.SocketPermission "*", "accept,connect";
+};
+
+grant codeBase "${codebase.netty-common}" {
+   // for reading the system-wide configuration for the backlog of established sockets
+   permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
+};
+
+grant codeBase "${codebase.netty-transport}" {
+   // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854
+   // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely!
+   permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
+};
+
+grant codeBase "${codebase.elasticsearch-rest-client}" {
+  // rest client uses system properties which gets the default proxy
+  permission java.net.NetPermission "getProxySelector";
+};
+
+grant codeBase "${codebase.httpasyncclient}" {
+  // rest client uses system properties which gets the default proxy
+  permission java.net.NetPermission "getProxySelector";
+};

x-pack/plugin/beats/src/test/java/org/elasticsearch/xpack/beats/BeatsFeatureSetTests.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.beats;
+
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage;
+
+import static org.mockito.Mockito.mock;
+import static org.hamcrest.core.Is.is;
+import static org.mockito.Mockito.when;
+
+public class BeatsFeatureSetTests extends ESTestCase {
+
+    public void testEnabledSetting() throws Exception {
+        boolean enabled = randomBoolean();
+        Settings settings = Settings.builder()
+            .put("path.home", createTempDir())
+            .put("xpack.beats.enabled", enabled)
+            .build();
+        BeatsFeatureSet featureSet = new BeatsFeatureSet(settings, null);
+        assertThat(featureSet.enabled(), is(enabled));
+
+        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
+        featureSet.usage(future);
+        XPackFeatureSet.Usage usage = future.get();
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        usage.writeTo(out);
+        XPackFeatureSet.Usage serializedUsage = new BeatsFeatureSetUsage(out.bytes().streamInput());
+        assertThat(serializedUsage.enabled(), is(enabled));
+    }
+
+    public void testEnabledDefault() throws Exception {
+        Settings settings = Settings.builder().put("path.home", createTempDir()).build();
+        BeatsFeatureSet featureSet = new BeatsFeatureSet(settings, null);
+        assertThat(featureSet.enabled(), is(true));
+    }
+
+    public void testAvailable() throws Exception {
+        final XPackLicenseState licenseState = mock(XPackLicenseState.class);
+        BeatsFeatureSet featureSet = new BeatsFeatureSet(Settings.EMPTY, licenseState);
+        boolean available = randomBoolean();
+        when(licenseState.isBeatsAllowed()).thenReturn(available);
+        assertThat(featureSet.available(), is(available));
+
+        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
+        featureSet.usage(future);
+        XPackFeatureSet.Usage usage = future.get();
+        assertThat(usage.available(), is(available));
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        usage.writeTo(out);
+        XPackFeatureSet.Usage serializedUsage = new BeatsFeatureSetUsage(out.bytes().streamInput());
+        assertThat(serializedUsage.available(), is(available));
+    }
+}

x-pack/plugin/build.gradle

@@ -15,7 +15,7 @@ archivesBaseName = 'x-pack'
 es_meta_plugin {
   name = 'x-pack'
   description = 'Elasticsearch Expanded Pack Plugin'
-  plugins = ['core', 'deprecation', 'graph', 'logstash',
+  plugins = ['core', 'deprecation', 'graph', 'logstash', 'beats',
              'ml', 'monitoring', 'security', 'upgrade', 'watcher', 'sql', 'rollup']
 }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java

@@ -52,6 +52,9 @@ public class XPackLicenseState {
         messages.put(XPackField.LOGSTASH, new String[] {
             "Logstash will continue to poll centrally-managed pipelines"
         });
+        messages.put(XPackField.BEATS, new String[] {
+            "Beats will continue to poll centrally-managed configuration"
+        });
         messages.put(XPackField.DEPRECATION, new String[] {
             "Deprecation APIs are disabled"
         });
@@ -81,6 +84,7 @@ public class XPackLicenseState {
         messages.put(XPackField.GRAPH, XPackLicenseState::graphAcknowledgementMessages);
         messages.put(XPackField.MACHINE_LEARNING, XPackLicenseState::machineLearningAcknowledgementMessages);
         messages.put(XPackField.LOGSTASH, XPackLicenseState::logstashAcknowledgementMessages);
+        messages.put(XPackField.BEATS, XPackLicenseState::beatsAcknowledgementMessages);
         messages.put(XPackField.SQL, XPackLicenseState::sqlAcknowledgementMessages);
         ACKNOWLEDGMENT_MESSAGES = Collections.unmodifiableMap(messages);
     }
@@ -217,6 +221,21 @@ private static String[] logstashAcknowledgementMessages(OperationMode currentMod
         return Strings.EMPTY_ARRAY;
     }
 
+    private static String[] beatsAcknowledgementMessages(OperationMode currentMode, OperationMode newMode) {
+        switch (newMode) {
+            case BASIC:
+                switch (currentMode) {
+                    case TRIAL:
+                    case STANDARD:
+                    case GOLD:
+                    case PLATINUM:
+                        return new String[] { "Logstash will no longer poll for centrally-managed configuration" };
+                }
+                break;
+        }
+        return Strings.EMPTY_ARRAY;
+    }
+
     private static String[] sqlAcknowledgementMessages(OperationMode currentMode, OperationMode newMode) {
         switch (newMode) {
             case BASIC:
@@ -512,6 +531,28 @@ public boolean isLogstashAllowed() {
         }
     }
 
+    /**
+     * Beats is allowed as long as there is an active license of type TRIAL, STANDARD, GOLD or PLATINUM
+     * @return {@code true} as long as there is a valid license
+     */
+    public boolean isBeatsAllowed() {
+        Status localStatus = status;
+
+        if (localStatus.active == false) {
+            return false;
+        }
+
+        switch (localStatus.mode) {
+            case TRIAL:
+            case GOLD:
+            case PLATINUM:
+            case STANDARD:
+                return true;
+            default:
+                return false;
+        }
+    }
+
     /**
      * Deprecation APIs are always allowed as long as there is an active license
      * @return {@code true} as long as there is a valid license

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -39,6 +39,7 @@
 import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage;
 import org.elasticsearch.xpack.core.graph.action.GraphExploreAction;
 import org.elasticsearch.xpack.core.logstash.LogstashFeatureSetUsage;
+import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage;
 import org.elasticsearch.xpack.core.ml.MachineLearningFeatureSetUsage;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
@@ -319,6 +320,8 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.GRAPH, GraphFeatureSetUsage::new),
                 // logstash
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.LOGSTASH, LogstashFeatureSetUsage::new),
+                // beats
+                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.BEATS, BeatsFeatureSetUsage::new),
                 // ML - Custom metadata
                 new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new),
                 new NamedWriteableRegistry.Entry(NamedDiff.class, "ml", MlMetadata.MlMetadataDiff::new),

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java

@@ -19,6 +19,8 @@ public final class XPackField {
     public static final String MACHINE_LEARNING = "ml";
     /** Name constant for the Logstash feature. */
     public static final String LOGSTASH = "logstash";
+    /** Name constant for the Beats feature. */
+    public static final String BEATS = "beats";
     /** Name constant for the Deprecation API feature. */
     public static final String DEPRECATION = "deprecation";
     /** Name constant for the upgrade feature. */

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java

@@ -60,6 +60,10 @@ public class XPackSettings {
     public static final Setting<Boolean> LOGSTASH_ENABLED = Setting.boolSetting("xpack.logstash.enabled", true,
             Setting.Property.NodeScope);
 
+    /** Setting for enabling or disabling Beats extensions. Defaults to true. */
+    public static final Setting<Boolean> BEATS_ENABLED = Setting.boolSetting("xpack.beats.enabled", true,
+        Setting.Property.NodeScope);
+
     /** Setting for enabling or disabling TLS. Defaults to false. */
     public static final Setting<Boolean> TRANSPORT_SSL_ENABLED = Setting.boolSetting("xpack.security.transport.ssl.enabled", false,
             Property.NodeScope);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/beats/BeatsFeatureSetUsage.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.beats;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
+
+import java.io.IOException;
+
+public class BeatsFeatureSetUsage extends XPackFeatureSet.Usage {
+
+    public BeatsFeatureSetUsage(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    public BeatsFeatureSetUsage(boolean available, boolean enabled) {
+        super(XPackField.BEATS, available, enabled);
+    }
+
+}