From 1ea4623764d12297a880fbf72715c468db395926 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Thu, 27 Apr 2023 10:53:57 +0200 Subject: [PATCH 01/36] Bootstrap profiling indices at startup With this commit we bootstrap all indices and related structures that are required for Elastic Universal Profiling. --- .../elasticsearch/test/ESIntegTestCase.java | 11 +- .../xpack/core/ClientHelper.java | 1 + x-pack/plugin/profiler/build.gradle | 9 + .../xpack/profiler/ProfilingTestCase.java | 49 ++- .../xpack/profiler/ProfilingIndexManager.java | 342 ++++++++++++++++++ .../xpack/profiler/ProfilingPlugin.java | 22 +- .../component-template/profiling-events.json} | 41 +-- .../profiling-executables.json} | 19 +- .../component-template/profiling-ilm.json | 11 + .../profiling-stackframes.json} | 22 +- .../profiling-stacktraces.json} | 28 +- .../xpack/profiler/ilm-policy/profiling.json | 37 ++ .../index-template/profiling-events.json | 17 + .../index-template/profiling-executables.json | 13 + .../index-template/profiling-stackframes.json | 14 + .../index-template/profiling-stacktraces.json | 14 + .../profiler/index/.profiling-ilm-lock.json | 18 + .../index/profiling-returnpads-private.json | 49 +++ .../index/profiling-sq-executables.json | 34 ++ .../index/profiling-sq-leafframes.json | 34 ++ .../index/profiling-symbols-private.json | 93 +++++ .../profiler/index/profiling-symbols.json | 93 +++++ .../security/authz/AuthorizationUtils.java | 2 + 23 files changed, 894 insertions(+), 79 deletions(-) create mode 100644 x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java rename x-pack/plugin/profiler/src/{internalClusterTest/resources/events.json => main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json} (69%) rename x-pack/plugin/profiler/src/{internalClusterTest/resources/executables.json => main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json} (65%) create mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json rename x-pack/plugin/profiler/src/{internalClusterTest/resources/stackframes.json => main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json} (69%) rename x-pack/plugin/profiler/src/{internalClusterTest/resources/stacktraces.json => main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json} (56%) create mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json create mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-events.json create mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json create mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stackframes.json create mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stacktraces.json create mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/.profiling-ilm-lock.json create mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-returnpads-private.json create mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-sq-executables.json create mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-sq-leafframes.json create mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-symbols-private.json create mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-symbols.json diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index c1f22875b55c3..d49989041ee0d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -362,7 +362,9 @@ protected final void beforeInternal() throws Exception { final Scope currentClusterScope = getCurrentClusterScope(); Callable setup = () -> { cluster().beforeTest(random()); - cluster().wipe(excludeTemplates()); + if (wipeCluster()) { + cluster().wipe(excludeTemplates()); + } randomIndexTemplate(); return null; }; @@ -606,6 +608,13 @@ protected Set excludeTemplates() { return Collections.emptySet(); } + /** + * @return true if the cluster should be wiped before each test. By default, the cluster is wiped. + */ + protected boolean wipeCluster() { + return true; + } + protected void beforeIndexDeletion() throws Exception { cluster().beforeIndexDeletion(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java index 37f5b32da1646..4cf8c72c087ca 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java @@ -186,6 +186,7 @@ private static String maybeRewriteSingleAuthenticationHeaderForVersion( public static final String TRANSFORM_ORIGIN = "transform"; public static final String ASYNC_SEARCH_ORIGIN = "async_search"; public static final String IDP_ORIGIN = "idp"; + public static final String PROFILING_ORIGIN = "profiling"; public static final String STACK_ORIGIN = "stack"; public static final String SEARCHABLE_SNAPSHOTS_ORIGIN = "searchable_snapshots"; public static final String LOGSTASH_MANAGEMENT_ORIGIN = "logstash_management"; diff --git a/x-pack/plugin/profiler/build.gradle b/x-pack/plugin/profiler/build.gradle index 80624ec9a1815..dc680693aca29 100644 --- a/x-pack/plugin/profiler/build.gradle +++ b/x-pack/plugin/profiler/build.gradle @@ -12,4 +12,13 @@ esplugin { name 'x-pack-profiling' description 'The profiler plugin adds support for retrieving data from Universal Profiler.' classname 'org.elasticsearch.xpack.profiler.ProfilingPlugin' + extendedPlugins = ['x-pack-core'] +} + +dependencies { + compileOnly project(path: xpackModule('core')) + testImplementation(testArtifact(project(xpackModule('core')))) + testImplementation project(path: xpackModule('mapper-unsigned-long')) + testImplementation project(path: xpackModule('ilm')) + testImplementation project(':modules:data-streams') } diff --git a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java index e79a79d4bbc28..cda7fe1aaf461 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java +++ b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java @@ -8,13 +8,19 @@ package org.elasticsearch.xpack.profiler; import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.datastreams.DataStreamsPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.transport.netty4.Netty4Plugin; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.elasticsearch.xpack.core.ilm.LifecycleSettings; +import org.elasticsearch.xpack.ilm.IndexLifecycle; +import org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin; import org.junit.Before; import java.io.IOException; @@ -26,7 +32,14 @@ public abstract class ProfilingTestCase extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return List.of(ProfilingPlugin.class, getTestTransportPlugin()); + return List.of( + LocalStateCompositeXPackPlugin.class, + DataStreamsPlugin.class, + ProfilingPlugin.class, + IndexLifecycle.class, + UnsignedLongMapperPlugin.class, + getTestTransportPlugin() + ); } @Override @@ -35,6 +48,9 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .put(super.nodeSettings(nodeOrdinal, otherSettings)) .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME) .put(NetworkModule.HTTP_TYPE_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME) + // .put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial") + // Disable ILM history index so that the tests don't have to clean it up + .put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING.getKey(), false) .build(); } @@ -48,6 +64,12 @@ protected boolean ignoreExternalCluster() { return true; } + @Override + protected boolean wipeCluster() { + // we're managing our own indices and the cluster should not be wiped. + return false; + } + private byte[] read(String resource) throws IOException { return GetProfilingAction.class.getClassLoader().getResourceAsStream(resource).readAllBytes(); } @@ -57,7 +79,7 @@ private void createIndex(String name, String bodyFileName) throws Exception { } private void indexDoc(String index, String id, Map source) { - IndexResponse indexResponse = client().prepareIndex(index).setId(id).setSource(source).get(); + IndexResponse indexResponse = client().prepareIndex(index).setId(id).setSource(source).setCreate(true).get(); assertEquals(RestStatus.CREATED, indexResponse.status()); } @@ -71,17 +93,24 @@ private void indexDoc(String index, String id, Map source) { */ protected abstract boolean useOnlyAllEvents(); + protected void waitForIndices() throws Exception { + // block until the templates are installed + assertBusy(() -> { + ClusterState state = client().admin().cluster().prepareState().get().getState(); + // simple check whether at least one of the indices is present + assertTrue( + "Timed out waiting for the indices to be created", + state.metadata().indices().containsKey("profiling-stackframes-000002") + ); + }); + } + @Before public void setupData() throws Exception { Collection eventsIndices = useOnlyAllEvents() ? List.of(EventsIndex.FULL_INDEX.getName()) : EventsIndex.indexNames(); - - for (String idx : eventsIndices) { - createIndex(idx, "events.json"); - } - createIndex("profiling-stackframes", "stackframes.json"); - createIndex("profiling-stacktraces", "stacktraces.json"); - createIndex("profiling-executables", "executables.json"); - ensureGreen(); + waitForIndices(); + // indices have replicas + ensureYellow(); // ensure that we have this in every index, so we find an event for (String idx : eventsIndices) { diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java new file mode 100644 index 0000000000000..c55b03e236d7b --- /dev/null +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java @@ -0,0 +1,342 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.profiler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.gateway.GatewayService; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +import static java.util.Collections.singletonMap; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; + +/** + * Creates all index-templates, ILM policies, aliases and indices that are required for using Elastic Universal Profiling. + */ +public class ProfilingIndexManager implements ClusterStateListener { + private static final Logger logger = LogManager.getLogger(ProfilingIndexManager.class); + + private static final String[] COMPONENT_TEMPLATES = new String[] { + "profiling-events", + "profiling-executables", + "profiling-ilm", + "profiling-stackframes", + "profiling-stacktraces" }; + + private static final String[] INDEX_TEMPLATES = new String[] { + "profiling-events", + "profiling-executables", + "profiling-stackframes", + "profiling-stacktraces" }; + + private static final String[] ILM_POLICIES = new String[] { "profiling" }; + private static final String[] REGULAR_INDICES = new String[] { + ".profiling-ilm-lock", + "profiling-returnpads-private", + "profiling-sq-executables", + "profiling-sq-leafframes", + "profiling-symbols", + "profiling-symbols-private" }; + + private static final String[] KV_INDICES = new String[] { "profiling-executables", "profiling-stackframes", "profiling-stacktraces" }; + + private final Client client; + private final NamedXContentRegistry xContentRegistry; + private final ClusterService clusterService; + private final AtomicBoolean creating = new AtomicBoolean(false); + + public ProfilingIndexManager(Client client, NamedXContentRegistry xContentRegistry, ClusterService clusterService) { + this.client = client; + this.xContentRegistry = xContentRegistry; + this.clusterService = clusterService; + } + + public void init() { + clusterService.addListener(this); + } + + private void done() { + clusterService.removeListener(this); + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + // wait for the cluster state to be recovered + if (event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { + return; + } + + // If this node is not a master node, exit. + if (event.state().nodes().isLocalNodeElectedMaster() == false) { + return; + } + + if (event.state().nodes().getMaxNodeVersion().after(event.state().nodes().getSmallestNonClientNodeVersion())) { + logger.debug("Skipping up-to-date check as cluster has mixed versions"); + return; + } + + if (creating.compareAndSet(false, true)) { + // tag::noformat + createIlmPolicies( + (p) -> createComponentTemplates( + (ct) -> createIndexTemplates( + (it) -> createIndices( + (i) -> createDataStreams( + ds -> done() + ) + ) + ) + ) + ); + // end::noformat + } + } + + private void onResourceFailure(String resource, Exception ex) { + done(); + } + + private void createComponentTemplates(Consumer onSuccess) { + ResourceListener listener = new CompositeListener( + "component template", + COMPONENT_TEMPLATES.length, + onSuccess, + this::onResourceFailure + ); + for (String componentTemplate : COMPONENT_TEMPLATES) { + createComponentTemplate(componentTemplate, listener); + } + } + + private void createComponentTemplate(String name, ResourceListener listener) { + logger.debug("adding component template [" + name + "]"); + PutComponentTemplateAction.Request request = new PutComponentTemplateAction.Request(name); + try { + XContentParser parser = parser("component-template", name); + request.componentTemplate(ComponentTemplate.parse(parser)); + request.cause("bootstrap"); + } catch (Exception ex) { + listener.onFailure(name, ex); + return; + } + executeAsyncWithOrigin( + client.threadPool().getThreadContext(), + ClientHelper.PROFILING_ORIGIN, + request, + ActionListener.wrap(r -> listener.onSuccess(name), e -> listener.onFailure(name, e)), + (r, l) -> client.execute(PutComponentTemplateAction.INSTANCE, r, l) + ); + } + + private void createIndexTemplates(Consumer onSuccess) { + ResourceListener listener = new CompositeListener("index template", INDEX_TEMPLATES.length, onSuccess, this::onResourceFailure); + for (String indexTemplate : INDEX_TEMPLATES) { + createIndexTemplate(indexTemplate, listener); + } + } + + private void createIndexTemplate(String name, ResourceListener listener) { + logger.debug("adding index template [" + name + "]"); + PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request(name); + try { + XContentParser parser = parser("index-template", name); + request.indexTemplate(ComposableIndexTemplate.parse(parser)); + } catch (Exception ex) { + listener.onFailure(name, ex); + return; + } + executeAsyncWithOrigin( + client.threadPool().getThreadContext(), + ClientHelper.PROFILING_ORIGIN, + request, + ActionListener.wrap(r -> listener.onSuccess(name), e -> listener.onFailure(name, e)), + (r, l) -> client.execute(PutComposableIndexTemplateAction.INSTANCE, r, l) + ); + } + + private void createIlmPolicies(Consumer onSuccess) { + ResourceListener listener = new CompositeListener("ilm policy", ILM_POLICIES.length, onSuccess, this::onResourceFailure); + for (String ilmPolicy : ILM_POLICIES) { + createIlmPolicy(ilmPolicy, listener); + } + } + + private void createIlmPolicy(String name, ResourceListener listener) { + logger.debug("adding ilm policy [" + name + "]"); + PutLifecycleAction.Request request; + try { + XContentParser parser = parser("ilm-policy", name); + request = PutLifecycleAction.Request.parseRequest(name, parser); + } catch (Exception ex) { + listener.onFailure(name, ex); + return; + } + executeAsyncWithOrigin( + client.threadPool().getThreadContext(), + ClientHelper.PROFILING_ORIGIN, + request, + ActionListener.wrap(r -> listener.onSuccess(name), e -> listener.onFailure(name, e)), + (r, l) -> client.execute(PutLifecycleAction.INSTANCE, r, l) + ); + } + + private void createIndices(Consumer onSuccess) { + ResourceListener listener = new CompositeListener( + "index", + REGULAR_INDICES.length + 2 * KV_INDICES.length, + onSuccess, + this::onResourceFailure + ); + for (String index : REGULAR_INDICES) { + createIndex(index, true, listener); + } + for (String index : KV_INDICES) { + createKvIndex(index + "-000001", index, listener); + createKvIndex(index + "-000002", index + "-next", listener); + } + } + + private void createIndex(String name, boolean includeBody, ResourceListener listener) { + logger.debug("adding index [" + name + "]"); + CreateIndexRequest request = new CreateIndexRequest(name); + if (includeBody) { + try { + XContentParser parser = parser("index", name); + Map sourceAsMap = new HashMap<>(parser.map()); + if (sourceAsMap.containsKey("mappings")) { + sourceAsMap.put("mappings", singletonMap(MapperService.SINGLE_MAPPING_NAME, sourceAsMap.get("mappings"))); + } + request.source(sourceAsMap, LoggingDeprecationHandler.INSTANCE); + } catch (Exception ex) { + listener.onFailure(name, ex); + return; + } + } + executeAsyncWithOrigin( + client.threadPool().getThreadContext(), + ClientHelper.PROFILING_ORIGIN, + request, + ActionListener.wrap(r -> listener.onSuccess(name), e -> listener.onFailure(name, e)), + (r, l) -> client.admin().indices().create(r, l) + ); + } + + private void createKvIndex(String name, String alias, ResourceListener listener) { + logger.debug("adding key-value index [" + name + "]"); + CreateIndexRequest request = new CreateIndexRequest(name); + try { + Map sourceAsMap = Map.of("aliases", Map.of(alias, Map.of("is_write_index", true))); + request.source(sourceAsMap, LoggingDeprecationHandler.INSTANCE); + } catch (Exception ex) { + listener.onFailure(name, ex); + return; + } + executeAsyncWithOrigin( + client.threadPool().getThreadContext(), + ClientHelper.PROFILING_ORIGIN, + request, + ActionListener.wrap(r -> listener.onSuccess(name), e -> listener.onFailure(name, e)), + (r, l) -> client.admin().indices().create(r, l) + ); + } + + private void createDataStreams(Consumer onSuccess) { + ResourceListener listener = new CompositeListener( + "data stream", + EventsIndex.indexNames().size(), + onSuccess, + this::onResourceFailure + ); + // create these indices based on index templates + for (String dataStream : EventsIndex.indexNames()) { + CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(dataStream); + executeAsyncWithOrigin( + client.threadPool().getThreadContext(), + ClientHelper.PROFILING_ORIGIN, + request, + ActionListener.wrap(r -> listener.onSuccess(dataStream), e -> listener.onFailure(dataStream, e)), + (r, l) -> client.execute(CreateDataStreamAction.INSTANCE, r, l) + ); + + } + } + + private XContentParser parser(String resourcePath, String name) throws IOException { + return XContentFactory.xContent(XContentType.JSON) + .createParser( + XContentParserConfiguration.EMPTY.withRegistry(xContentRegistry), + getClass().getClassLoader() + .getResourceAsStream("org/elasticsearch/xpack/profiler/" + resourcePath + "/" + name + ".json") + .readAllBytes() + ); + } + + private interface ResourceListener { + void onSuccess(String resourceName); + + void onFailure(String resourceName, Exception ex); + } + + private static class CompositeListener implements ResourceListener { + private final String resourceType; + private final AtomicInteger remaining; + private final Consumer onSuccess; + private final BiConsumer onFailure; + + CompositeListener(String resourceType, int compositeLength, Consumer onSuccess, BiConsumer onFailure) { + this.resourceType = resourceType; + this.remaining = new AtomicInteger(compositeLength); + this.onSuccess = onSuccess; + this.onFailure = onFailure; + } + + @Override + public void onSuccess(String resourceName) { + logger.debug("added " + resourceType + " [" + resourceName + "] successfully."); + if (remaining.decrementAndGet() == 0) { + onSuccess.accept(resourceType); + } + } + + @Override + public void onFailure(String resourceName, Exception ex) { + logger.warn("adding " + resourceType + " [" + resourceName + "] failed.", ex); + onFailure.accept(resourceType, ex); + } + } +} diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java index 7c26cf252de32..2adfa4ffb439f 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java @@ -77,21 +77,13 @@ public Collection createComponents( AllocationService allocationService ) { logger.info("Profiling is {}", enabled ? "enabled" : "disabled"); - return super.createComponents( - client, - clusterService, - threadPool, - resourceWatcherService, - scriptService, - xContentRegistry, - environment, - nodeEnvironment, - namedWriteableRegistry, - indexNameExpressionResolver, - repositoriesServiceSupplier, - tracer, - allocationService - ); + if (enabled) { + ProfilingIndexManager indexManager = new ProfilingIndexManager(client, xContentRegistry, clusterService); + indexManager.init(); + return Collections.singletonList(indexManager); + } else { + return Collections.emptyList(); + } } @Override diff --git a/x-pack/plugin/profiler/src/internalClusterTest/resources/events.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json similarity index 69% rename from x-pack/plugin/profiler/src/internalClusterTest/resources/events.json rename to x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json index 855c754191db5..4e6df7b3ae498 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/resources/events.json +++ b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json @@ -1,24 +1,24 @@ { - "settings": { - "index": { - "number_of_shards": "4", - "max_result_window": 150000, - "refresh_interval": "10s", - "sort": { - "field": [ - "service.name", - "@timestamp", - "orchestrator.resource.name", - "container.name", - "process.thread.name", - "host.id" - ] - } + "template": { + "settings": { + "index": { + "number_of_shards": 4, + "max_result_window": 150000, + "refresh_interval": "10s", + "sort": { + "field": [ + "service.name", + "@timestamp", + "orchestrator.resource.name", + "container.name", + "process.thread.name", + "host.id" + ] + } + }, + "codec": "best_compression" }, - "codec": "best_compression" - }, - "mappings": { - "_doc": { + "mappings": { "_source": { "enabled": false }, @@ -60,9 +60,6 @@ "host.ip": { "type": "ip" }, - "host.ipstring": { - "type": "keyword" - }, "host.name": { "type": "keyword" }, diff --git a/x-pack/plugin/profiler/src/internalClusterTest/resources/executables.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json similarity index 65% rename from x-pack/plugin/profiler/src/internalClusterTest/resources/executables.json rename to x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json index f5db60318bcf2..8e062e232e01d 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/resources/executables.json +++ b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json @@ -1,11 +1,11 @@ { - "settings": { - "index": { - "refresh_interval": "10s" - } - }, - "mappings": { - "_doc": { + "template": { + "settings": { + "index": { + "refresh_interval": "10s" + } + }, + "mappings": { "_source": { "mode": "synthetic" }, @@ -25,6 +25,11 @@ "@timestamp": { "type": "date", "format": "epoch_second" + }, + "Symbolization.lastprocessed": { + "type": "date", + "format": "epoch_second", + "index": false } } } diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json new file mode 100644 index 0000000000000..1a24ecd0557da --- /dev/null +++ b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json @@ -0,0 +1,11 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "profiling" + } + } + } + } +} diff --git a/x-pack/plugin/profiler/src/internalClusterTest/resources/stackframes.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json similarity index 69% rename from x-pack/plugin/profiler/src/internalClusterTest/resources/stackframes.json rename to x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json index ced7b081a3d36..bde41dec9ebb3 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/resources/stackframes.json +++ b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json @@ -1,12 +1,12 @@ { - "settings": { - "index": { - "number_of_shards": "16", - "refresh_interval": "10s" - } - }, - "mappings": { - "_doc": { + "template": { + "settings": { + "index": { + "number_of_shards": 16, + "refresh_interval": "10s" + } + }, + "mappings": { "_source": { "enabled": true }, @@ -34,6 +34,12 @@ "index": false, "doc_values": false, "store": false + }, + "Stackframe.function.offset": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false } } } diff --git a/x-pack/plugin/profiler/src/internalClusterTest/resources/stacktraces.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json similarity index 56% rename from x-pack/plugin/profiler/src/internalClusterTest/resources/stacktraces.json rename to x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json index 0b72dccdfe5c5..472b056a8fcbe 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/resources/stacktraces.json +++ b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json @@ -1,17 +1,17 @@ { - "settings": { - "index": { - "number_of_shards": "16", - "refresh_interval": "10s", - "sort": { - "field": [ - "Stacktrace.frame.ids" - ] + "template": { + "settings": { + "index": { + "number_of_shards": 16, + "refresh_interval": "10s", + "sort": { + "field": [ + "Stacktrace.frame.ids" + ] + } } - } - }, - "mappings": { - "_doc": { + }, + "mappings": { "_source": { "mode": "synthetic" }, @@ -27,10 +27,6 @@ "Stacktrace.frame.types": { "type": "keyword", "index": false - }, - "@timestamp": { - "type": "date", - "format": "epoch_second" } } } diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json new file mode 100644 index 0000000000000..2be9a20d899a3 --- /dev/null +++ b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json @@ -0,0 +1,37 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_primary_shard_size": "50gb", + "max_age": "7d" + }, + "set_priority": { + "priority": 100 + } + } + }, + "warm": { + "min_age": "30d", + "actions": { + "set_priority": { + "priority": 50 + }, + "shrink": { + "number_of_shards": 2 + } + } + }, + "delete": { + "min_age": "60d", + "actions": { + "delete": { + "delete_searchable_snapshot": true + } + } + } + } + } +} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-events.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-events.json new file mode 100644 index 0000000000000..56413746680b6 --- /dev/null +++ b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-events.json @@ -0,0 +1,17 @@ +{ + "index_patterns": [ + "profiling-events*" + ], + "data_stream": { + "hidden": false + }, + "composed_of": [ + "profiling-events", + "profiling-ilm" + ], + "priority": 100, + "version": 1, + "_meta": { + "description": "Index template for profiling-events" + } +} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json new file mode 100644 index 0000000000000..0531f3393f454 --- /dev/null +++ b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json @@ -0,0 +1,13 @@ +{ + "index_patterns": [ + "profiling-executables*" + ], + "composed_of": [ + "profiling-executables" + ], + "priority": 100, + "version": 1, + "_meta": { + "description": "Index template for profiling-executables" + } +} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stackframes.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stackframes.json new file mode 100644 index 0000000000000..c4182ba6daa52 --- /dev/null +++ b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stackframes.json @@ -0,0 +1,14 @@ +{ + "index_patterns": [ + "profiling-stackframes*" + ], + "composed_of": [ + "profiling-stackframes" + ], + "priority": 100, + "version": 1, + "_meta": { + "description": "Index template for profiling-stackframes", + "version": 1 + } +} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stacktraces.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stacktraces.json new file mode 100644 index 0000000000000..f3bf23ec65c4f --- /dev/null +++ b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stacktraces.json @@ -0,0 +1,14 @@ +{ + "index_patterns": [ + "profiling-stacktraces*" + ], + "composed_of": [ + "profiling-stacktraces" + ], + "priority": 100, + "version": 1, + "_meta": { + "description": "Index template for profiling-stacktraces", + "version": 1 + } +} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/.profiling-ilm-lock.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/.profiling-ilm-lock.json new file mode 100644 index 0000000000000..9b17b7e97376a --- /dev/null +++ b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/.profiling-ilm-lock.json @@ -0,0 +1,18 @@ +{ + "settings": { + "index": { + "hidden": true + } + }, + "mappings": { + "properties": { + "@timestamp": { + "type": "date", + "format": "epoch_second" + }, + "phase": { + "type": "keyword" + } + } + } +} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-returnpads-private.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-returnpads-private.json new file mode 100644 index 0000000000000..533b196ff9ce1 --- /dev/null +++ b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-returnpads-private.json @@ -0,0 +1,49 @@ +{ + "settings": { + "index": { + "refresh_interval": "10s" + } + }, + "mappings": { + "_source": { + "enabled": true + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbfile.created": { + "type": "date", + "doc_values": false, + "index": true, + "store": false + }, + "Symbfile.file.id": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbfile.part": { + "type": "short", + "index": false, + "doc_values": false, + "store": false + }, + "Symbfile.parts": { + "type": "short", + "index": false, + "doc_values": false, + "store": false + }, + "Symbfile.data": { + "type": "binary", + "doc_values": false, + "store": false + } + } + } +} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-sq-executables.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-sq-executables.json new file mode 100644 index 0000000000000..8e5e9935c6ecb --- /dev/null +++ b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-sq-executables.json @@ -0,0 +1,34 @@ +{ + "settings": { + "index": { + "refresh_interval": "10s" + } + }, + "mappings": { + "_source": { + "mode": "synthetic" + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true + }, + "Executable.file.id": { + "type": "keyword", + "index": false + }, + "Time.created": { + "type": "date", + "index": true + }, + "Symbolization.time.next": { + "type": "date", + "index": true + }, + "Symbolization.retries": { + "type": "short", + "index": true + } + } + } +} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-sq-leafframes.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-sq-leafframes.json new file mode 100644 index 0000000000000..ed778a1309bc5 --- /dev/null +++ b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-sq-leafframes.json @@ -0,0 +1,34 @@ +{ + "settings": { + "index": { + "refresh_interval": "10s" + } + }, + "mappings": { + "_source": { + "mode": "synthetic" + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true + }, + "Stacktrace.frame.id": { + "type": "keyword", + "index": false + }, + "Time.created": { + "type": "date", + "index": true + }, + "Symbolization.time.next": { + "type": "date", + "index": true + }, + "Symbolization.retries": { + "type": "short", + "index": true + } + } + } +} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-symbols-private.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-symbols-private.json new file mode 100644 index 0000000000000..7736a6202a9ad --- /dev/null +++ b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-symbols-private.json @@ -0,0 +1,93 @@ +{ + "settings": { + "index": { + "number_of_shards": "16", + "refresh_interval": "10s" + } + }, + "mappings": { + "_source": { + "enabled": true + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbol.function.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.file.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.call.file.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.call.line": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.function.line": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.depth": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.base": { + "type": "unsigned_long", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.length": { + "type": "unsigned_long", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.offsets": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.lines": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.file.id": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbol.exec.pcrange": { + "type": "ip_range", + "index": true, + "doc_values": false, + "store": false + } + } + } +} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-symbols.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-symbols.json new file mode 100644 index 0000000000000..519894977f0c9 --- /dev/null +++ b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-symbols.json @@ -0,0 +1,93 @@ +{ + "settings": { + "index": { + "number_of_shards": "16", + "refresh_interval": "10s" + } + }, + "mappings": { + "_source": { + "enabled": true + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbol.function.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.file.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.call.file.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.call.line": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.function.line": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.depth": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.base": { + "type": "unsigned_long", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.length": { + "type": "unsigned_long", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.offsets": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.lines": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.file.id": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbol.exec.pcrange": { + "type": "ip_range", + "index": true, + "doc_values": false, + "store": false + } + } + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java index b9f72ea0e0028..8867f6a591df1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java @@ -36,6 +36,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.LOGSTASH_MANAGEMENT_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.MONITORING_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.PROFILING_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.ROLLUP_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.SEARCHABLE_SNAPSHOTS_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; @@ -137,6 +138,7 @@ public static void switchUserBasedOnActionOriginAndExecute( case ENRICH_ORIGIN: case IDP_ORIGIN: case INGEST_ORIGIN: + case PROFILING_ORIGIN: case STACK_ORIGIN: case SEARCHABLE_SNAPSHOTS_ORIGIN: case LOGSTASH_MANAGEMENT_ORIGIN: From a3e92dd920cb438549f2f51995f25937a5163419 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Tue, 2 May 2023 11:09:15 +0200 Subject: [PATCH 02/36] Use IndexTemplateRegistry --- .../elasticsearch/test/ESIntegTestCase.java | 4 +- .../core/template/LifecyclePolicyConfig.java | 2 + .../component-template/profiling-events.json | 3 +- .../profiling-executables.json | 3 +- .../component-template/profiling-ilm.json | 3 +- .../profiling-stackframes.json | 3 +- .../profiling-stacktraces.json | 3 +- .../xpack/profiler/ilm-policy/profiling.json | 39 ++ .../index-template/.profiling-ilm-lock.json | 28 ++ .../index-template/profiling-events.json | 4 +- .../index-template/profiling-executables.json | 4 +- .../profiling-returnpads-private.json | 59 +++ .../profiling-sq-executables.json | 45 +++ .../profiling-sq-leafframes.json | 44 +++ .../index-template/profiling-stackframes.json | 7 +- .../index-template/profiling-stacktraces.json | 7 +- .../index-template/profiling-symbols.json | 103 +++++ .../xpack/profiler/ProfilingTestCase.java | 10 - .../xpack/profiler/ProfilingIndexManager.java | 361 +++++------------- .../ProfilingIndexTemplateRegistry.java | 206 ++++++++++ .../xpack/profiler/ProfilingPlugin.java | 22 +- .../profiler/TransportGetProfilingAction.java | 16 +- .../xpack/profiler/ilm-policy/profiling.json | 37 -- .../profiler/index/.profiling-ilm-lock.json | 18 - .../index/profiling-returnpads-private.json | 49 --- .../index/profiling-sq-executables.json | 34 -- .../index/profiling-sq-leafframes.json | 34 -- .../index/profiling-symbols-private.json | 93 ----- .../profiler/index/profiling-symbols.json | 93 ----- 29 files changed, 676 insertions(+), 658 deletions(-) rename x-pack/plugin/{profiler => core}/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json (96%) rename x-pack/plugin/{profiler => core}/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json (93%) rename x-pack/plugin/{profiler => core}/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json (70%) rename x-pack/plugin/{profiler => core}/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json (94%) rename x-pack/plugin/{profiler => core}/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json (92%) create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/.profiling-ilm-lock.json rename x-pack/plugin/{profiler => core}/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-events.json (82%) rename x-pack/plugin/{profiler => core}/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json (79%) create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json rename x-pack/plugin/{profiler => core}/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stackframes.json (55%) rename x-pack/plugin/{profiler => core}/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stacktraces.json (55%) create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json create mode 100644 x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java delete mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json delete mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/.profiling-ilm-lock.json delete mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-returnpads-private.json delete mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-sq-executables.json delete mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-sq-leafframes.json delete mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-symbols-private.json delete mode 100644 x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-symbols.json diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index d49989041ee0d..65bbfbb0cb520 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -580,7 +580,9 @@ private void afterInternal(boolean afterClass) throws Exception { ensureClusterStateCanBeReadByNodeTool(); ensureClusterInfoServiceRunning(); beforeIndexDeletion(); - cluster().wipe(excludeTemplates()); // wipe after to make sure we fail in the test that didn't ack the delete + if (wipeCluster()) { + cluster().wipe(excludeTemplates()); // wipe after to make sure we fail in the test that didn't ack the delete + } if (afterClass || currentClusterScope == Scope.TEST) { cluster().close(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java index b7d69bc160d2a..99c8df4d75ea3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.core.ilm.LifecyclePolicyUtils; import org.elasticsearch.xpack.core.ilm.LifecycleType; import org.elasticsearch.xpack.core.ilm.RolloverAction; +import org.elasticsearch.xpack.core.ilm.SetPriorityAction; import org.elasticsearch.xpack.core.ilm.ShrinkAction; import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType; @@ -36,6 +37,7 @@ public class LifecyclePolicyConfig { (p) -> TimeseriesLifecycleType.INSTANCE ), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(SetPriorityAction.NAME), SetPriorityAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse) diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json similarity index 96% rename from x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json rename to x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json index 4e6df7b3ae498..7d5d0c061ad02 100644 --- a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json @@ -71,5 +71,6 @@ } } } - } + }, + "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json similarity index 93% rename from x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json rename to x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json index 8e062e232e01d..5f31cd8b186cb 100644 --- a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json @@ -33,5 +33,6 @@ } } } - } + }, + "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json similarity index 70% rename from x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json rename to x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json index 1a24ecd0557da..42fbad496c0e9 100644 --- a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json @@ -7,5 +7,6 @@ } } } - } + }, + "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json similarity index 94% rename from x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json rename to x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json index bde41dec9ebb3..6f309f353ef9a 100644 --- a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json @@ -43,5 +43,6 @@ } } } - } + }, + "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json similarity index 92% rename from x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json rename to x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json index 472b056a8fcbe..f4da624d6c7d8 100644 --- a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json @@ -30,5 +30,6 @@ } } } - } + }, + "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json new file mode 100644 index 0000000000000..bacbf33ed41a6 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json @@ -0,0 +1,39 @@ +{ + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_primary_shard_size": "50gb", + "max_age": "7d" + }, + "set_priority": { + "priority": 100 + } + } + }, + "warm": { + "min_age": "30d", + "actions": { + "set_priority": { + "priority": 50 + }, + "shrink": { + "number_of_shards": 2 + } + } + }, + "delete": { + "min_age": "60d", + "actions": { + "delete": { + "delete_searchable_snapshot": true + } + } + } + }, + "_meta": { + "description": "default ILM policy for Elastic Universal Profiling", + "managed": true + } +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/.profiling-ilm-lock.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/.profiling-ilm-lock.json new file mode 100644 index 0000000000000..69a93342f22c8 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/.profiling-ilm-lock.json @@ -0,0 +1,28 @@ +{ + "index_patterns": [ + ".profiling-ilm-lock*" + ], + "template": { + "settings": { + "index": { + "hidden": true + } + }, + "mappings": { + "properties": { + "@timestamp": { + "type": "date", + "format": "epoch_second" + }, + "phase": { + "type": "keyword" + } + } + } + }, + "priority": 100, + "_meta": { + "description": "Index template for .profiling-ilm-lock" + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-events.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-events.json similarity index 82% rename from x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-events.json rename to x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-events.json index 56413746680b6..2ade70d6a0a81 100644 --- a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-events.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-events.json @@ -10,8 +10,8 @@ "profiling-ilm" ], "priority": 100, - "version": 1, "_meta": { "description": "Index template for profiling-events" - } + }, + "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json similarity index 79% rename from x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json rename to x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json index 0531f3393f454..504d870e3f754 100644 --- a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json @@ -6,8 +6,8 @@ "profiling-executables" ], "priority": 100, - "version": 1, "_meta": { "description": "Index template for profiling-executables" - } + }, + "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json new file mode 100644 index 0000000000000..acfb08109884f --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json @@ -0,0 +1,59 @@ +{ + "index_patterns": [ + "profiling-returnpads-private*" + ], + "template": { + "settings": { + "index": { + "refresh_interval": "10s" + } + }, + "mappings": { + "_source": { + "enabled": true + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbfile.created": { + "type": "date", + "doc_values": false, + "index": true, + "store": false + }, + "Symbfile.file.id": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbfile.part": { + "type": "short", + "index": false, + "doc_values": false, + "store": false + }, + "Symbfile.parts": { + "type": "short", + "index": false, + "doc_values": false, + "store": false + }, + "Symbfile.data": { + "type": "binary", + "doc_values": false, + "store": false + } + } + } + }, + "priority": 100, + "_meta": { + "description": "Index template for profiling-returnpads-private" + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json new file mode 100644 index 0000000000000..d410b704443d1 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json @@ -0,0 +1,45 @@ +{ + "index_patterns": [ + "profiling-sq-executables*" + ], + "template": { + "settings": { + "index": { + "refresh_interval": "10s" + } + }, + "mappings": { + "_source": { + "mode": "synthetic" + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true + }, + "Executable.file.id": { + "type": "keyword", + "index": false + }, + "Time.created": { + "type": "date", + "index": true + }, + "Symbolization.time.next": { + "type": "date", + "index": true + }, + "Symbolization.retries": { + "type": "short", + "index": true + } + } + } + }, + "priority": 100, + "_meta": { + "description": "Index template for profiling-sq-executables" + }, + "version": ${xpack.profiling.template.version} +} + diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json new file mode 100644 index 0000000000000..3e6b32b4fffab --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json @@ -0,0 +1,44 @@ +{ + "index_patterns": [ + "profiling-sq-leafframes*" + ], + "template": { + "settings": { + "index": { + "refresh_interval": "10s" + } + }, + "mappings": { + "_source": { + "mode": "synthetic" + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true + }, + "Stacktrace.frame.id": { + "type": "keyword", + "index": false + }, + "Time.created": { + "type": "date", + "index": true + }, + "Symbolization.time.next": { + "type": "date", + "index": true + }, + "Symbolization.retries": { + "type": "short", + "index": true + } + } + } + }, + "priority": 100, + "_meta": { + "description": "Index template for profiling-sq-leafframes" + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stackframes.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stackframes.json similarity index 55% rename from x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stackframes.json rename to x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stackframes.json index c4182ba6daa52..6eba29855b075 100644 --- a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stackframes.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stackframes.json @@ -6,9 +6,8 @@ "profiling-stackframes" ], "priority": 100, - "version": 1, "_meta": { - "description": "Index template for profiling-stackframes", - "version": 1 - } + "description": "Index template for profiling-stackframes" + }, + "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stacktraces.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stacktraces.json similarity index 55% rename from x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stacktraces.json rename to x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stacktraces.json index f3bf23ec65c4f..2e169e19b72d5 100644 --- a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stacktraces.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stacktraces.json @@ -6,9 +6,8 @@ "profiling-stacktraces" ], "priority": 100, - "version": 1, "_meta": { - "description": "Index template for profiling-stacktraces", - "version": 1 - } + "description": "Index template for profiling-stacktraces" + }, + "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json new file mode 100644 index 0000000000000..859608ea35dc7 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json @@ -0,0 +1,103 @@ +{ + "index_patterns": [ + "profiling-symbols*" + ], + "template": { + "settings": { + "index": { + "number_of_shards": "16", + "refresh_interval": "10s" + } + }, + "mappings": { + "_source": { + "enabled": true + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbol.function.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.file.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.call.file.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.call.line": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.function.line": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.depth": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.base": { + "type": "unsigned_long", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.length": { + "type": "unsigned_long", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.offsets": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.linetable.lines": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + "Symbol.file.id": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbol.exec.pcrange": { + "type": "ip_range", + "index": true, + "doc_values": false, + "store": false + } + } + } + }, + "priority": 100, + "_meta": { + "description": "Index template for profiling-symbols" + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java index cda7fe1aaf461..9b7985a103596 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java +++ b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java @@ -16,14 +16,12 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.transport.netty4.Netty4Plugin; -import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.core.ilm.LifecycleSettings; import org.elasticsearch.xpack.ilm.IndexLifecycle; import org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin; import org.junit.Before; -import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Map; @@ -70,14 +68,6 @@ protected boolean wipeCluster() { return false; } - private byte[] read(String resource) throws IOException { - return GetProfilingAction.class.getClassLoader().getResourceAsStream(resource).readAllBytes(); - } - - private void createIndex(String name, String bodyFileName) throws Exception { - client().admin().indices().prepareCreate(name).setSource(read(bodyFileName), XContentType.JSON).execute().get(); - } - private void indexDoc(String index, String id, Map source) { IndexResponse indexResponse = client().prepareIndex(index).setId(id).setSource(source).setCreate(true).get(); assertEquals(RestStatus.CREATED, indexResponse.status()); diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java index c55b03e236d7b..dd8ae0d70108f 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java @@ -12,84 +12,72 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; -import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; -import org.elasticsearch.action.datastreams.CreateDataStreamAction; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; -import org.elasticsearch.cluster.metadata.ComponentTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.gateway.GatewayService; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentParserConfiguration; -import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ClientHelper; -import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction; -import java.io.IOException; +import java.io.Closeable; +import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiConsumer; -import java.util.function.Consumer; -import static java.util.Collections.singletonMap; +import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; /** - * Creates all index-templates, ILM policies, aliases and indices that are required for using Elastic Universal Profiling. + * Creates all indices that are required for using Elastic Universal Profiling. */ -public class ProfilingIndexManager implements ClusterStateListener { +public class ProfilingIndexManager implements ClusterStateListener, Closeable { private static final Logger logger = LogManager.getLogger(ProfilingIndexManager.class); - - private static final String[] COMPONENT_TEMPLATES = new String[] { - "profiling-events", - "profiling-executables", - "profiling-ilm", - "profiling-stackframes", - "profiling-stacktraces" }; - - private static final String[] INDEX_TEMPLATES = new String[] { - "profiling-events", - "profiling-executables", - "profiling-stackframes", - "profiling-stacktraces" }; - - private static final String[] ILM_POLICIES = new String[] { "profiling" }; - private static final String[] REGULAR_INDICES = new String[] { - ".profiling-ilm-lock", - "profiling-returnpads-private", - "profiling-sq-executables", - "profiling-sq-leafframes", - "profiling-symbols", - "profiling-symbols-private" }; - - private static final String[] KV_INDICES = new String[] { "profiling-executables", "profiling-stackframes", "profiling-stacktraces" }; - + private static final Map INDICES_AND_ALIASES; + + static { + Map indicesAndAliases = new HashMap<>(); + indicesAndAliases.put(".profiling-ilm-lock", null); + indicesAndAliases.put("profiling-returnpads-private", null); + indicesAndAliases.put("profiling-sq-executables", null); + indicesAndAliases.put("profiling-sq-leafframes", null); + indicesAndAliases.put("profiling-symbols", null); + indicesAndAliases.put("profiling-symbols-private", null); + // TODO: Update these to the new K/V strategy after all readers have been adjusted + String[] kvIndices = new String[] { "profiling-executables", "profiling-stackframes", "profiling-stacktraces" }; + for (String idx : kvIndices) { + indicesAndAliases.put(idx + "-000001", idx); + indicesAndAliases.put(idx + "-000002", idx + "-next"); + } + INDICES_AND_ALIASES = Collections.unmodifiableMap(indicesAndAliases); + } + + private final ThreadPool threadPool; private final Client client; - private final NamedXContentRegistry xContentRegistry; private final ClusterService clusterService; - private final AtomicBoolean creating = new AtomicBoolean(false); + private final ConcurrentMap indexCreationInProgress = new ConcurrentHashMap<>(); - public ProfilingIndexManager(Client client, NamedXContentRegistry xContentRegistry, ClusterService clusterService) { + public ProfilingIndexManager(ThreadPool threadPool, Client client, ClusterService clusterService) { + this.threadPool = threadPool; this.client = client; - this.xContentRegistry = xContentRegistry; this.clusterService = clusterService; } - public void init() { + public void initialize() { clusterService.addListener(this); } - private void done() { + @Override + public void close() { clusterService.removeListener(this); } @@ -110,233 +98,78 @@ public void clusterChanged(ClusterChangedEvent event) { return; } - if (creating.compareAndSet(false, true)) { - // tag::noformat - createIlmPolicies( - (p) -> createComponentTemplates( - (ct) -> createIndexTemplates( - (it) -> createIndices( - (i) -> createDataStreams( - ds -> done() - ) - ) - ) - ) - ); - // end::noformat - } - } - - private void onResourceFailure(String resource, Exception ex) { - done(); - } - - private void createComponentTemplates(Consumer onSuccess) { - ResourceListener listener = new CompositeListener( - "component template", - COMPONENT_TEMPLATES.length, - onSuccess, - this::onResourceFailure - ); - for (String componentTemplate : COMPONENT_TEMPLATES) { - createComponentTemplate(componentTemplate, listener); - } - } - - private void createComponentTemplate(String name, ResourceListener listener) { - logger.debug("adding component template [" + name + "]"); - PutComponentTemplateAction.Request request = new PutComponentTemplateAction.Request(name); - try { - XContentParser parser = parser("component-template", name); - request.componentTemplate(ComponentTemplate.parse(parser)); - request.cause("bootstrap"); - } catch (Exception ex) { - listener.onFailure(name, ex); - return; - } - executeAsyncWithOrigin( - client.threadPool().getThreadContext(), - ClientHelper.PROFILING_ORIGIN, - request, - ActionListener.wrap(r -> listener.onSuccess(name), e -> listener.onFailure(name, e)), - (r, l) -> client.execute(PutComponentTemplateAction.INSTANCE, r, l) - ); - } - - private void createIndexTemplates(Consumer onSuccess) { - ResourceListener listener = new CompositeListener("index template", INDEX_TEMPLATES.length, onSuccess, this::onResourceFailure); - for (String indexTemplate : INDEX_TEMPLATES) { - createIndexTemplate(indexTemplate, listener); - } - } - - private void createIndexTemplate(String name, ResourceListener listener) { - logger.debug("adding index template [" + name + "]"); - PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request(name); - try { - XContentParser parser = parser("index-template", name); - request.indexTemplate(ComposableIndexTemplate.parse(parser)); - } catch (Exception ex) { - listener.onFailure(name, ex); + // ensure that index templates are present + if (ProfilingIndexTemplateRegistry.isAllTemplatesCreated(event.state()) == false) { + logger.trace("Skipping index creation; not all templates are present yet"); return; } - executeAsyncWithOrigin( - client.threadPool().getThreadContext(), - ClientHelper.PROFILING_ORIGIN, - request, - ActionListener.wrap(r -> listener.onSuccess(name), e -> listener.onFailure(name, e)), - (r, l) -> client.execute(PutComposableIndexTemplateAction.INSTANCE, r, l) - ); - } - private void createIlmPolicies(Consumer onSuccess) { - ResourceListener listener = new CompositeListener("ilm policy", ILM_POLICIES.length, onSuccess, this::onResourceFailure); - for (String ilmPolicy : ILM_POLICIES) { - createIlmPolicy(ilmPolicy, listener); - } - } - - private void createIlmPolicy(String name, ResourceListener listener) { - logger.debug("adding ilm policy [" + name + "]"); - PutLifecycleAction.Request request; - try { - XContentParser parser = parser("ilm-policy", name); - request = PutLifecycleAction.Request.parseRequest(name, parser); - } catch (Exception ex) { - listener.onFailure(name, ex); - return; - } - executeAsyncWithOrigin( - client.threadPool().getThreadContext(), - ClientHelper.PROFILING_ORIGIN, - request, - ActionListener.wrap(r -> listener.onSuccess(name), e -> listener.onFailure(name, e)), - (r, l) -> client.execute(PutLifecycleAction.INSTANCE, r, l) - ); - } - - private void createIndices(Consumer onSuccess) { - ResourceListener listener = new CompositeListener( - "index", - REGULAR_INDICES.length + 2 * KV_INDICES.length, - onSuccess, - this::onResourceFailure - ); - for (String index : REGULAR_INDICES) { - createIndex(index, true, listener); - } - for (String index : KV_INDICES) { - createKvIndex(index + "-000001", index, listener); - createKvIndex(index + "-000002", index + "-next", listener); - } - } - - private void createIndex(String name, boolean includeBody, ResourceListener listener) { - logger.debug("adding index [" + name + "]"); - CreateIndexRequest request = new CreateIndexRequest(name); - if (includeBody) { - try { - XContentParser parser = parser("index", name); - Map sourceAsMap = new HashMap<>(parser.map()); - if (sourceAsMap.containsKey("mappings")) { - sourceAsMap.put("mappings", singletonMap(MapperService.SINGLE_MAPPING_NAME, sourceAsMap.get("mappings"))); + addIndicesIfMissing(event.state()); + } + + private void addIndicesIfMissing(ClusterState state) { + Optional> maybeMeta = Optional.ofNullable(state.metadata().indices()); + for (Map.Entry idxAlias : INDICES_AND_ALIASES.entrySet()) { + String index = idxAlias.getKey(); + String alias = idxAlias.getValue(); + final AtomicBoolean creationCheck = indexCreationInProgress.computeIfAbsent(index, key -> new AtomicBoolean(false)); + if (creationCheck.compareAndSet(false, true)) { + final boolean indexNeedsToBeCreated = maybeMeta.flatMap(idxMeta -> Optional.ofNullable(idxMeta.get(index))) + .isPresent() == false; + if (indexNeedsToBeCreated) { + logger.debug("adding index [{}], because it doesn't exist", index); + putIndex(index, alias, creationCheck); + } else { + logger.trace("not adding index [{}], because it already exists", index); + creationCheck.set(false); } - request.source(sourceAsMap, LoggingDeprecationHandler.INSTANCE); - } catch (Exception ex) { - listener.onFailure(name, ex); - return; } } - executeAsyncWithOrigin( - client.threadPool().getThreadContext(), - ClientHelper.PROFILING_ORIGIN, - request, - ActionListener.wrap(r -> listener.onSuccess(name), e -> listener.onFailure(name, e)), - (r, l) -> client.admin().indices().create(r, l) - ); } - private void createKvIndex(String name, String alias, ResourceListener listener) { - logger.debug("adding key-value index [" + name + "]"); - CreateIndexRequest request = new CreateIndexRequest(name); - try { - Map sourceAsMap = Map.of("aliases", Map.of(alias, Map.of("is_write_index", true))); - request.source(sourceAsMap, LoggingDeprecationHandler.INSTANCE); - } catch (Exception ex) { - listener.onFailure(name, ex); - return; - } - executeAsyncWithOrigin( - client.threadPool().getThreadContext(), - ClientHelper.PROFILING_ORIGIN, - request, - ActionListener.wrap(r -> listener.onSuccess(name), e -> listener.onFailure(name, e)), - (r, l) -> client.admin().indices().create(r, l) - ); + private void onPutIndexFailure(String index, Exception ex) { + logger.error(() -> format("error adding index [%s] for [%s]", index, ClientHelper.PROFILING_ORIGIN), ex); } - private void createDataStreams(Consumer onSuccess) { - ResourceListener listener = new CompositeListener( - "data stream", - EventsIndex.indexNames().size(), - onSuccess, - this::onResourceFailure - ); - // create these indices based on index templates - for (String dataStream : EventsIndex.indexNames()) { - CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(dataStream); + private void putIndex(final String index, final String alias, final AtomicBoolean creationCheck) { + final Executor executor = threadPool.generic(); + executor.execute(() -> { + CreateIndexRequest request = new CreateIndexRequest(index); + if (alias != null) { + try { + Map sourceAsMap = Map.of("aliases", Map.of(alias, Map.of("is_write_index", true))); + request.source(sourceAsMap, LoggingDeprecationHandler.INSTANCE); + } catch (Exception ex) { + creationCheck.set(false); + onPutIndexFailure(index, ex); + } + } + request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); executeAsyncWithOrigin( client.threadPool().getThreadContext(), ClientHelper.PROFILING_ORIGIN, request, - ActionListener.wrap(r -> listener.onSuccess(dataStream), e -> listener.onFailure(dataStream, e)), - (r, l) -> client.execute(CreateDataStreamAction.INSTANCE, r, l) + new ActionListener() { + @Override + public void onResponse(CreateIndexResponse response) { + creationCheck.set(false); + if (response.isAcknowledged() == false) { + logger.error( + "error adding index [{}] for [{}], request was not acknowledged", + index, + ClientHelper.PROFILING_ORIGIN + ); + } + } + + @Override + public void onFailure(Exception e) { + creationCheck.set(false); + onPutIndexFailure(index, e); + } + }, + (req, listener) -> client.admin().indices().create(req, listener) ); - - } - } - - private XContentParser parser(String resourcePath, String name) throws IOException { - return XContentFactory.xContent(XContentType.JSON) - .createParser( - XContentParserConfiguration.EMPTY.withRegistry(xContentRegistry), - getClass().getClassLoader() - .getResourceAsStream("org/elasticsearch/xpack/profiler/" + resourcePath + "/" + name + ".json") - .readAllBytes() - ); - } - - private interface ResourceListener { - void onSuccess(String resourceName); - - void onFailure(String resourceName, Exception ex); - } - - private static class CompositeListener implements ResourceListener { - private final String resourceType; - private final AtomicInteger remaining; - private final Consumer onSuccess; - private final BiConsumer onFailure; - - CompositeListener(String resourceType, int compositeLength, Consumer onSuccess, BiConsumer onFailure) { - this.resourceType = resourceType; - this.remaining = new AtomicInteger(compositeLength); - this.onSuccess = onSuccess; - this.onFailure = onFailure; - } - - @Override - public void onSuccess(String resourceName) { - logger.debug("added " + resourceType + " [" + resourceName + "] successfully."); - if (remaining.decrementAndGet() == 0) { - onSuccess.accept(resourceType); - } - } - - @Override - public void onFailure(String resourceName, Exception ex) { - logger.warn("adding " + resourceType + " [" + resourceName + "] failed.", ex); - onFailure.accept(resourceType, ex); - } + }); } } diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java new file mode 100644 index 0000000000000..3dee116ec48da --- /dev/null +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.profiler; + +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.template.IndexTemplateConfig; +import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; +import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Creates all index-templates and ILM policies that are required for using Elastic Universal Profiling. + */ +public class ProfilingIndexTemplateRegistry extends IndexTemplateRegistry { + // history (please add a comment why you increased the version here) + // version 1: initial + public static final int INDEX_TEMPLATE_VERSION = 1; + + public static final String PROFILING_TEMPLATE_VERSION_VARIABLE = "xpack.profiling.template.version"; + + public ProfilingIndexTemplateRegistry( + Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry + ) { + super(nodeSettings, clusterService, threadPool, client, xContentRegistry); + } + + public void close() { + clusterService.removeListener(this); + } + + @Override + protected String getOrigin() { + return ClientHelper.PROFILING_ORIGIN; + } + + @Override + protected boolean requiresMasterNode() { + return true; + } + + private static final List LIFECYCLE_POLICIES = List.of( + new LifecyclePolicyConfig("profiling", "/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json").load( + LifecyclePolicyConfig.DEFAULT_X_CONTENT_REGISTRY + ) + ); + + @Override + protected List getPolicyConfigs() { + return LIFECYCLE_POLICIES; + } + + private static final Map COMPONENT_TEMPLATE_CONFIGS; + + static { + final Map componentTemplates = new HashMap<>(); + for (IndexTemplateConfig config : List.of( + new IndexTemplateConfig( + "profiling-events", + "/org/elasticsearch/xpack/profiler/component-template/profiling-events.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-executables", + "/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-ilm", + "/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-stackframes", + "/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-stacktraces", + "/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ) + )) { + try { + componentTemplates.put( + config.getTemplateName(), + ComponentTemplate.parse(JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, config.loadBytes())) + ); + } catch (IOException e) { + throw new AssertionError(e); + } + } + COMPONENT_TEMPLATE_CONFIGS = Collections.unmodifiableMap(componentTemplates); + } + + @Override + protected Map getComponentTemplateConfigs() { + return COMPONENT_TEMPLATE_CONFIGS; + } + + private static final Map COMPOSABLE_INDEX_TEMPLATE_CONFIGS = parseComposableTemplates( + new IndexTemplateConfig( + "profiling-events", + "/org/elasticsearch/xpack/profiler/index-template/profiling-events.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-executables", + "/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-stackframes", + "/org/elasticsearch/xpack/profiler/index-template/profiling-stackframes.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-stacktraces", + "/org/elasticsearch/xpack/profiler/index-template/profiling-stacktraces.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + // templates for regular indices + new IndexTemplateConfig( + ".profiling-ilm-lock", + "/org/elasticsearch/xpack/profiler/index-template/.profiling-ilm-lock.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-returnpads-private", + "/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-sq-executables", + "/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-sq-leafframes", + "/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-symbols", + "/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ) + ); + + @Override + protected Map getComposableTemplateConfigs() { + return COMPOSABLE_INDEX_TEMPLATE_CONFIGS; + } + + public static boolean isAllTemplatesCreated(ClusterState state) { + for (String componentTemplate : COMPONENT_TEMPLATE_CONFIGS.keySet()) { + if (state.metadata().componentTemplates().containsKey(componentTemplate) == false) { + return false; + } + } + for (String composableTemplate : COMPOSABLE_INDEX_TEMPLATE_CONFIGS.keySet()) { + if (state.metadata().templatesV2().containsKey(composableTemplate) == false) { + return false; + } + } + return true; + } +} diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java index 2adfa4ffb439f..b21b74edf79eb 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.client.internal.Client; @@ -53,10 +54,15 @@ public class ProfilingPlugin extends Plugin implements ActionPlugin { Setting.Property.NodeScope ); public static final String PROFILING_THREAD_POOL_NAME = "profiling"; - + private final Settings settings; private final boolean enabled; + private final SetOnce registry = new SetOnce<>(); + + private final SetOnce indexManager = new SetOnce<>(); + public ProfilingPlugin(Settings settings) { + this.settings = settings; this.enabled = PROFILING_ENABLED.get(settings); } @@ -77,10 +83,12 @@ public Collection createComponents( AllocationService allocationService ) { logger.info("Profiling is {}", enabled ? "enabled" : "disabled"); + registry.set(new ProfilingIndexTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry)); + indexManager.set(new ProfilingIndexManager(threadPool, client, clusterService)); if (enabled) { - ProfilingIndexManager indexManager = new ProfilingIndexManager(client, xContentRegistry, clusterService); - indexManager.init(); - return Collections.singletonList(indexManager); + registry.get().initialize(); + indexManager.get().initialize(); + return List.of(registry.get(), indexManager.get()); } else { return Collections.emptyList(); } @@ -131,4 +139,10 @@ public static ExecutorBuilder responseExecutorBuilder() { public List> getActions() { return List.of(new ActionHandler<>(GetProfilingAction.INSTANCE, TransportGetProfilingAction.class)); } + + @Override + public void close() { + registry.get().close(); + indexManager.get().close(); + } } diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/TransportGetProfilingAction.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/TransportGetProfilingAction.java index 873094f580535..0587ee5dbe08e 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/TransportGetProfilingAction.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/TransportGetProfilingAction.java @@ -104,9 +104,9 @@ protected void doExecute(Task submitTask, GetProfilingRequest request, ActionLis log.debug("getResampledIndex took [" + (System.nanoTime() - start) / 1_000_000.0d + " ms]."); searchEventGroupByStackTrace(client, request, resampledIndex, submitListener); }, e -> { - // Apart from profiling-events-all, indices are created lazily. In a relatively empty cluster it can happen - // that there are so few data that we need to resort to the full index. As this is an edge case we'd rather - // fail instead of prematurely checking for existence in all cases. + // All profiling-events data streams are created lazily. In a relatively empty cluster it can happen that there are so few + // data that we need to resort to the "full" events stream. As this is an edge case we'd rather fail instead of prematurely + // checking for existence in all cases. if (e instanceof IndexNotFoundException) { String missingIndex = ((IndexNotFoundException) e).getIndex().getName(); EventsIndex fullIndex = EventsIndex.FULL_INDEX; @@ -163,7 +163,15 @@ private void searchEventGroupByStackTrace( } else { submitListener.onResponse(responseBuilder.build()); } - }, submitListener::onFailure)); + }, e -> { + // Data streams are created lazily; if even the "full" index does not exist no data have been indexed yet. + if (e instanceof IndexNotFoundException) { + log.debug("Index [{}] does not exist. Returning empty response.", ((IndexNotFoundException) e).getIndex()); + submitListener.onResponse(responseBuilder.build()); + } else { + submitListener.onFailure(e); + } + })); } private void retrieveStackTraces( diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json deleted file mode 100644 index 2be9a20d899a3..0000000000000 --- a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "policy": { - "phases": { - "hot": { - "min_age": "0ms", - "actions": { - "rollover": { - "max_primary_shard_size": "50gb", - "max_age": "7d" - }, - "set_priority": { - "priority": 100 - } - } - }, - "warm": { - "min_age": "30d", - "actions": { - "set_priority": { - "priority": 50 - }, - "shrink": { - "number_of_shards": 2 - } - } - }, - "delete": { - "min_age": "60d", - "actions": { - "delete": { - "delete_searchable_snapshot": true - } - } - } - } - } -} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/.profiling-ilm-lock.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/.profiling-ilm-lock.json deleted file mode 100644 index 9b17b7e97376a..0000000000000 --- a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/.profiling-ilm-lock.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "settings": { - "index": { - "hidden": true - } - }, - "mappings": { - "properties": { - "@timestamp": { - "type": "date", - "format": "epoch_second" - }, - "phase": { - "type": "keyword" - } - } - } -} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-returnpads-private.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-returnpads-private.json deleted file mode 100644 index 533b196ff9ce1..0000000000000 --- a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-returnpads-private.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "settings": { - "index": { - "refresh_interval": "10s" - } - }, - "mappings": { - "_source": { - "enabled": true - }, - "properties": { - "ecs.version": { - "type": "keyword", - "index": true, - "doc_values": false, - "store": false - }, - "Symbfile.created": { - "type": "date", - "doc_values": false, - "index": true, - "store": false - }, - "Symbfile.file.id": { - "type": "keyword", - "index": true, - "doc_values": false, - "store": false - }, - "Symbfile.part": { - "type": "short", - "index": false, - "doc_values": false, - "store": false - }, - "Symbfile.parts": { - "type": "short", - "index": false, - "doc_values": false, - "store": false - }, - "Symbfile.data": { - "type": "binary", - "doc_values": false, - "store": false - } - } - } -} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-sq-executables.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-sq-executables.json deleted file mode 100644 index 8e5e9935c6ecb..0000000000000 --- a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-sq-executables.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "settings": { - "index": { - "refresh_interval": "10s" - } - }, - "mappings": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "ecs.version": { - "type": "keyword", - "index": true - }, - "Executable.file.id": { - "type": "keyword", - "index": false - }, - "Time.created": { - "type": "date", - "index": true - }, - "Symbolization.time.next": { - "type": "date", - "index": true - }, - "Symbolization.retries": { - "type": "short", - "index": true - } - } - } -} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-sq-leafframes.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-sq-leafframes.json deleted file mode 100644 index ed778a1309bc5..0000000000000 --- a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-sq-leafframes.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "settings": { - "index": { - "refresh_interval": "10s" - } - }, - "mappings": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "ecs.version": { - "type": "keyword", - "index": true - }, - "Stacktrace.frame.id": { - "type": "keyword", - "index": false - }, - "Time.created": { - "type": "date", - "index": true - }, - "Symbolization.time.next": { - "type": "date", - "index": true - }, - "Symbolization.retries": { - "type": "short", - "index": true - } - } - } -} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-symbols-private.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-symbols-private.json deleted file mode 100644 index 7736a6202a9ad..0000000000000 --- a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-symbols-private.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": "16", - "refresh_interval": "10s" - } - }, - "mappings": { - "_source": { - "enabled": true - }, - "properties": { - "ecs.version": { - "type": "keyword", - "index": true, - "doc_values": false, - "store": false - }, - "Symbol.function.name": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.file.name": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.call.file.name": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.call.line": { - "type": "integer", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.function.line": { - "type": "integer", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.depth": { - "type": "integer", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.linetable.base": { - "type": "unsigned_long", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.linetable.length": { - "type": "unsigned_long", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.linetable.offsets": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.linetable.lines": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.file.id": { - "type": "keyword", - "index": true, - "doc_values": false, - "store": false - }, - "Symbol.exec.pcrange": { - "type": "ip_range", - "index": true, - "doc_values": false, - "store": false - } - } - } -} diff --git a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-symbols.json b/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-symbols.json deleted file mode 100644 index 519894977f0c9..0000000000000 --- a/x-pack/plugin/profiler/src/main/resources/org/elasticsearch/xpack/profiler/index/profiling-symbols.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": "16", - "refresh_interval": "10s" - } - }, - "mappings": { - "_source": { - "enabled": true - }, - "properties": { - "ecs.version": { - "type": "keyword", - "index": true, - "doc_values": false, - "store": false - }, - "Symbol.function.name": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.file.name": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.call.file.name": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.call.line": { - "type": "integer", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.function.line": { - "type": "integer", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.depth": { - "type": "integer", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.linetable.base": { - "type": "unsigned_long", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.linetable.length": { - "type": "unsigned_long", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.linetable.offsets": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.linetable.lines": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.file.id": { - "type": "keyword", - "index": true, - "doc_values": false, - "store": false - }, - "Symbol.exec.pcrange": { - "type": "ip_range", - "index": true, - "doc_values": false, - "store": false - } - } - } -} From 2c76e0ecf671f9fcb31fd1a31f7c9bf08779b4b3 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Tue, 2 May 2023 13:34:27 +0200 Subject: [PATCH 03/36] Temporarily disable profiling by default --- .../org/elasticsearch/xpack/profiler/ProfilingTestCase.java | 1 + .../java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java index 9b7985a103596..1872a95453efb 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java +++ b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java @@ -46,6 +46,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .put(super.nodeSettings(nodeOrdinal, otherSettings)) .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME) .put(NetworkModule.HTTP_TYPE_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME) + .put(ProfilingPlugin.PROFILING_ENABLED.getKey(), true) // .put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial") // Disable ILM history index so that the tests don't have to clean it up .put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING.getKey(), false) diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java index b21b74edf79eb..22b5a3caad69e 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java @@ -50,7 +50,7 @@ public class ProfilingPlugin extends Plugin implements ActionPlugin { private static final Logger logger = LogManager.getLogger(ProfilingPlugin.class); public static final Setting PROFILING_ENABLED = Setting.boolSetting( "xpack.profiling.enabled", - true, + false, Setting.Property.NodeScope ); public static final String PROFILING_THREAD_POOL_NAME = "profiling"; From 0cb1a58b78aa754b0c874e5123e539222ac13914 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Tue, 2 May 2023 14:35:31 +0200 Subject: [PATCH 04/36] Use hidden / versioned indices --- .../profiling-returnpads-private.json | 7 ++++--- .../index-template/profiling-sq-executables.json | 7 ++++--- .../index-template/profiling-sq-leafframes.json | 7 ++++--- .../profiler/index-template/profiling-symbols.json | 7 ++++--- .../xpack/profiler/ProfilingIndexManager.java | 14 +++++++++----- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json index acfb08109884f..e5436abdd824b 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json @@ -1,11 +1,12 @@ { "index_patterns": [ - "profiling-returnpads-private*" + ".profiling-returnpads-private*" ], "template": { "settings": { "index": { - "refresh_interval": "10s" + "refresh_interval": "10s", + "hidden": true } }, "mappings": { @@ -53,7 +54,7 @@ }, "priority": 100, "_meta": { - "description": "Index template for profiling-returnpads-private" + "description": "Index template for .profiling-returnpads-private" }, "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json index d410b704443d1..35ae771907fd4 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json @@ -1,11 +1,12 @@ { "index_patterns": [ - "profiling-sq-executables*" + ".profiling-sq-executables*" ], "template": { "settings": { "index": { - "refresh_interval": "10s" + "refresh_interval": "10s", + "hidden": true } }, "mappings": { @@ -38,7 +39,7 @@ }, "priority": 100, "_meta": { - "description": "Index template for profiling-sq-executables" + "description": "Index template for .profiling-sq-executables" }, "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json index 3e6b32b4fffab..0491d75ac0153 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json @@ -1,11 +1,12 @@ { "index_patterns": [ - "profiling-sq-leafframes*" + ".profiling-sq-leafframes*" ], "template": { "settings": { "index": { - "refresh_interval": "10s" + "refresh_interval": "10s", + "hidden": true } }, "mappings": { @@ -38,7 +39,7 @@ }, "priority": 100, "_meta": { - "description": "Index template for profiling-sq-leafframes" + "description": "Index template for .profiling-sq-leafframes" }, "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json index 859608ea35dc7..d0cd3986e412e 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json @@ -1,12 +1,13 @@ { "index_patterns": [ - "profiling-symbols*" + ".profiling-symbols*" ], "template": { "settings": { "index": { "number_of_shards": "16", - "refresh_interval": "10s" + "refresh_interval": "10s", + "hidden": true } }, "mappings": { @@ -97,7 +98,7 @@ }, "priority": 100, "_meta": { - "description": "Index template for profiling-symbols" + "description": "Index template for .profiling-symbols" }, "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java index dd8ae0d70108f..db1bd918f2c52 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java @@ -45,13 +45,17 @@ public class ProfilingIndexManager implements ClusterStateListener, Closeable { private static final Map INDICES_AND_ALIASES; static { + String versionSuffix = "-v" + ProfilingIndexTemplateRegistry.INDEX_TEMPLATE_VERSION; + Map indicesAndAliases = new HashMap<>(); + // TODO: Define behavior on upgrade (delete, reindex, ...), to be done after 8.9.0 + // TODO: This index will be gone with the 8.9 release. Don't bother to implement versioning support. indicesAndAliases.put(".profiling-ilm-lock", null); - indicesAndAliases.put("profiling-returnpads-private", null); - indicesAndAliases.put("profiling-sq-executables", null); - indicesAndAliases.put("profiling-sq-leafframes", null); - indicesAndAliases.put("profiling-symbols", null); - indicesAndAliases.put("profiling-symbols-private", null); + indicesAndAliases.put(".profiling-returnpads-private" + versionSuffix, "profiling-returnpads-private"); + indicesAndAliases.put(".profiling-sq-executables" + versionSuffix, "profiling-sq-executables"); + indicesAndAliases.put(".profiling-sq-leafframes" + versionSuffix, "profiling-sq-leafframes"); + indicesAndAliases.put(".profiling-symbols" + versionSuffix, "profiling-symbols"); + indicesAndAliases.put(".profiling-symbols-private" + versionSuffix, "profiling-symbols-private"); // TODO: Update these to the new K/V strategy after all readers have been adjusted String[] kvIndices = new String[] { "profiling-executables", "profiling-stackframes", "profiling-stacktraces" }; for (String idx : kvIndices) { From 1ae10121107c1a16b77588e678b10b0479c013ae Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Tue, 2 May 2023 14:46:22 +0200 Subject: [PATCH 05/36] Port comments in index templates --- .../profiling-returnpads-private.json | 6 +++++ .../index-template/profiling-symbols.json | 24 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json index e5436abdd824b..ae2691528d06e 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json @@ -10,6 +10,10 @@ } }, "mappings": { + /* + Enable source as the main purpose of this index is to store relatively few BLOBs. + Disabling source is another option, but adds the 'recovery_source' extra storage costs. + */ "_source": { "enabled": true }, @@ -27,6 +31,7 @@ "store": false }, "Symbfile.file.id": { + /* 'binary' type fields don't allow using 'index: true'. */ "type": "keyword", "index": true, "doc_values": false, @@ -38,6 +43,7 @@ "doc_values": false, "store": false }, + /* This is the number of parts for the file.id. It only exists in the last part. */ "Symbfile.parts": { "type": "short", "index": false, diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json index d0cd3986e412e..0c93845e0ac02 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json @@ -11,6 +11,10 @@ } }, "mappings": { + /* + Enable source for now because ip_range is not supported by synthetic source. + And we currently want to use the reindex API for experiments with storage costs. + */ "_source": { "enabled": true }, @@ -21,72 +25,92 @@ "doc_values": false, "store": false }, + /* name of the function */ "Symbol.function.name": { "type": "keyword", "index": false, "doc_values": false, "store": false }, + /* file path */ "Symbol.file.name": { "type": "keyword", "index": false, "doc_values": false, "store": false }, + /* (for inlined functions) file path where inline function was called */ "Symbol.call.file.name": { "type": "keyword", "index": false, "doc_values": false, "store": false }, + /* (for inlined functions) line where inline function was called */ "Symbol.call.line": { "type": "integer", "index": false, "doc_values": false, "store": false }, + /* function start line (only available from DWARF). Currently unused. */ "Symbol.function.line": { "type": "integer", "index": false, "doc_values": false, "store": false }, + /* inline depth */ "Symbol.depth": { "type": "integer", "index": false, "doc_values": false, "store": false }, + /* + pairs of (32bit PC offset, 32bit line number) followed by 64bit PC range base at the end. + To find line number for a given PC: find lowest offset such as offsetBase+PC >= offset, then read corresponding line number. + offsetBase could seemingly be available from exec_pc_range (it's the first value of the pair), but it's not the case. + Ranges are stored as points, which cannot be retrieve when disabling _source. + See https://www.elastic.co/guide/en/elasticsearch/reference/current/point.html . + + Linetable: base for offsets (64bit PC range base) + */ "Symbol.linetable.base": { "type": "unsigned_long", "index": false, "doc_values": false, "store": false }, + /* Linetable: length of range (PC range is [base, base+length)) */ "Symbol.linetable.length": { "type": "unsigned_long", "index": false, "doc_values": false, "store": false }, + /* Linetable: concatenated offsets (each value is ULEB128encoded) */ "Symbol.linetable.offsets": { "type": "keyword", "index": false, "doc_values": false, "store": false }, + /* Linetable: concatenated lines (each value is ULEB128 encoded) */ "Symbol.linetable.lines": { "type": "keyword", "index": false, "doc_values": false, "store": false }, + /* fileID. used for deletion and Symbol.exec.pcrange collision handling on symbolization */ "Symbol.file.id": { "type": "keyword", "index": true, "doc_values": false, "store": false }, + /* PC ranges [begin, end) */ "Symbol.exec.pcrange": { "type": "ip_range", "index": true, From 4e0bb384491414e791f2a6fefd81434772f2fe68 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Tue, 2 May 2023 14:49:42 +0200 Subject: [PATCH 06/36] Port profiling-metrics --- .../component-template/profiling-metrics.json | 42 +++++++++++++++++++ .../index-template/profiling-metrics.json | 15 +++++++ .../ProfilingIndexTemplateRegistry.java | 12 ++++++ 3 files changed, 69 insertions(+) create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-metrics.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-metrics.json diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-metrics.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-metrics.json new file mode 100644 index 0000000000000..a079026e5b50c --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-metrics.json @@ -0,0 +1,42 @@ +{ + "template": { + "settings": { + "index": { + "refresh_interval": "10s", + "sort": { + "field": [ + "project.id", + "@timestamp", + "host.id" + ] + } + }, + "codec": "best_compression" + }, + "mappings": { + "_source": { + "enabled": false + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true + }, + "project.id": { + "type": "keyword" + }, + "host.id": { + "type": "keyword" + }, + "@timestamp": { + "type": "date", + "format": "epoch_second" + }, + "Elasticsearch.cluster.id": { + "type": "keyword" + } + } + } + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-metrics.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-metrics.json new file mode 100644 index 0000000000000..080bd7d5bcef4 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-metrics.json @@ -0,0 +1,15 @@ +{ + "index_patterns": [ + "profiling-metrics*" + ], + "data_stream": {}, + "composed_of": [ + "profiling-metrics", + "profiling-ilm" + ], + "priority": 100, + "_meta": { + "description": "Template for profiling-metrics" + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java index 3dee116ec48da..6017446a6f551 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java @@ -97,6 +97,12 @@ protected List getPolicyConfigs() { INDEX_TEMPLATE_VERSION, PROFILING_TEMPLATE_VERSION_VARIABLE ), + new IndexTemplateConfig( + "profiling-metrics", + "/org/elasticsearch/xpack/profiler/component-template/profiling-metrics.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), new IndexTemplateConfig( "profiling-stackframes", "/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json", @@ -134,6 +140,12 @@ protected Map getComponentTemplateConfigs() { INDEX_TEMPLATE_VERSION, PROFILING_TEMPLATE_VERSION_VARIABLE ), + new IndexTemplateConfig( + "profiling-metrics", + "/org/elasticsearch/xpack/profiler/index-template/profiling-metrics.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), new IndexTemplateConfig( "profiling-executables", "/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json", From 1f455426629ead0b3cdfec20b5dc3d2affbb0606 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Wed, 3 May 2023 14:11:52 +0200 Subject: [PATCH 07/36] Add test --- .../ProfilingIndexTemplateRegistryTests.java | 375 ++++++++++++++++++ 1 file changed, 375 insertions(+) create mode 100644 x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistryTests.java diff --git a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistryTests.java b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistryTests.java new file mode 100644 index 0000000000000..f6fc79e19c473 --- /dev/null +++ b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistryTests.java @@ -0,0 +1,375 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.profiler; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.node.TestDiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.TriFunction; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.ilm.DeleteAction; +import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; +import org.elasticsearch.xpack.core.ilm.LifecycleAction; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.ilm.OperationMode; +import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class ProfilingIndexTemplateRegistryTests extends ESTestCase { + private ProfilingIndexTemplateRegistry registry; + private ClusterService clusterService; + private ThreadPool threadPool; + private VerifyingClient client; + + @Before + public void createRegistryAndClient() { + threadPool = new TestThreadPool(this.getClass().getName()); + client = new VerifyingClient(threadPool); + clusterService = ClusterServiceUtils.createClusterService(threadPool); + registry = new ProfilingIndexTemplateRegistry(Settings.EMPTY, clusterService, threadPool, client, NamedXContentRegistry.EMPTY); + } + + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdownNow(); + } + + public void testThatMissingMasterNodeDoesNothing() { + DiscoveryNode localNode = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build(); + + client.setVerifier((a, r, l) -> { + fail("if the master is missing nothing should happen"); + return null; + }); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), Collections.emptyMap(), nodes); + registry.clusterChanged(event); + } + + public void testThatNonExistingTemplatesAreAddedImmediately() throws Exception { + DiscoveryNode node = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + // component templates are already existing, just add composable templates + Map componentTemplates = new HashMap<>(); + for (String templateName : registry.getComponentTemplateConfigs().keySet()) { + componentTemplates.put(templateName, ProfilingIndexTemplateRegistry.INDEX_TEMPLATE_VERSION); + } + ClusterChangedEvent event = createClusterChangedEvent(componentTemplates, Collections.emptyMap(), nodes); + + AtomicInteger calledTimes = new AtomicInteger(0); + + client.setVerifier((action, request, listener) -> verifyComposableTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComposableTemplateConfigs().size()))); + + calledTimes.set(0); + + // attempting to register the event multiple times as a race condition can yield this test flaky, namely: + // when calling registry.clusterChanged(newEvent) the templateCreationsInProgress state that the IndexTemplateRegistry maintains + // might've not yet been updated to reflect that the first template registration was complete, so a second template registration + // will not be issued anymore, leaving calledTimes to 0 + assertBusy(() -> { + // now delete one template from the cluster state and lets retry + ClusterChangedEvent newEvent = createClusterChangedEvent(Collections.emptyMap(), Collections.emptyMap(), nodes); + registry.clusterChanged(newEvent); + assertThat(calledTimes.get(), greaterThan(1)); + }); + } + + public void testThatNonExistingPoliciesAreAddedImmediately() { + DiscoveryNode node = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> { + if (action instanceof PutLifecycleAction) { + calledTimes.incrementAndGet(); + assertThat(action, instanceOf(PutLifecycleAction.class)); + assertThat(request, instanceOf(PutLifecycleAction.Request.class)); + final PutLifecycleAction.Request putRequest = (PutLifecycleAction.Request) request; + assertThat(putRequest.getPolicy().getName(), equalTo("profiling")); + assertNotNull(listener); + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutComposableIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else { + fail("client called with unexpected request: " + request.toString()); + return null; + } + }); + } + + public void testPolicyAlreadyExists() { + DiscoveryNode node = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + Map policyMap = new HashMap<>(); + List policies = registry.getPolicyConfigs(); + assertThat(policies, hasSize(1)); + policies.forEach(p -> policyMap.put(p.getName(), p)); + + client.setVerifier((action, request, listener) -> { + if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutComposableIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutLifecycleAction) { + fail("if the policy already exists it should not be re-put"); + } else { + fail("client called with unexpected request: " + request.toString()); + } + return null; + }); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), Collections.emptyMap(), policyMap, nodes); + registry.clusterChanged(event); + } + + public void testPolicyAlreadyExistsButDiffers() throws IOException { + DiscoveryNode node = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + Map policyMap = new HashMap<>(); + String policyStr = "{\"phases\":{\"delete\":{\"min_age\":\"1m\",\"actions\":{\"delete\":{}}}}}"; + List policies = registry.getPolicyConfigs(); + assertThat(policies, hasSize(1)); + policies.forEach(p -> policyMap.put(p.getName(), p)); + + client.setVerifier((action, request, listener) -> { + if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutComposableIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutLifecycleAction) { + fail("if the policy already exists it should not be re-put"); + } else { + fail("client called with unexpected request: " + request.toString()); + } + return null; + }); + + try ( + XContentParser parser = XContentType.JSON.xContent() + .createParser( + XContentParserConfiguration.EMPTY.withRegistry( + new NamedXContentRegistry( + List.of( + new NamedXContentRegistry.Entry( + LifecycleAction.class, + new ParseField(DeleteAction.NAME), + DeleteAction::parse + ) + ) + ) + ), + policyStr + ) + ) { + LifecyclePolicy different = LifecyclePolicy.parse(parser, policies.get(0).getName()); + policyMap.put(policies.get(0).getName(), different); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), Collections.emptyMap(), policyMap, nodes); + registry.clusterChanged(event); + } + } + + /** + * A client that delegates to a verifying function for action/request/listener + */ + public static class VerifyingClient extends NoOpClient { + + private TriFunction, ActionRequest, ActionListener, ActionResponse> verifier = (a, r, l) -> { + fail("verifier not set"); + return null; + }; + + VerifyingClient(ThreadPool threadPool) { + super(threadPool); + } + + @Override + @SuppressWarnings("unchecked") + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + try { + listener.onResponse((Response) verifier.apply(action, request, listener)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public VerifyingClient setVerifier(TriFunction, ActionRequest, ActionListener, ActionResponse> verifier) { + this.verifier = verifier; + return this; + } + } + + private ActionResponse verifyComposableTemplateInstalled( + AtomicInteger calledTimes, + ActionType action, + ActionRequest request, + ActionListener listener + ) { + if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutLifecycleAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutComposableIndexTemplateAction) { + calledTimes.incrementAndGet(); + assertThat(action, instanceOf(PutComposableIndexTemplateAction.class)); + assertThat(request, instanceOf(PutComposableIndexTemplateAction.Request.class)); + final PutComposableIndexTemplateAction.Request putRequest = ((PutComposableIndexTemplateAction.Request) request); + assertThat(putRequest.indexTemplate().version(), equalTo((long) ProfilingIndexTemplateRegistry.INDEX_TEMPLATE_VERSION)); + assertNotNull(listener); + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else { + fail("client called with unexpected request:" + request.toString()); + return null; + } + } + + private ClusterChangedEvent createClusterChangedEvent( + Map existingComponentTemplates, + Map existingComposableTemplates, + DiscoveryNodes nodes + ) { + return createClusterChangedEvent(existingComponentTemplates, existingComposableTemplates, Collections.emptyMap(), nodes); + } + + private ClusterChangedEvent createClusterChangedEvent( + Map existingComponentTemplates, + Map existingComposableTemplates, + Map existingPolicies, + DiscoveryNodes nodes + ) { + ClusterState cs = createClusterState( + Settings.EMPTY, + existingComponentTemplates, + existingComposableTemplates, + existingPolicies, + nodes + ); + ClusterChangedEvent realEvent = new ClusterChangedEvent( + "created-from-test", + cs, + ClusterState.builder(new ClusterName("test")).build() + ); + ClusterChangedEvent event = spy(realEvent); + when(event.localNodeMaster()).thenReturn(nodes.isLocalNodeElectedMaster()); + + return event; + } + + private ClusterState createClusterState( + Settings nodeSettings, + Map existingComponentTemplates, + Map existingComposableTemplates, + Map existingPolicies, + DiscoveryNodes nodes + ) { + Map componentTemplates = new HashMap<>(); + for (Map.Entry template : existingComponentTemplates.entrySet()) { + ComponentTemplate mockTemplate = mock(ComponentTemplate.class); + when(mockTemplate.version()).thenReturn(template.getValue() == null ? null : (long) template.getValue()); + componentTemplates.put(template.getKey(), mockTemplate); + } + Map composableTemplates = new HashMap<>(); + for (Map.Entry template : existingComposableTemplates.entrySet()) { + ComposableIndexTemplate mockTemplate = mock(ComposableIndexTemplate.class); + when(mockTemplate.version()).thenReturn(template.getValue() == null ? null : (long) template.getValue()); + composableTemplates.put(template.getKey(), mockTemplate); + } + + Map existingILMMeta = existingPolicies.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> new LifecyclePolicyMetadata(e.getValue(), Collections.emptyMap(), 1, 1))); + IndexLifecycleMetadata ilmMeta = new IndexLifecycleMetadata(existingILMMeta, OperationMode.RUNNING); + + return ClusterState.builder(new ClusterName("test")) + .metadata( + Metadata.builder() + .componentTemplates(componentTemplates) + .indexTemplates(composableTemplates) + .transientSettings(nodeSettings) + .putCustom(IndexLifecycleMetadata.TYPE, ilmMeta) + .build() + ) + .blocks(new ClusterBlocks.Builder().build()) + .nodes(nodes) + .build(); + } +} From 72b0698551d8528f360bd1a3dfe47db2f3a8e06e Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Fri, 5 May 2023 07:33:57 +0200 Subject: [PATCH 08/36] Add test for ProfilingIndexManager --- .../xpack/profiler/ProfilingIndexManager.java | 13 +- .../profiler/ProfilingIndexManagerTests.java | 183 ++++++++++++++++++ .../ProfilingIndexTemplateRegistryTests.java | 36 ---- .../xpack/profiler/VerifyingClient.java | 51 +++++ 4 files changed, 243 insertions(+), 40 deletions(-) create mode 100644 x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexManagerTests.java create mode 100644 x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/VerifyingClient.java diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java index db1bd918f2c52..13cfe251ff5f8 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java @@ -42,7 +42,8 @@ */ public class ProfilingIndexManager implements ClusterStateListener, Closeable { private static final Logger logger = LogManager.getLogger(ProfilingIndexManager.class); - private static final Map INDICES_AND_ALIASES; + // For testing + public static final Map INDICES_AND_ALIASES; static { String versionSuffix = "-v" + ProfilingIndexTemplateRegistry.INDEX_TEMPLATE_VERSION; @@ -59,8 +60,8 @@ public class ProfilingIndexManager implements ClusterStateListener, Closeable { // TODO: Update these to the new K/V strategy after all readers have been adjusted String[] kvIndices = new String[] { "profiling-executables", "profiling-stackframes", "profiling-stacktraces" }; for (String idx : kvIndices) { - indicesAndAliases.put(idx + "-000001", idx); - indicesAndAliases.put(idx + "-000002", idx + "-next"); + indicesAndAliases.put(idx + versionSuffix + "-000001", idx); + indicesAndAliases.put(idx + versionSuffix + "-000002", idx + "-next"); } INDICES_AND_ALIASES = Collections.unmodifiableMap(indicesAndAliases); } @@ -103,7 +104,7 @@ public void clusterChanged(ClusterChangedEvent event) { } // ensure that index templates are present - if (ProfilingIndexTemplateRegistry.isAllTemplatesCreated(event.state()) == false) { + if (isAllTemplatesCreated(event) == false) { logger.trace("Skipping index creation; not all templates are present yet"); return; } @@ -111,6 +112,10 @@ public void clusterChanged(ClusterChangedEvent event) { addIndicesIfMissing(event.state()); } + protected boolean isAllTemplatesCreated(ClusterChangedEvent event) { + return ProfilingIndexTemplateRegistry.isAllTemplatesCreated(event.state()); + } + private void addIndicesIfMissing(ClusterState state) { Optional> maybeMeta = Optional.ofNullable(state.metadata().indices()); for (Map.Entry idxAlias : INDICES_AND_ALIASES.entrySet()) { diff --git a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexManagerTests.java b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexManagerTests.java new file mode 100644 index 0000000000000..5fe54c86fa8cc --- /dev/null +++ b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexManagerTests.java @@ -0,0 +1,183 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.profiler; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.node.TestDiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class ProfilingIndexManagerTests extends ESTestCase { + private final AtomicBoolean templatesCreated = new AtomicBoolean(); + private ProfilingIndexManager indexManager; + private ClusterService clusterService; + private ThreadPool threadPool; + private VerifyingClient client; + + @Before + public void createRegistryAndClient() { + templatesCreated.set(false); + threadPool = new TestThreadPool(this.getClass().getName()); + client = new VerifyingClient(threadPool); + clusterService = ClusterServiceUtils.createClusterService(threadPool); + indexManager = new ProfilingIndexManager(threadPool, client, clusterService) { + @Override + protected boolean isAllTemplatesCreated(ClusterChangedEvent event) { + return templatesCreated.get(); + } + }; + } + + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdownNow(); + } + + public void testThatMissingMasterNodeDoesNothing() { + DiscoveryNode localNode = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build(); + + client.setVerifier((a, r, l) -> { + fail("if the master is missing nothing should happen"); + return null; + }); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes); + indexManager.clusterChanged(event); + } + + public void testThatMissingTemplatesDoesNothing() { + DiscoveryNode node = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + client.setVerifier((a, r, l) -> { + fail("if any templates are missing nothing should happen"); + return null; + }); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes); + indexManager.clusterChanged(event); + } + + public void testThatNonExistingIndicesAreAddedImmediately() throws Exception { + DiscoveryNode node = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + templatesCreated.set(true); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes); + + AtomicInteger calledTimes = new AtomicInteger(0); + + client.setVerifier((action, request, listener) -> verifyIndexInstalled(calledTimes, action, request, listener)); + indexManager.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(ProfilingIndexManager.INDICES_AND_ALIASES.size()))); + + calledTimes.set(0); + } + + public void testThatExistingIndicesAreNotCreatedTwice() throws Exception { + DiscoveryNode node = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + templatesCreated.set(true); + + String existingIndex = randomFrom(ProfilingIndexManager.INDICES_AND_ALIASES.keySet()); + ClusterChangedEvent event = createClusterChangedEvent(List.of(existingIndex), nodes); + + AtomicInteger calledTimes = new AtomicInteger(0); + + client.setVerifier((action, request, listener) -> verifyIndexInstalled(calledTimes, action, request, listener)); + indexManager.clusterChanged(event); + // should not create the existing index + assertBusy(() -> assertThat(calledTimes.get(), equalTo(ProfilingIndexManager.INDICES_AND_ALIASES.size() - 1))); + + calledTimes.set(0); + } + + private ActionResponse verifyIndexInstalled( + AtomicInteger calledTimes, + ActionType action, + ActionRequest request, + ActionListener listener + ) { + if (action instanceof CreateIndexAction) { + calledTimes.incrementAndGet(); + assertThat(action, instanceOf(CreateIndexAction.class)); + assertThat(request, instanceOf(CreateIndexRequest.class)); + assertNotNull(listener); + return new CreateIndexResponse(true, true, ((CreateIndexRequest) request).index()); + } else { + fail("client called with unexpected request:" + request.toString()); + return null; + } + } + + private ClusterChangedEvent createClusterChangedEvent(Iterable existingIndices, DiscoveryNodes nodes) { + ClusterState cs = createClusterState(Settings.EMPTY, existingIndices, nodes); + ClusterChangedEvent realEvent = new ClusterChangedEvent( + "created-from-test", + cs, + ClusterState.builder(new ClusterName("test")).build() + ); + ClusterChangedEvent event = spy(realEvent); + when(event.localNodeMaster()).thenReturn(nodes.isLocalNodeElectedMaster()); + + return event; + } + + private ClusterState createClusterState(Settings nodeSettings, Iterable existingIndices, DiscoveryNodes nodes) { + Map indices = new HashMap<>(); + for (String index : existingIndices) { + IndexMetadata mockMetadata = mock(IndexMetadata.class); + when(mockMetadata.getIndex()).thenReturn(new Index(index, index)); + when(mockMetadata.getCompatibilityVersion()).thenReturn(Version.CURRENT); + indices.put(index, mockMetadata); + } + return ClusterState.builder(new ClusterName("test")) + .metadata(Metadata.builder().indices(indices).transientSettings(nodeSettings).build()) + .blocks(new ClusterBlocks.Builder().build()) + .nodes(nodes) + .build(); + } +} diff --git a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistryTests.java b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistryTests.java index f6fc79e19c473..f433c08537c36 100644 --- a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistryTests.java +++ b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistryTests.java @@ -26,11 +26,9 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.node.TestDiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -239,40 +237,6 @@ public void testPolicyAlreadyExistsButDiffers() throws IOException { } } - /** - * A client that delegates to a verifying function for action/request/listener - */ - public static class VerifyingClient extends NoOpClient { - - private TriFunction, ActionRequest, ActionListener, ActionResponse> verifier = (a, r, l) -> { - fail("verifier not set"); - return null; - }; - - VerifyingClient(ThreadPool threadPool) { - super(threadPool); - } - - @Override - @SuppressWarnings("unchecked") - protected void doExecute( - ActionType action, - Request request, - ActionListener listener - ) { - try { - listener.onResponse((Response) verifier.apply(action, request, listener)); - } catch (Exception e) { - listener.onFailure(e); - } - } - - public VerifyingClient setVerifier(TriFunction, ActionRequest, ActionListener, ActionResponse> verifier) { - this.verifier = verifier; - return this; - } - } - private ActionResponse verifyComposableTemplateInstalled( AtomicInteger calledTimes, ActionType action, diff --git a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/VerifyingClient.java b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/VerifyingClient.java new file mode 100644 index 0000000000000..10282adf8a56f --- /dev/null +++ b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/VerifyingClient.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.profiler; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.common.TriFunction; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.Assert; + +/** + * A client that delegates to a verifying function for action/request/listener + */ +public class VerifyingClient extends NoOpClient { + + private TriFunction, ActionRequest, ActionListener, ActionResponse> verifier = (a, r, l) -> { + Assert.fail("verifier not set"); + return null; + }; + + VerifyingClient(ThreadPool threadPool) { + super(threadPool); + } + + @Override + @SuppressWarnings("unchecked") + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + try { + listener.onResponse((Response) verifier.apply(action, request, listener)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public VerifyingClient setVerifier(TriFunction, ActionRequest, ActionListener, ActionResponse> verifier) { + this.verifier = verifier; + return this; + } +} From 62aad06e609e4e94ff2ef3e07dc55f1ea6fe0396 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Fri, 5 May 2023 08:42:13 +0200 Subject: [PATCH 09/36] More robust precondition check in integration tests --- .../org/elasticsearch/xpack/profiler/ProfilingTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java index 1872a95453efb..ba38dcb9f562c 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java +++ b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java @@ -91,7 +91,7 @@ protected void waitForIndices() throws Exception { // simple check whether at least one of the indices is present assertTrue( "Timed out waiting for the indices to be created", - state.metadata().indices().containsKey("profiling-stackframes-000002") + state.metadata().indices().keySet().containsAll(ProfilingIndexManager.INDICES_AND_ALIASES.keySet()) ); }); } From 6de3263799599cb925f4f6bd9621b682da8a80c3 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Fri, 5 May 2023 09:31:03 +0200 Subject: [PATCH 10/36] Update docs/changelog/95666.yaml --- docs/changelog/95666.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/95666.yaml diff --git a/docs/changelog/95666.yaml b/docs/changelog/95666.yaml new file mode 100644 index 0000000000000..5f8d5c62fc05e --- /dev/null +++ b/docs/changelog/95666.yaml @@ -0,0 +1,5 @@ +pr: 95666 +summary: Bootstrap profiling indices at startup +area: Indices APIs +type: enhancement +issues: [] From 8b2ecef307cb7e0e9deba5693641d2039535b4e0 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Mon, 8 May 2023 09:17:01 +0200 Subject: [PATCH 11/36] Introduce xpack.profiling.templates.enabled --- .../elasticsearch/test/ESIntegTestCase.java | 15 ++---------- x-pack/plugin/build.gradle | 1 + x-pack/plugin/eql/qa/mixed-node/build.gradle | 1 + .../multi-cluster-with-security/build.gradle | 1 + .../plugin/ilm/qa/multi-cluster/build.gradle | 3 ++- x-pack/plugin/ilm/qa/multi-node/build.gradle | 1 + .../plugin/ilm/qa/with-security/build.gradle | 2 +- .../ml/qa/basic-multi-node/build.gradle | 1 + .../build.gradle | 2 ++ .../qa/native-multi-node-tests/build.gradle | 1 + .../xpack/profiler/ProfilingTestCase.java | 24 ++++++++++++++----- .../xpack/profiler/ProfilingIndexManager.java | 8 +++++++ .../ProfilingIndexTemplateRegistry.java | 12 +++++++--- .../xpack/profiler/ProfilingPlugin.java | 22 ++++++++++++++++- .../profiler/ProfilingIndexManagerTests.java | 1 + .../ProfilingIndexTemplateRegistryTests.java | 1 + x-pack/plugin/sql/qa/jdbc/build.gradle | 1 + x-pack/plugin/sql/qa/mixed-node/build.gradle | 1 + x-pack/plugin/sql/qa/server/build.gradle | 1 + .../multi-cluster-with-security/build.gradle | 2 ++ .../build.gradle | 2 ++ .../watcher/qa/with-monitoring/build.gradle | 1 + ...CoreWithSecurityClientYamlTestSuiteIT.java | 1 + x-pack/qa/mixed-tier-cluster/build.gradle | 1 + .../GlobalCheckpointSyncActionIT.java | 1 + .../elasticsearch/multi_node/RollupIT.java | 1 + x-pack/qa/rolling-upgrade-basic/build.gradle | 3 ++- .../build.gradle | 1 + .../runtime-fields/with-security/build.gradle | 1 + .../build.gradle | 1 + .../qa/xpack-prefix-rest-compat/build.gradle | 1 + 31 files changed, 89 insertions(+), 26 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 65bbfbb0cb520..c1f22875b55c3 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -362,9 +362,7 @@ protected final void beforeInternal() throws Exception { final Scope currentClusterScope = getCurrentClusterScope(); Callable setup = () -> { cluster().beforeTest(random()); - if (wipeCluster()) { - cluster().wipe(excludeTemplates()); - } + cluster().wipe(excludeTemplates()); randomIndexTemplate(); return null; }; @@ -580,9 +578,7 @@ private void afterInternal(boolean afterClass) throws Exception { ensureClusterStateCanBeReadByNodeTool(); ensureClusterInfoServiceRunning(); beforeIndexDeletion(); - if (wipeCluster()) { - cluster().wipe(excludeTemplates()); // wipe after to make sure we fail in the test that didn't ack the delete - } + cluster().wipe(excludeTemplates()); // wipe after to make sure we fail in the test that didn't ack the delete if (afterClass || currentClusterScope == Scope.TEST) { cluster().close(); } @@ -610,13 +606,6 @@ protected Set excludeTemplates() { return Collections.emptySet(); } - /** - * @return true if the cluster should be wiped before each test. By default, the cluster is wiped. - */ - protected boolean wipeCluster() { - return true; - } - protected void beforeIndexDeletion() throws Exception { cluster().beforeIndexDeletion(); } diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 93aa30ec68f44..3a28f3d6c3379 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -187,6 +187,7 @@ testClusters.configureEach { setting 'xpack.ml.enabled', 'true' setting 'xpack.security.enabled', 'true' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' // Integration tests are supposed to enable/disable exporters before/after each test setting 'xpack.security.authc.token.enabled', 'true' setting 'xpack.security.authc.api_key.enabled', 'true' diff --git a/x-pack/plugin/eql/qa/mixed-node/build.gradle b/x-pack/plugin/eql/qa/mixed-node/build.gradle index 3e1010e2c0eb9..9a3bdd94cb160 100644 --- a/x-pack/plugin/eql/qa/mixed-node/build.gradle +++ b/x-pack/plugin/eql/qa/mixed-node/build.gradle @@ -21,6 +21,7 @@ BuildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter("7.10.0") && testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.eql.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle b/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle index 5fc247693c453..8008505cb9437 100644 --- a/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle +++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle @@ -22,6 +22,7 @@ def integTestClusterReg = testClusters.register('javaRestTest') { testDistribution = 'DEFAULT' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'cluster.remote.my_remote_cluster.seeds', { remoteClusterReg.get().getAllTransportPortURI().collect { "\"$it\"" }.toString() } diff --git a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle index a1a2a5129e3f7..33ce1b0ce76cc 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle @@ -48,6 +48,7 @@ testClusters.matching{ it.name == 'follow-cluster' }.configureEach { setting 'xpack.ccr.enabled', 'true' setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.poll_interval', '1000ms' @@ -59,4 +60,4 @@ tasks.named("check").configure { dependsOn 'follow-cluster' } // Security is explicitly disabled for follow-cluster and leader-cluster, do not run these in FIPS mode tasks.withType(Test).configureEach { enabled = BuildParams.inFipsJvm == false -} \ No newline at end of file +} diff --git a/x-pack/plugin/ilm/qa/multi-node/build.gradle b/x-pack/plugin/ilm/qa/multi-node/build.gradle index 523bf5d04ae4f..f117bb646c50d 100644 --- a/x-pack/plugin/ilm/qa/multi-node/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-node/build.gradle @@ -24,6 +24,7 @@ testClusters.configureEach { setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.poll_interval', '1000ms' diff --git a/x-pack/plugin/ilm/qa/with-security/build.gradle b/x-pack/plugin/ilm/qa/with-security/build.gradle index 7aaa43f14fe3f..0a0ca062c0adf 100644 --- a/x-pack/plugin/ilm/qa/with-security/build.gradle +++ b/x-pack/plugin/ilm/qa/with-security/build.gradle @@ -9,7 +9,7 @@ dependencies { testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' rolesFile file('roles.yml') diff --git a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle index 72793d1f9a414..3cbede0af94ff 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle +++ b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle @@ -8,6 +8,7 @@ testClusters.configureEach { setting 'xpack.security.enabled', 'false' setting 'xpack.monitoring.elasticsearch.collection.enabled', 'false' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.ml.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.history_index_enabled', 'false' diff --git a/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle b/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle index 368aab4615fc7..945bf2c33efa8 100644 --- a/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle @@ -25,6 +25,7 @@ def remoteCluster = testClusters.register('remote-cluster') { setting 'node.roles', '[data,ingest,master]' setting 'xpack.security.enabled', 'true' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' user username: "test_user", password: "x-pack-test-password" @@ -36,6 +37,7 @@ testClusters.register('mixed-cluster') { setting 'node.roles', '[data,ingest,master]' setting 'xpack.security.enabled', 'true' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'cluster.remote.my_remote_cluster.seeds', { remoteCluster.get().getAllTransportPortURI().collect { "\"$it\"" }.toString() diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle index c9643ce25dddb..013df22a00925 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle @@ -42,6 +42,7 @@ testClusters.configureEach { setting 'xpack.security.enabled', 'true' setting 'xpack.ml.enabled', 'true' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.authc.token.enabled', 'true' setting 'xpack.security.transport.ssl.enabled', 'true' setting 'xpack.security.transport.ssl.key', nodeKey.name diff --git a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java index ba38dcb9f562c..17c34412e4b7c 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java +++ b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.profiler; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.network.NetworkModule; @@ -20,6 +22,7 @@ import org.elasticsearch.xpack.core.ilm.LifecycleSettings; import org.elasticsearch.xpack.ilm.IndexLifecycle; import org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin; +import org.junit.After; import org.junit.Before; import java.util.Collection; @@ -47,6 +50,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME) .put(NetworkModule.HTTP_TYPE_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME) .put(ProfilingPlugin.PROFILING_ENABLED.getKey(), true) + .put(ProfilingPlugin.PROFILING_TEMPLATES_ENABLED.getKey(), false) // .put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial") // Disable ILM history index so that the tests don't have to clean it up .put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING.getKey(), false) @@ -63,12 +67,6 @@ protected boolean ignoreExternalCluster() { return true; } - @Override - protected boolean wipeCluster() { - // we're managing our own indices and the cluster should not be wiped. - return false; - } - private void indexDoc(String index, String id, Map source) { IndexResponse indexResponse = client().prepareIndex(index).setId(id).setSource(source).setCreate(true).get(); assertEquals(RestStatus.CREATED, indexResponse.status()); @@ -96,8 +94,17 @@ protected void waitForIndices() throws Exception { }); } + protected void updateProfilingTemplatesEnabled(boolean newValue) { + ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest(); + request.persistentSettings(Settings.builder().put(ProfilingPlugin.PROFILING_TEMPLATES_ENABLED.getKey(), newValue).build()); + ClusterUpdateSettingsResponse response = client().admin().cluster().updateSettings(request).actionGet(); + assertTrue("Update of profiling templates enabled setting is not acknowledged", response.isAcknowledged()); + } + @Before public void setupData() throws Exception { + // only enable index management while setting up indices to avoid interfering with the rest of the test infrastructure + updateProfilingTemplatesEnabled(true); Collection eventsIndices = useOnlyAllEvents() ? List.of(EventsIndex.FULL_INDEX.getName()) : EventsIndex.indexNames(); waitForIndices(); // indices have replicas @@ -126,4 +133,9 @@ public void setupData() throws Exception { refresh(); } + + @After + public void disable() { + updateProfilingTemplatesEnabled(false); + } } diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java index 13cfe251ff5f8..acb87c8ebf562 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java @@ -70,6 +70,7 @@ public class ProfilingIndexManager implements ClusterStateListener, Closeable { private final Client client; private final ClusterService clusterService; private final ConcurrentMap indexCreationInProgress = new ConcurrentHashMap<>(); + private volatile boolean templatesEnabled; public ProfilingIndexManager(ThreadPool threadPool, Client client, ClusterService clusterService) { this.threadPool = threadPool; @@ -86,8 +87,15 @@ public void close() { clusterService.removeListener(this); } + public void setTemplatesEnabled(boolean templatesEnabled) { + this.templatesEnabled = templatesEnabled; + } + @Override public void clusterChanged(ClusterChangedEvent event) { + if (templatesEnabled == false) { + return; + } // wait for the cluster state to be recovered if (event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { return; diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java index 6017446a6f551..3019c1f5bb674 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java @@ -39,6 +39,8 @@ public class ProfilingIndexTemplateRegistry extends IndexTemplateRegistry { public static final String PROFILING_TEMPLATE_VERSION_VARIABLE = "xpack.profiling.template.version"; + private volatile boolean templatesEnabled; + public ProfilingIndexTemplateRegistry( Settings nodeSettings, ClusterService clusterService, @@ -49,6 +51,10 @@ public ProfilingIndexTemplateRegistry( super(nodeSettings, clusterService, threadPool, client, xContentRegistry); } + public void setTemplatesEnabled(boolean templatesEnabled) { + this.templatesEnabled = templatesEnabled; + } + public void close() { clusterService.removeListener(this); } @@ -71,7 +77,7 @@ protected boolean requiresMasterNode() { @Override protected List getPolicyConfigs() { - return LIFECYCLE_POLICIES; + return templatesEnabled ? LIFECYCLE_POLICIES : Collections.emptyList(); } private static final Map COMPONENT_TEMPLATE_CONFIGS; @@ -130,7 +136,7 @@ protected List getPolicyConfigs() { @Override protected Map getComponentTemplateConfigs() { - return COMPONENT_TEMPLATE_CONFIGS; + return templatesEnabled ? COMPONENT_TEMPLATE_CONFIGS : Collections.emptyMap(); } private static final Map COMPOSABLE_INDEX_TEMPLATE_CONFIGS = parseComposableTemplates( @@ -199,7 +205,7 @@ protected Map getComponentTemplateConfigs() { @Override protected Map getComposableTemplateConfigs() { - return COMPOSABLE_INDEX_TEMPLATE_CONFIGS; + return templatesEnabled ? COMPOSABLE_INDEX_TEMPLATE_CONFIGS : Collections.emptyMap(); } public static boolean isAllTemplatesCreated(ClusterState state) { diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java index 22b5a3caad69e..8d20d027810ce 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java @@ -50,9 +50,17 @@ public class ProfilingPlugin extends Plugin implements ActionPlugin { private static final Logger logger = LogManager.getLogger(ProfilingPlugin.class); public static final Setting PROFILING_ENABLED = Setting.boolSetting( "xpack.profiling.enabled", - false, + true, Setting.Property.NodeScope ); + + public static final Setting PROFILING_TEMPLATES_ENABLED = Setting.boolSetting( + "xpack.profiling.templates.enabled", + // Enable by default iff the profiling plugin is enabled + (settings) -> String.valueOf(PROFILING_ENABLED.get(settings)), + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); public static final String PROFILING_THREAD_POOL_NAME = "profiling"; private final Settings settings; private final boolean enabled; @@ -85,6 +93,9 @@ public Collection createComponents( logger.info("Profiling is {}", enabled ? "enabled" : "disabled"); registry.set(new ProfilingIndexTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry)); indexManager.set(new ProfilingIndexManager(threadPool, client, clusterService)); + // set initial value + updateTemplatesEnabled(PROFILING_TEMPLATES_ENABLED.get(settings)); + clusterService.getClusterSettings().addSettingsUpdateConsumer(PROFILING_TEMPLATES_ENABLED, this::updateTemplatesEnabled); if (enabled) { registry.get().initialize(); indexManager.get().initialize(); @@ -94,6 +105,14 @@ public Collection createComponents( } } + public void updateTemplatesEnabled(boolean newValue) { + if (newValue == false) { + logger.info("profiling index templates will not be installed or reinstalled"); + } + registry.get().setTemplatesEnabled(newValue); + indexManager.get().setTemplatesEnabled(newValue); + } + @Override public List getRestHandlers( final Settings settings, @@ -115,6 +134,7 @@ public List getRestHandlers( public List> getSettings() { return List.of( PROFILING_ENABLED, + PROFILING_TEMPLATES_ENABLED, TransportGetProfilingAction.PROFILING_MAX_STACKTRACE_QUERY_SLICES, TransportGetProfilingAction.PROFILING_MAX_DETAIL_QUERY_SLICES, TransportGetProfilingAction.PROFILING_QUERY_REALTIME diff --git a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexManagerTests.java b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexManagerTests.java index 5fe54c86fa8cc..fea912b1d5e97 100644 --- a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexManagerTests.java +++ b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexManagerTests.java @@ -66,6 +66,7 @@ protected boolean isAllTemplatesCreated(ClusterChangedEvent event) { return templatesCreated.get(); } }; + indexManager.setTemplatesEnabled(true); } @After diff --git a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistryTests.java b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistryTests.java index f433c08537c36..3e8cbe05465ba 100644 --- a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistryTests.java +++ b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistryTests.java @@ -74,6 +74,7 @@ public void createRegistryAndClient() { client = new VerifyingClient(threadPool); clusterService = ClusterServiceUtils.createClusterService(threadPool); registry = new ProfilingIndexTemplateRegistry(Settings.EMPTY, clusterService, threadPool, client, NamedXContentRegistry.EMPTY); + registry.setTemplatesEnabled(true); } @After diff --git a/x-pack/plugin/sql/qa/jdbc/build.gradle b/x-pack/plugin/sql/qa/jdbc/build.gradle index 65d074538361e..b79e3ffff74cd 100644 --- a/x-pack/plugin/sql/qa/jdbc/build.gradle +++ b/x-pack/plugin/sql/qa/jdbc/build.gradle @@ -63,6 +63,7 @@ subprojects { testDistribution = 'DEFAULT' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' } tasks.named("javaRestTest").configure { diff --git a/x-pack/plugin/sql/qa/mixed-node/build.gradle b/x-pack/plugin/sql/qa/mixed-node/build.gradle index 13bf18e6c080d..e05c7a944a1dd 100644 --- a/x-pack/plugin/sql/qa/mixed-node/build.gradle +++ b/x-pack/plugin/sql/qa/mixed-node/build.gradle @@ -28,6 +28,7 @@ BuildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter("7.10.3") && testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' // for debugging purposes diff --git a/x-pack/plugin/sql/qa/server/build.gradle b/x-pack/plugin/sql/qa/server/build.gradle index 56ee4ef971f4f..490a7cbeaeb06 100644 --- a/x-pack/plugin/sql/qa/server/build.gradle +++ b/x-pack/plugin/sql/qa/server/build.gradle @@ -47,6 +47,7 @@ subprojects { testDistribution = 'DEFAULT' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' } diff --git a/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle b/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle index b56d8ff211990..1d5073f13829e 100644 --- a/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle +++ b/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle @@ -11,6 +11,7 @@ def remoteClusterReg = testClusters.register('remote-cluster') { setting 'node.roles', '[data,ingest,master]' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.autoconfiguration.enabled', 'false' @@ -23,6 +24,7 @@ def javaRestTestClusterReg = testClusters.register('javaRestTest') { testDistribution = 'DEFAULT' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'cluster.remote.my_remote_cluster.seeds', { remoteClusterReg.get().getAllTransportPortURI().collect { "\"$it\"" }.toString() } diff --git a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle index 7c0eaf1450567..5fb75b2353870 100644 --- a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle +++ b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle @@ -26,6 +26,7 @@ def remoteCluster = testClusters.register('remote-cluster') { setting 'node.roles', '[data,ingest,master]' setting 'xpack.security.enabled', 'true' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' user username: "test_user", password: "x-pack-test-password" @@ -40,6 +41,7 @@ testClusters.register('mixed-cluster') { nodes."mixed-cluster-1".setting 'node.roles', '[data,ingest,master,transform,remote_cluster_client]' setting 'xpack.security.enabled', 'true' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'cluster.remote.my_remote_cluster.seeds', { remoteCluster.get().getAllTransportPortURI().collect { "\"$it\"" }.toString() diff --git a/x-pack/plugin/watcher/qa/with-monitoring/build.gradle b/x-pack/plugin/watcher/qa/with-monitoring/build.gradle index 6a2ef6fbd7bd5..db2702b86b633 100644 --- a/x-pack/plugin/watcher/qa/with-monitoring/build.gradle +++ b/x-pack/plugin/watcher/qa/with-monitoring/build.gradle @@ -12,6 +12,7 @@ testClusters.configureEach { setting 'xpack.monitoring.collection.enabled', 'true' setting 'xpack.monitoring.collection.interval', '1s' setting 'xpack.watcher.enabled', 'true' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' diff --git a/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java b/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java index 172a776b61242..e680d0c3c39b9 100644 --- a/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java +++ b/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java @@ -37,6 +37,7 @@ public class CoreWithSecurityClientYamlTestSuiteIT extends ESClientYamlSuiteTest .module("x-pack-stack") .setting("xpack.security.enabled", "true") .setting("xpack.watcher.enabled", "false") + .setting("xpack.profiling.enabled", "false") .setting("xpack.ml.enabled", "false") .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.autoconfiguration.enabled", "false") diff --git a/x-pack/qa/mixed-tier-cluster/build.gradle b/x-pack/qa/mixed-tier-cluster/build.gradle index bf05be45e18a0..b546cfec8c2fb 100644 --- a/x-pack/qa/mixed-tier-cluster/build.gradle +++ b/x-pack/qa/mixed-tier-cluster/build.gradle @@ -19,6 +19,7 @@ BuildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter("7.9.0") && testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' nodes."${baseName}-0".setting 'node.roles', '["master"]' diff --git a/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/GlobalCheckpointSyncActionIT.java b/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/GlobalCheckpointSyncActionIT.java index 12cfa8e48b3e4..490f1c3983fc3 100644 --- a/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/GlobalCheckpointSyncActionIT.java +++ b/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/GlobalCheckpointSyncActionIT.java @@ -28,6 +28,7 @@ public class GlobalCheckpointSyncActionIT extends ESRestTestCase { .nodes(2) .setting("xpack.security.enabled", "true") .setting("xpack.watcher.enabled", "false") + .setting("xpack.profiling.enabled", "false") .setting("xpack.ml.enabled", "false") .setting("xpack.license.self_generated.type", "trial") .rolesFile(Resource.fromClasspath("roles.yml")) diff --git a/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/RollupIT.java b/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/RollupIT.java index 91e273f7e34b4..5d607c19ccee3 100644 --- a/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/RollupIT.java +++ b/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/RollupIT.java @@ -43,6 +43,7 @@ public class RollupIT extends ESRestTestCase { .module("x-pack-rollup") .setting("xpack.security.enabled", "true") .setting("xpack.watcher.enabled", "false") + .setting("xpack.profiling.enabled", "false") .setting("xpack.ml.enabled", "false") .setting("xpack.license.self_generated.type", "trial") .user("super-user", "x-pack-super-password") diff --git a/x-pack/qa/rolling-upgrade-basic/build.gradle b/x-pack/qa/rolling-upgrade-basic/build.gradle index a7ea1695c477a..467f7886f00d0 100644 --- a/x-pack/qa/rolling-upgrade-basic/build.gradle +++ b/x-pack/qa/rolling-upgrade-basic/build.gradle @@ -19,6 +19,7 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'basic' } @@ -79,4 +80,4 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> // Security is explicitly disabled, do not run tests in FIPS mode tasks.withType(Test).configureEach { enabled = BuildParams.inFipsJvm == false -} \ No newline at end of file +} diff --git a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle index 969ba23e19254..4f3440bdadd36 100644 --- a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle +++ b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle @@ -25,6 +25,7 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' } diff --git a/x-pack/qa/runtime-fields/with-security/build.gradle b/x-pack/qa/runtime-fields/with-security/build.gradle index e664ee888a313..ce016fa5d3308 100644 --- a/x-pack/qa/runtime-fields/with-security/build.gradle +++ b/x-pack/qa/runtime-fields/with-security/build.gradle @@ -9,6 +9,7 @@ dependencies { testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' rolesFile file('roles.yml') diff --git a/x-pack/qa/smoke-test-security-with-mustache/build.gradle b/x-pack/qa/smoke-test-security-with-mustache/build.gradle index 373e6966df512..02fd97ddbb5a0 100644 --- a/x-pack/qa/smoke-test-security-with-mustache/build.gradle +++ b/x-pack/qa/smoke-test-security-with-mustache/build.gradle @@ -13,6 +13,7 @@ restResources { testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' user username: "test_admin", password: "x-pack-test-password" diff --git a/x-pack/qa/xpack-prefix-rest-compat/build.gradle b/x-pack/qa/xpack-prefix-rest-compat/build.gradle index d609ede0efd2e..465ebf31fb7c4 100644 --- a/x-pack/qa/xpack-prefix-rest-compat/build.gradle +++ b/x-pack/qa/xpack-prefix-rest-compat/build.gradle @@ -82,6 +82,7 @@ testClusters.configureEach { setting 'xpack.ml.enabled', 'true' setting 'xpack.security.enabled', 'true' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.authc.token.enabled', 'true' setting 'xpack.security.authc.api_key.enabled', 'true' setting 'xpack.security.transport.ssl.enabled', 'true' From 8c951e56bc40768b3236c624bab0ac17914be240 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Mon, 8 May 2023 09:53:40 +0200 Subject: [PATCH 12/36] Disable profiling in more tests --- x-pack/plugin/ccr/qa/rest/build.gradle | 1 + x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle | 1 + x-pack/plugin/ilm/qa/multi-cluster/build.gradle | 1 + x-pack/plugin/ilm/qa/rest/build.gradle | 1 + x-pack/plugin/ml/qa/disabled/build.gradle | 1 + x-pack/plugin/security/qa/jwt-realm/build.gradle | 1 + .../xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java | 1 + x-pack/plugin/security/qa/profile/build.gradle | 1 + x-pack/plugin/security/qa/security-basic/build.gradle | 1 + x-pack/plugin/security/qa/security-disabled/build.gradle | 1 + x-pack/plugin/security/qa/security-trial/build.gradle | 1 + x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle | 1 + x-pack/plugin/security/qa/tls-basic/build.gradle | 1 + x-pack/plugin/stack/qa/rest/build.gradle | 1 + x-pack/plugin/watcher/qa/rest/build.gradle | 1 + x-pack/plugin/watcher/qa/with-security/build.gradle | 1 + x-pack/qa/kerberos-tests/build.gradle | 1 + x-pack/qa/reindex-tests-with-security/build.gradle | 1 + x-pack/qa/saml-idp-tests/build.gradle | 1 + x-pack/qa/security-example-spi-extension/build.gradle | 1 + x-pack/qa/security-setup-password-tests/build.gradle | 1 + x-pack/qa/smoke-test-plugins-ssl/build.gradle | 1 + x-pack/qa/third-party/jira/build.gradle | 1 + x-pack/qa/third-party/pagerduty/build.gradle | 1 + x-pack/qa/third-party/slack/build.gradle | 1 + 25 files changed, 25 insertions(+) diff --git a/x-pack/plugin/ccr/qa/rest/build.gradle b/x-pack/plugin/ccr/qa/rest/build.gradle index 5f173b8831df8..7ba9597d9b1ce 100644 --- a/x-pack/plugin/ccr/qa/rest/build.gradle +++ b/x-pack/plugin/ccr/qa/rest/build.gradle @@ -18,6 +18,7 @@ testClusters.configureEach { // that is thrown in production when indexing a document directly in a follower index. jvmArgs '-da:org.elasticsearch.xpack.ccr.index.engine.FollowingEngineAssertions' setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' // TODO: reduce the need for superuser here diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle b/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle index 8008505cb9437..abfb8c5d8ac36 100644 --- a/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle +++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle @@ -11,6 +11,7 @@ def remoteClusterReg = testClusters.register('remote-cluster') { numberOfNodes = 2 setting 'node.roles', '[data,ingest,master]' setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.watcher.enabled', 'false' setting 'xpack.security.enabled', 'true' setting 'xpack.security.autoconfiguration.enabled', 'false' diff --git a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle index 33ce1b0ce76cc..bf54d13bb2c83 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle @@ -26,6 +26,7 @@ testClusters.matching { it.name == 'leader-cluster' }.configureEach { setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.poll_interval', '1000ms' } diff --git a/x-pack/plugin/ilm/qa/rest/build.gradle b/x-pack/plugin/ilm/qa/rest/build.gradle index a17c130d4bbb4..c6e4efb5ee456 100644 --- a/x-pack/plugin/ilm/qa/rest/build.gradle +++ b/x-pack/plugin/ilm/qa/rest/build.gradle @@ -15,6 +15,7 @@ restResources { testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.autoconfiguration.enabled', 'false' } diff --git a/x-pack/plugin/ml/qa/disabled/build.gradle b/x-pack/plugin/ml/qa/disabled/build.gradle index 97a7b0eed73ad..01b67e288f285 100644 --- a/x-pack/plugin/ml/qa/disabled/build.gradle +++ b/x-pack/plugin/ml/qa/disabled/build.gradle @@ -11,6 +11,7 @@ testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' } if (BuildParams.inFipsJvm){ diff --git a/x-pack/plugin/security/qa/jwt-realm/build.gradle b/x-pack/plugin/security/qa/jwt-realm/build.gradle index d561f0c8ceeed..6afb12dcdcae7 100644 --- a/x-pack/plugin/security/qa/jwt-realm/build.gradle +++ b/x-pack/plugin/security/qa/jwt-realm/build.gradle @@ -32,6 +32,7 @@ testClusters.matching { it.name == 'javaRestTest' }.configureEach { extraConfigFile 'rsa.jwkset', file('src/javaRestTest/resources/jwk/rsa-public-jwkset.json') setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' setting 'xpack.security.http.ssl.enabled', 'true' diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java index 47e265b19bde2..d527a30083d58 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java @@ -50,6 +50,7 @@ public class RemoteClusterSecurityBwcRestIT extends AbstractRemoteClusterSecurit .name("fulfilling-cluster") .apply(commonClusterConfig) .setting("xpack.ml.enabled", "false") + .setting("xpack.profiling.enabled", "false") .build(); queryCluster = ElasticsearchCluster.local() diff --git a/x-pack/plugin/security/qa/profile/build.gradle b/x-pack/plugin/security/qa/profile/build.gradle index eae33a5ba5876..4398fb3fb1674 100644 --- a/x-pack/plugin/security/qa/profile/build.gradle +++ b/x-pack/plugin/security/qa/profile/build.gradle @@ -18,6 +18,7 @@ testClusters.matching { it.name == 'javaRestTest' }.configureEach { rolesFile file('src/javaRestTest/resources/roles.yml') setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' diff --git a/x-pack/plugin/security/qa/security-basic/build.gradle b/x-pack/plugin/security/qa/security-basic/build.gradle index 10a8081789652..a98bc715ca18a 100644 --- a/x-pack/plugin/security/qa/security-basic/build.gradle +++ b/x-pack/plugin/security/qa/security-basic/build.gradle @@ -19,6 +19,7 @@ testClusters.configureEach { numberOfNodes = 2 setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'basic' setting 'xpack.security.enabled', 'true' setting 'xpack.security.ssl.diagnose.trust', 'true' diff --git a/x-pack/plugin/security/qa/security-disabled/build.gradle b/x-pack/plugin/security/qa/security-disabled/build.gradle index e3d9c2f4f65ba..36105365f77dc 100644 --- a/x-pack/plugin/security/qa/security-disabled/build.gradle +++ b/x-pack/plugin/security/qa/security-disabled/build.gradle @@ -19,6 +19,7 @@ testClusters.configureEach { numberOfNodes = 2 setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' // We run with a trial license, but explicitly disable security. // This means the security plugin is loaded and all feature are permitted, but they are not enabled setting 'xpack.license.self_generated.type', 'trial' diff --git a/x-pack/plugin/security/qa/security-trial/build.gradle b/x-pack/plugin/security/qa/security-trial/build.gradle index 1b1d5495f1d37..e3c4f21e6b070 100644 --- a/x-pack/plugin/security/qa/security-trial/build.gradle +++ b/x-pack/plugin/security/qa/security-trial/build.gradle @@ -15,6 +15,7 @@ testClusters.matching { it.name == 'javaRestTest' }.configureEach { numberOfNodes = 2 setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' setting 'xpack.security.ssl.diagnose.trust', 'true' diff --git a/x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle b/x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle index ca33b61de4ca5..c6bc42911bf6c 100644 --- a/x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle +++ b/x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle @@ -26,6 +26,7 @@ testClusters.matching { it.name == 'javaRestTest' }.configureEach { extraConfigFile 'oidc-jwkset.json', file('src/javaRestTest/resources/oidc-jwkset.json') setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' setting 'xpack.security.authc.api_key.enabled', 'true' diff --git a/x-pack/plugin/security/qa/tls-basic/build.gradle b/x-pack/plugin/security/qa/tls-basic/build.gradle index fbe91009011e3..327a99b94a4fd 100644 --- a/x-pack/plugin/security/qa/tls-basic/build.gradle +++ b/x-pack/plugin/security/qa/tls-basic/build.gradle @@ -23,6 +23,7 @@ testClusters.matching { it.name == 'javaRestTest' }.configureEach { extraConfigFile 'ca.crt', file('src/javaRestTest/resources/ssl/ca.crt') setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'basic' setting 'xpack.security.enabled', 'true' setting 'xpack.security.http.ssl.enabled', 'true' diff --git a/x-pack/plugin/stack/qa/rest/build.gradle b/x-pack/plugin/stack/qa/rest/build.gradle index a5bdb388d610f..78e1d9a293935 100644 --- a/x-pack/plugin/stack/qa/rest/build.gradle +++ b/x-pack/plugin/stack/qa/rest/build.gradle @@ -15,5 +15,6 @@ restResources { testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' } diff --git a/x-pack/plugin/watcher/qa/rest/build.gradle b/x-pack/plugin/watcher/qa/rest/build.gradle index 3f61bdcb3c2ed..c2ccb61333b3f 100644 --- a/x-pack/plugin/watcher/qa/rest/build.gradle +++ b/x-pack/plugin/watcher/qa/rest/build.gradle @@ -25,6 +25,7 @@ testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG' } diff --git a/x-pack/plugin/watcher/qa/with-security/build.gradle b/x-pack/plugin/watcher/qa/with-security/build.gradle index 5c01573e09417..c4011f716ade7 100644 --- a/x-pack/plugin/watcher/qa/with-security/build.gradle +++ b/x-pack/plugin/watcher/qa/with-security/build.gradle @@ -22,6 +22,7 @@ restResources { testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'true' // settings to test settings filtering on setting 'xpack.notification.email.account._email.smtp.host', 'host.domain' diff --git a/x-pack/qa/kerberos-tests/build.gradle b/x-pack/qa/kerberos-tests/build.gradle index 536758dcfb71f..1300cbfc373dc 100644 --- a/x-pack/qa/kerberos-tests/build.gradle +++ b/x-pack/qa/kerberos-tests/build.gradle @@ -21,6 +21,7 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { setting 'xpack.security.enabled', 'true' setting 'xpack.security.authc.realms.file.file1.order', '0' setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.audit.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' // Kerberos realm diff --git a/x-pack/qa/reindex-tests-with-security/build.gradle b/x-pack/qa/reindex-tests-with-security/build.gradle index fdb61e02f8186..4af80e850a952 100644 --- a/x-pack/qa/reindex-tests-with-security/build.gradle +++ b/x-pack/qa/reindex-tests-with-security/build.gradle @@ -33,6 +33,7 @@ testClusters.matching { it.name == "yamlRestTest" }.configureEach { setting 'reindex.remote.whitelist', '127.0.0.1:*' setting 'xpack.security.enabled', 'true' setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.http.ssl.enabled', 'true' setting 'xpack.security.http.ssl.certificate', 'http.crt' diff --git a/x-pack/qa/saml-idp-tests/build.gradle b/x-pack/qa/saml-idp-tests/build.gradle index 859977f7b337c..8efe8840e00c5 100644 --- a/x-pack/qa/saml-idp-tests/build.gradle +++ b/x-pack/qa/saml-idp-tests/build.gradle @@ -81,6 +81,7 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { setting 'xpack.security.authc.realms.native.native.order', '4' setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'logger.org.elasticsearch.xpack.security', 'TRACE' extraConfigFile 'idp-metadata.xml', file(outputDir + "/idp-metadata.xml") diff --git a/x-pack/qa/security-example-spi-extension/build.gradle b/x-pack/qa/security-example-spi-extension/build.gradle index 3fb73d615b8c9..1bccd2c2936cf 100644 --- a/x-pack/qa/security-example-spi-extension/build.gradle +++ b/x-pack/qa/security-example-spi-extension/build.gradle @@ -33,6 +33,7 @@ testClusters.configureEach { setting 'xpack.security.authc.realms.custom_role_mapping.role_map.order', '3' setting 'xpack.security.enabled', 'true' setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' user username: "test_user", password: "x-pack-test-password" diff --git a/x-pack/qa/security-setup-password-tests/build.gradle b/x-pack/qa/security-setup-password-tests/build.gradle index 8f17e6d92291a..c4bdb32225980 100644 --- a/x-pack/qa/security-setup-password-tests/build.gradle +++ b/x-pack/qa/security-setup-password-tests/build.gradle @@ -13,6 +13,7 @@ tasks.named("javaRestTest").configure { testClusters.matching { it.name == "javaRestTest" }.configureEach { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'true' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' // Setup passwords doesn't work when there is a password auto-configured on first start setting 'xpack.security.autoconfiguration.enabled', 'false' diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle index 1e8e88b91c330..8a32db00f3fbe 100644 --- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle +++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle @@ -60,6 +60,7 @@ testClusters.matching { it.name == "yamlRestTest" }.configureEach { keystore 'xpack.security.http.ssl.secure_key_passphrase', 'testnode' setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' // copy keystores, keys and certificates into config/ extraConfigFile nodeKeystore.name, nodeKeystore extraConfigFile nodeKey.name, nodeKey diff --git a/x-pack/qa/third-party/jira/build.gradle b/x-pack/qa/third-party/jira/build.gradle index b7268af807535..ce16072bfe778 100644 --- a/x-pack/qa/third-party/jira/build.gradle +++ b/x-pack/qa/third-party/jira/build.gradle @@ -43,6 +43,7 @@ if (!jiraUrl && !jiraUser && !jiraPassword && !jiraProject) { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG' setting 'xpack.notification.jira.account.test.issue_defaults.issuetype.name', 'Bug' diff --git a/x-pack/qa/third-party/pagerduty/build.gradle b/x-pack/qa/third-party/pagerduty/build.gradle index 4b5a0bbeeeb4a..8f21a7a3f34cf 100644 --- a/x-pack/qa/third-party/pagerduty/build.gradle +++ b/x-pack/qa/third-party/pagerduty/build.gradle @@ -22,6 +22,7 @@ if (!pagerDutyServiceKey) { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG' keystore 'xpack.notification.pagerduty.account.test_account.secure_service_api_key', pagerDutyServiceKey diff --git a/x-pack/qa/third-party/slack/build.gradle b/x-pack/qa/third-party/slack/build.gradle index b2b0478da0471..9b3e839156d57 100644 --- a/x-pack/qa/third-party/slack/build.gradle +++ b/x-pack/qa/third-party/slack/build.gradle @@ -22,6 +22,7 @@ if (!slackUrl) { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG' keystore 'xpack.notification.slack.account.test_account.secure_url', slackUrl From 942849257b8f414ebc2c2de22a6d6a47cbcb3794 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Mon, 8 May 2023 10:07:06 +0200 Subject: [PATCH 13/36] Disable profiling in more tests --- plugins/mapper-murmur3/build.gradle | 1 + x-pack/plugin/ccr/qa/restart/build.gradle | 2 ++ x-pack/plugin/ccr/qa/security/build.gradle | 2 ++ .../org/elasticsearch/xpack/CcrIntegTestCase.java | 1 + .../elasticsearch/xpack/CcrSingleNodeTestCase.java | 1 + .../org/elasticsearch/xpack/core/XPackSettings.java | 8 ++++++++ .../license/LicensesManagerServiceTests.java | 1 + .../qa/rest-with-advanced-security/build.gradle | 1 + .../plugin/enrich/qa/rest-with-security/build.gradle | 1 + x-pack/plugin/enrich/qa/rest/build.gradle | 1 + .../xpack/eql/action/AbstractEqlIntegTestCase.java | 1 + .../ilm/ClusterStateWaitThresholdBreachTests.java | 1 + .../xpack/ilm/DataAndIndexLifecycleMixingTests.java | 1 + .../xpack/ilm/DataTiersMigrationsTests.java | 1 + .../xpack/ilm/IndexLifecycleInitialisationTests.java | 1 + .../slm/SnapshotLifecycleInitialisationTests.java | 1 + .../xpack/ml/integration/MlNativeIntegTestCase.java | 1 + .../xpack/ml/support/BaseMlIntegTestCase.java | 1 + .../xpack/monitoring/integration/MonitoringIT.java | 1 + .../TransportMonitoringMigrateAlertsActionTests.java | 1 + .../exporter/local/LocalExporterIntegTestCase.java | 1 + .../xpack/profiler/ProfilingTestCase.java | 3 ++- .../xpack/profiler/ProfilingPlugin.java | 12 +++--------- .../sql/action/AbstractSqlBlockingIntegTestCase.java | 1 + .../xpack/sql/action/AbstractSqlIntegTestCase.java | 1 + 25 files changed, 37 insertions(+), 10 deletions(-) diff --git a/plugins/mapper-murmur3/build.gradle b/plugins/mapper-murmur3/build.gradle index 7e93e6d295397..e53f6eee0e022 100644 --- a/plugins/mapper-murmur3/build.gradle +++ b/plugins/mapper-murmur3/build.gradle @@ -36,4 +36,5 @@ restResources { testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' } diff --git a/x-pack/plugin/ccr/qa/restart/build.gradle b/x-pack/plugin/ccr/qa/restart/build.gradle index d354cd911f2f8..9ed3c5cbb10cf 100644 --- a/x-pack/plugin/ccr/qa/restart/build.gradle +++ b/x-pack/plugin/ccr/qa/restart/build.gradle @@ -12,6 +12,7 @@ def leaderCluster = testClusters.register('leader-cluster') { testDistribution = 'DEFAULT' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' + setting 'xpack.profiling.enabled', 'false' user username: 'admin', password: 'admin-password', role: 'superuser' } @@ -20,6 +21,7 @@ def followCluster = testClusters.register('follow-cluster') { setting 'xpack.monitoring.collection.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' + setting 'xpack.profiling.enabled', 'false' user username: 'admin', password: 'admin-password', role: 'superuser' setting 'cluster.remote.leader_cluster.seeds', { "\"${leaderCluster.get().getAllTransportPortURI().get(0)}\"" } diff --git a/x-pack/plugin/ccr/qa/security/build.gradle b/x-pack/plugin/ccr/qa/security/build.gradle index ff4a4f857fe32..02b8364b6f250 100644 --- a/x-pack/plugin/ccr/qa/security/build.gradle +++ b/x-pack/plugin/ccr/qa/security/build.gradle @@ -13,6 +13,7 @@ def leadCluster = testClusters.register('leader-cluster') { testDistribution = 'DEFAULT' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' + setting 'xpack.profiling.enabled', 'false' extraConfigFile 'roles.yml', file('leader-roles.yml') user username: "test_admin", role: "superuser" user username: "test_ccr", role: "ccruser" @@ -25,6 +26,7 @@ testClusters.register('follow-cluster') { } setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.monitoring.collection.enabled', 'false' // will be enabled by tests extraConfigFile 'roles.yml', file('follower-roles.yml') user username: "test_admin", role: "superuser" diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java index d09b27cb06a58..141e569738de2 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java @@ -304,6 +304,7 @@ private NodeConfigurationSource createNodeConfigurationSource(final String leade builder.put(XPackSettings.SECURITY_ENABLED.getKey(), false); builder.put(XPackSettings.WATCHER_ENABLED.getKey(), false); builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); + builder.put(XPackSettings.PROFILING_ENABLED.getKey(), false); builder.put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial"); // Let cluster state api return quickly in order to speed up auto follow tests: builder.put(CcrSettings.CCR_WAIT_FOR_METADATA_TIMEOUT.getKey(), TimeValue.timeValueMillis(100)); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java index a540631dc040e..d043b58094fe2 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java @@ -48,6 +48,7 @@ protected Settings nodeSettings() { builder.put(XPackSettings.SECURITY_ENABLED.getKey(), false); builder.put(XPackSettings.WATCHER_ENABLED.getKey(), false); builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); + builder.put(XPackSettings.PROFILING_ENABLED.getKey(), false); builder.put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial"); // Let cluster state api return quickly in order to speed up auto follow tests: builder.put(CcrSettings.CCR_WAIT_FOR_METADATA_TIMEOUT.getKey(), TimeValue.timeValueMillis(100)); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java index 8e543f4466a8b..3f785eb6f2a04 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java @@ -93,6 +93,13 @@ public Iterator> settings() { Setting.Property.NodeScope ); + /** Setting for enabling or disabling universal profiling. Defaults to true. */ + public static final Setting PROFILING_ENABLED = Setting.boolSetting( + "xpack.profiling.enabled", + true, + Setting.Property.NodeScope + ); + /** Setting for enabling or disabling enterprise search. Defaults to true. */ public static final Setting ENTERPRISE_SEARCH_ENABLED = Setting.boolSetting( "xpack.ent_search.enabled", @@ -318,6 +325,7 @@ public static List> getAllSettings() { settings.add(SECURITY_ENABLED); settings.add(GRAPH_ENABLED); settings.add(MACHINE_LEARNING_ENABLED); + settings.add(PROFILING_ENABLED); settings.add(ENTERPRISE_SEARCH_ENABLED); settings.add(AUDIT_ENABLED); settings.add(WATCHER_ENABLED); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java index d545bef0e1e7a..261b9d10a47dd 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java @@ -41,6 +41,7 @@ protected Settings nodeSettings() { .put(XPackSettings.WATCHER_ENABLED.getKey(), false) .put(XPackSettings.GRAPH_ENABLED.getKey(), false) .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) + .put(XPackSettings.PROFILING_ENABLED.getKey(), false) .build(); } diff --git a/x-pack/plugin/enrich/qa/rest-with-advanced-security/build.gradle b/x-pack/plugin/enrich/qa/rest-with-advanced-security/build.gradle index 2e649e718b081..eddaf8ca5208e 100644 --- a/x-pack/plugin/enrich/qa/rest-with-advanced-security/build.gradle +++ b/x-pack/plugin/enrich/qa/rest-with-advanced-security/build.gradle @@ -14,5 +14,6 @@ testClusters.configureEach { user username: "test_enrich", password: "x-pack-test-password", role: "integ_test_role" setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.monitoring.collection.enabled', 'true' } diff --git a/x-pack/plugin/enrich/qa/rest-with-security/build.gradle b/x-pack/plugin/enrich/qa/rest-with-security/build.gradle index 69fec4ad32c74..f51349e54e0e1 100644 --- a/x-pack/plugin/enrich/qa/rest-with-security/build.gradle +++ b/x-pack/plugin/enrich/qa/rest-with-security/build.gradle @@ -19,5 +19,6 @@ testClusters.configureEach { user username: "test_enrich_no_privs", password: "x-pack-test-password", role: "enrich_no_privs" setting 'xpack.license.self_generated.type', 'basic' setting 'xpack.security.enabled', 'true' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.monitoring.collection.enabled', 'true' } diff --git a/x-pack/plugin/enrich/qa/rest/build.gradle b/x-pack/plugin/enrich/qa/rest/build.gradle index e8473d15ed9ef..c9feca4a91247 100644 --- a/x-pack/plugin/enrich/qa/rest/build.gradle +++ b/x-pack/plugin/enrich/qa/rest/build.gradle @@ -30,5 +30,6 @@ testClusters.configureEach { setting 'xpack.license.self_generated.type', 'basic' setting 'xpack.monitoring.collection.enabled', 'true' setting 'xpack.security.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.4.0") } diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java index 329d0e3d0a049..d7f3460631cf7 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java @@ -28,6 +28,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); + settings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); settings.put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial"); return settings.build(); } diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java index 02f9107d12b6e..0720ab166f908 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java @@ -75,6 +75,7 @@ protected Collection> nodePlugins() { protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); + settings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataAndIndexLifecycleMixingTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataAndIndexLifecycleMixingTests.java index 8ecd36fbd6139..a1b507f5437e0 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataAndIndexLifecycleMixingTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataAndIndexLifecycleMixingTests.java @@ -88,6 +88,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { settings.put(DataLifecycleService.DLM_POLL_INTERVAL, "1s"); settings.put(DataLifecycle.CLUSTER_DLM_DEFAULT_ROLLOVER_SETTING.getKey(), "min_docs=1,max_docs=1"); settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); + settings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java index 0fb550e250312..480e6356a0e54 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java @@ -69,6 +69,7 @@ protected Collection> nodePlugins() { protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); + settings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java index 5696abe2f651c..70170001da82c 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java @@ -99,6 +99,7 @@ public class IndexLifecycleInitialisationTests extends ESIntegTestCase { protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { Settings.Builder nodeSettings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); nodeSettings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); + nodeSettings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); nodeSettings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); nodeSettings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); nodeSettings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java index fa312cdac8e14..2344c28ed90bf 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java @@ -46,6 +46,7 @@ public static void setupRepositoryPath() { protected Settings nodeSettings() { Settings.Builder settings = Settings.builder().put(super.nodeSettings()); settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); + settings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java index cda03265fec28..1fc09cacd4c73 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java @@ -187,6 +187,7 @@ protected Settings externalClusterClientSettings() { builder.putList("node.roles", Collections.emptyList()); builder.put(NetworkModule.TRANSPORT_TYPE_KEY, SecurityField.NAME4); builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), true); + builder.put(XPackSettings.PROFILING_ENABLED.getKey(), false); builder.put(XPackSettings.SECURITY_ENABLED.getKey(), true); builder.put(MachineLearningField.AUTODETECT_PROCESS.getKey(), false); builder.put(XPackSettings.WATCHER_ENABLED.getKey(), false); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java index 8987df1ece6b9..51711e6324a45 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java @@ -130,6 +130,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); settings.put(MachineLearningField.AUTODETECT_PROCESS.getKey(), false); settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), true); + settings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); settings.put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial"); settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); diff --git a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java index 0b51e72846531..a96ced9a3fb14 100644 --- a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java +++ b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java @@ -83,6 +83,7 @@ protected Settings nodeSettings() { return Settings.builder() .put(super.nodeSettings()) .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) + .put(XPackSettings.PROFILING_ENABLED.getKey(), false) .put("xpack.monitoring.collection.interval", MonitoringService.MIN_INTERVAL) .put("xpack.monitoring.exporters._local.type", "local") .put("xpack.monitoring.exporters._local.enabled", false) diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java index 9d47fc3e973b3..e4ce67e15e7c5 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java @@ -91,6 +91,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { // X-Pack configuration .put("xpack.license.self_generated.type", "trial") .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) + .put(XPackSettings.PROFILING_ENABLED.getKey(), false) .build(); } diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java index f717dfe50639d..4bee07f3f5701 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java @@ -48,6 +48,7 @@ protected Settings localExporterSettings() { .put("xpack.monitoring.exporters." + exporterName + ".enabled", false) .put("xpack.monitoring.exporters." + exporterName + ".cluster_alerts.management.enabled", false) .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) + .put(XPackSettings.PROFILING_ENABLED.getKey(), false) .build(); } diff --git a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java index 17c34412e4b7c..00380eba8f073 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java +++ b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java @@ -19,6 +19,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.transport.netty4.Netty4Plugin; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.ilm.LifecycleSettings; import org.elasticsearch.xpack.ilm.IndexLifecycle; import org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin; @@ -49,7 +50,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .put(super.nodeSettings(nodeOrdinal, otherSettings)) .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME) .put(NetworkModule.HTTP_TYPE_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME) - .put(ProfilingPlugin.PROFILING_ENABLED.getKey(), true) + .put(XPackSettings.PROFILING_ENABLED.getKey(), true) .put(ProfilingPlugin.PROFILING_TEMPLATES_ENABLED.getKey(), false) // .put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial") // Disable ILM history index so that the tests don't have to clean it up diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java index 8d20d027810ce..9a10f76dfcff0 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java @@ -38,6 +38,7 @@ import org.elasticsearch.tracing.Tracer; import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xpack.core.XPackSettings; import java.util.Collection; import java.util.Collections; @@ -48,16 +49,10 @@ public class ProfilingPlugin extends Plugin implements ActionPlugin { private static final Logger logger = LogManager.getLogger(ProfilingPlugin.class); - public static final Setting PROFILING_ENABLED = Setting.boolSetting( - "xpack.profiling.enabled", - true, - Setting.Property.NodeScope - ); - public static final Setting PROFILING_TEMPLATES_ENABLED = Setting.boolSetting( "xpack.profiling.templates.enabled", // Enable by default iff the profiling plugin is enabled - (settings) -> String.valueOf(PROFILING_ENABLED.get(settings)), + (settings) -> String.valueOf(XPackSettings.PROFILING_ENABLED.get(settings)), Setting.Property.NodeScope, Setting.Property.Dynamic ); @@ -71,7 +66,7 @@ public class ProfilingPlugin extends Plugin implements ActionPlugin { public ProfilingPlugin(Settings settings) { this.settings = settings; - this.enabled = PROFILING_ENABLED.get(settings); + this.enabled = XPackSettings.PROFILING_ENABLED.get(settings); } @Override @@ -133,7 +128,6 @@ public List getRestHandlers( @Override public List> getSettings() { return List.of( - PROFILING_ENABLED, PROFILING_TEMPLATES_ENABLED, TransportGetProfilingAction.PROFILING_MAX_STACKTRACE_QUERY_SLICES, TransportGetProfilingAction.PROFILING_MAX_DETAIL_QUERY_SLICES, diff --git a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/AbstractSqlBlockingIntegTestCase.java b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/AbstractSqlBlockingIntegTestCase.java index 4ac4827c9d520..6e192af3eac93 100644 --- a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/AbstractSqlBlockingIntegTestCase.java +++ b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/AbstractSqlBlockingIntegTestCase.java @@ -61,6 +61,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); + settings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); settings.put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial"); return settings.build(); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/AbstractSqlIntegTestCase.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/AbstractSqlIntegTestCase.java index 4f7646127cf38..e5c770925e91f 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/AbstractSqlIntegTestCase.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/AbstractSqlIntegTestCase.java @@ -27,6 +27,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); + settings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); settings.put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial"); return settings.build(); } From 20d2e18c1bd102aac08b22c187a9a216547d2b16 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Mon, 8 May 2023 10:53:58 +0200 Subject: [PATCH 14/36] Tweak tests, use auto expand replicas --- docs/build.gradle | 1 + modules/data-streams/build.gradle | 2 ++ modules/dlm/build.gradle | 1 + .../main/java/org/elasticsearch/test/rest/ESRestTestCase.java | 1 + x-pack/docs/build.gradle | 2 ++ x-pack/plugin/async-search/qa/rest/build.gradle | 1 + x-pack/plugin/async-search/qa/security/build.gradle | 1 + x-pack/plugin/autoscaling/qa/rest/build.gradle | 1 + x-pack/plugin/core/build.gradle | 2 ++ .../xpack/profiler/component-template/profiling-events.json | 2 ++ .../profiler/component-template/profiling-executables.json | 2 ++ .../xpack/profiler/component-template/profiling-metrics.json | 2 ++ .../profiler/component-template/profiling-stackframes.json | 2 ++ .../profiler/component-template/profiling-stacktraces.json | 2 ++ .../xpack/profiler/index-template/.profiling-ilm-lock.json | 2 ++ .../profiler/index-template/profiling-returnpads-private.json | 2 ++ .../xpack/profiler/index-template/profiling-sq-executables.json | 2 ++ .../xpack/profiler/index-template/profiling-sq-leafframes.json | 2 ++ .../xpack/profiler/index-template/profiling-symbols.json | 2 ++ .../xpack/monitoring/exporter/http/HttpExporterIT.java | 1 + .../xpack/monitoring/test/MonitoringIntegTestCase.java | 1 + .../xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java | 1 - .../xpack/watcher/test/AbstractWatcherIntegrationTestCase.java | 1 + 23 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/build.gradle b/docs/build.gradle index 927419c6b3a69..87eb4af808d8f 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -61,6 +61,7 @@ restResources { testClusters.matching { it.name == "yamlRestTest"}.configureEach { if (singleNode().testDistribution == DEFAULT) { setting 'xpack.license.self_generated.type', 'trial' + setting 'xpack.profiling.templates.enabled', 'false' setting 'indices.lifecycle.history_index_enabled', 'false' keystorePassword 'keystore-password' } diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index 39d96c1955ab1..dff5b4a536e77 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -22,6 +22,7 @@ restResources { testClusters.configureEach { module ':modules:reindex' testDistribution = 'DEFAULT' + setting 'xpack.profiling.templates.enabled', 'false' // disable ILM history, since it disturbs tests using _all setting 'indices.lifecycle.history_index_enabled', 'false' setting 'xpack.security.enabled', 'true' @@ -33,6 +34,7 @@ testClusters.configureEach { testClusters.matching { it.name == "javaRestTest" }.configureEach { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' + setting 'xpack.profiling.templates.enabled', 'false' // disable ILM history, since it disturbs tests using _all setting 'indices.lifecycle.history_index_enabled', 'false' } diff --git a/modules/dlm/build.gradle b/modules/dlm/build.gradle index 20d738eabc03a..74dce2c7deed2 100644 --- a/modules/dlm/build.gradle +++ b/modules/dlm/build.gradle @@ -28,6 +28,7 @@ addQaCheckDependencies(project) testClusters.configureEach { module ':modules:reindex' testDistribution = 'DEFAULT' + setting 'xpack.profiling.templates.enabled', 'false' // disable ILM history, since it disturbs tests using _all setting 'indices.lifecycle.history_index_enabled', 'false' setting 'xpack.security.enabled', 'true' diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 40522422d6ba2..7f7649d94b49c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -621,6 +621,7 @@ protected Set preserveILMPolicyIds() { "ml-size-based-ilm-policy", "logs", "metrics", + "profiling", "synthetics", "7-days-default", "30-days-default", diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index c834689a0d873..2cda8ded3887d 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -46,6 +46,8 @@ testClusters.matching { it.name == "yamlRestTest" }.configureEach { setting 'xpack.security.enabled', 'true' setting 'xpack.security.authc.api_key.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' + // disabling universal profiling template management for doc tests to avoid potential lingering tasks that'd cause test flakiness + setting 'xpack.profiling.templates.enabled', 'false' // disable the ILM history for doc tests to avoid potential lingering tasks that'd cause test flakiness setting 'indices.lifecycle.history_index_enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' diff --git a/x-pack/plugin/async-search/qa/rest/build.gradle b/x-pack/plugin/async-search/qa/rest/build.gradle index b41600407be5c..79dd0c4b71e2a 100644 --- a/x-pack/plugin/async-search/qa/rest/build.gradle +++ b/x-pack/plugin/async-search/qa/rest/build.gradle @@ -18,6 +18,7 @@ restResources { testClusters.configureEach { testDistribution = 'DEFAULT' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'false' } diff --git a/x-pack/plugin/async-search/qa/security/build.gradle b/x-pack/plugin/async-search/qa/security/build.gradle index 3e29a711b0a8c..f464f03b04216 100644 --- a/x-pack/plugin/async-search/qa/security/build.gradle +++ b/x-pack/plugin/async-search/qa/security/build.gradle @@ -11,6 +11,7 @@ testClusters.configureEach { testDistribution = 'DEFAULT' numberOfNodes = 2 setting 'xpack.license.self_generated.type', 'trial' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'true' extraConfigFile 'roles.yml', file('roles.yml') user username: "test_kibana_user", password: "x-pack-test-password", role: "kibana_system" diff --git a/x-pack/plugin/autoscaling/qa/rest/build.gradle b/x-pack/plugin/autoscaling/qa/rest/build.gradle index 19254880a7089..c3c40bb42a101 100644 --- a/x-pack/plugin/autoscaling/qa/rest/build.gradle +++ b/x-pack/plugin/autoscaling/qa/rest/build.gradle @@ -21,6 +21,7 @@ tasks.named("yamlRestTestV7CompatTest").configure { testClusters.configureEach { testDistribution = 'DEFAULT' + setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' extraConfigFile 'roles.yml', file('autoscaling-roles.yml') diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle index e09c82fa27e44..e0caf1bde3eed 100644 --- a/x-pack/plugin/core/build.gradle +++ b/x-pack/plugin/core/build.gradle @@ -139,6 +139,8 @@ testClusters.configureEach { testDistribution = 'default' setting 'xpack.security.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' + // disabling universal profiling template management as it disturbs testDSXpackUsage test + setting 'xpack.profiling.templates.enabled', 'false' //disabling ILM history as it disturbs testDSXpackUsage test setting 'indices.lifecycle.history_index_enabled', 'false' keystore 'bootstrap.password', 'x-pack-test-password' diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json index 7d5d0c061ad02..e952e24bd5358 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json @@ -3,6 +3,8 @@ "settings": { "index": { "number_of_shards": 4, + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", "max_result_window": 150000, "refresh_interval": "10s", "sort": { diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json index 5f31cd8b186cb..2fc9774200f47 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json @@ -2,6 +2,8 @@ "template": { "settings": { "index": { + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", "refresh_interval": "10s" } }, diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-metrics.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-metrics.json index a079026e5b50c..2a4895b2fea0f 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-metrics.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-metrics.json @@ -2,6 +2,8 @@ "template": { "settings": { "index": { + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", "refresh_interval": "10s", "sort": { "field": [ diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json index 6f309f353ef9a..a238c7805b549 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json @@ -3,6 +3,8 @@ "settings": { "index": { "number_of_shards": 16, + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", "refresh_interval": "10s" } }, diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json index f4da624d6c7d8..7f08399e12fe9 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json @@ -3,6 +3,8 @@ "settings": { "index": { "number_of_shards": 16, + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", "refresh_interval": "10s", "sort": { "field": [ diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/.profiling-ilm-lock.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/.profiling-ilm-lock.json index 69a93342f22c8..84f6c013f6356 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/.profiling-ilm-lock.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/.profiling-ilm-lock.json @@ -5,6 +5,8 @@ "template": { "settings": { "index": { + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", "hidden": true } }, diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json index ae2691528d06e..dc2508e2853c9 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json @@ -5,6 +5,8 @@ "template": { "settings": { "index": { + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", "refresh_interval": "10s", "hidden": true } diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json index 35ae771907fd4..602cddb665391 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json @@ -5,6 +5,8 @@ "template": { "settings": { "index": { + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", "refresh_interval": "10s", "hidden": true } diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json index 0491d75ac0153..2749d2746408d 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json @@ -5,6 +5,8 @@ "template": { "settings": { "index": { + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", "refresh_interval": "10s", "hidden": true } diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json index 0c93845e0ac02..e266cb4e59896 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json @@ -6,6 +6,8 @@ "settings": { "index": { "number_of_shards": "16", + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", "refresh_interval": "10s", "hidden": true } diff --git a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java index e2e2cada89b48..f1062d1054a85 100644 --- a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java +++ b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java @@ -111,6 +111,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .put(MonitoringService.INTERVAL.getKey(), MonitoringService.MIN_INTERVAL) // we do this by default in core, but for monitoring this isn't needed and only adds noise. .put("indices.lifecycle.history_index_enabled", false) + .put("xpack.profiling.templates.enabled", false) .put("index.store.mock.check_index_on_close", false); return builder.build(); } diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java index bb9a7b097f032..1d3544f837f34 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java @@ -55,6 +55,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { // .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) // we do this by default in core, but for monitoring this isn't needed and only adds noise. .put("indices.lifecycle.history_index_enabled", false) + .put("xpack.profiling.templates.enabled", false) .put("index.store.mock.check_index_on_close", false); return builder.build(); diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java index d527a30083d58..47e265b19bde2 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java @@ -50,7 +50,6 @@ public class RemoteClusterSecurityBwcRestIT extends AbstractRemoteClusterSecurit .name("fulfilling-cluster") .apply(commonClusterConfig) .setting("xpack.ml.enabled", "false") - .setting("xpack.profiling.enabled", "false") .build(); queryCluster = ElasticsearchCluster.local() diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java index 8755ee282cb5e..02bd510425b1c 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -117,6 +117,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .put("xpack.watcher.execution.scroll.size", randomIntBetween(1, 100)) .put("xpack.watcher.watch.scroll.size", randomIntBetween(1, 100)) .put("indices.lifecycle.history_index_enabled", false) + .put("xpack.profiling.templates.enabled", false) .build(); } From a551d499dbca7c634447963628bee3a1bb186d0e Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Mon, 8 May 2023 11:46:01 +0200 Subject: [PATCH 15/36] Ignore profiling templates in REST tests --- .../java/org/elasticsearch/test/rest/ESRestTestCase.java | 5 ++++- .../xpack/security/EnableSecurityOnBasicLicenseIT.java | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 7f7649d94b49c..05356e9269e99 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -100,7 +100,6 @@ import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; - import javax.net.ssl.SSLContext; import static java.util.Collections.sort; @@ -1815,11 +1814,15 @@ protected static boolean isXPackTemplate(String name) { if (name.startsWith("behavioral_analytics-")) { return true; } + if (name.startsWith("profiling-")) { + return true; + } switch (name) { case ".watches": case "security_audit_log": case ".slm-history": case ".async-search": + case ".profiling-ilm-lock": // TODO: Remove after switch to K/V indices case "saml-service-provider": case "logs": case "logs-settings": diff --git a/x-pack/plugin/security/qa/basic-enable-security/src/javaRestTest/java/org/elasticsearch/xpack/security/EnableSecurityOnBasicLicenseIT.java b/x-pack/plugin/security/qa/basic-enable-security/src/javaRestTest/java/org/elasticsearch/xpack/security/EnableSecurityOnBasicLicenseIT.java index ec2a3e2b01803..cba3c205e8e34 100644 --- a/x-pack/plugin/security/qa/basic-enable-security/src/javaRestTest/java/org/elasticsearch/xpack/security/EnableSecurityOnBasicLicenseIT.java +++ b/x-pack/plugin/security/qa/basic-enable-security/src/javaRestTest/java/org/elasticsearch/xpack/security/EnableSecurityOnBasicLicenseIT.java @@ -46,6 +46,7 @@ public class EnableSecurityOnBasicLicenseIT extends ESRestTestCase { private static MutableSettingsProvider clusterSettings = new MutableSettingsProvider() { { put("xpack.ml.enabled", "false"); + put("xpack.profiling.enabled", "false"); put("xpack.security.enabled", "false"); put("xpack.license.self_generated.type", "basic"); } From 078ea02d0e4ef63d51ede5a7a46814d2bb3d5c36 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Mon, 8 May 2023 11:52:28 +0200 Subject: [PATCH 16/36] Make the linter happy --- .../main/java/org/elasticsearch/test/rest/ESRestTestCase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 05356e9269e99..57c2ab1c64170 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -100,6 +100,7 @@ import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; + import javax.net.ssl.SSLContext; import static java.util.Collections.sort; From 74f4fcaa3ce2674d6199061bfbe6b6375fae76e8 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Mon, 8 May 2023 11:54:28 +0200 Subject: [PATCH 17/36] Take advantage of auto expand replicas in test --- .../org/elasticsearch/xpack/profiler/ProfilingTestCase.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java index 00380eba8f073..c12a234bed4d1 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java +++ b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java @@ -108,8 +108,7 @@ public void setupData() throws Exception { updateProfilingTemplatesEnabled(true); Collection eventsIndices = useOnlyAllEvents() ? List.of(EventsIndex.FULL_INDEX.getName()) : EventsIndex.indexNames(); waitForIndices(); - // indices have replicas - ensureYellow(); + ensureGreen(); // ensure that we have this in every index, so we find an event for (String idx : eventsIndices) { From 508ecd55e5abc7873aff8fe5cfe731d08701533a Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Mon, 8 May 2023 13:20:49 +0200 Subject: [PATCH 18/36] Remove profiling plugin where unknown --- x-pack/plugin/eql/qa/mixed-node/build.gradle | 1 - .../xpack/monitoring/test/MonitoringIntegTestCase.java | 1 - 2 files changed, 2 deletions(-) diff --git a/x-pack/plugin/eql/qa/mixed-node/build.gradle b/x-pack/plugin/eql/qa/mixed-node/build.gradle index 9a3bdd94cb160..3e1010e2c0eb9 100644 --- a/x-pack/plugin/eql/qa/mixed-node/build.gradle +++ b/x-pack/plugin/eql/qa/mixed-node/build.gradle @@ -21,7 +21,6 @@ BuildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter("7.10.0") && testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.eql.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java index 1d3544f837f34..bb9a7b097f032 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java @@ -55,7 +55,6 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { // .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) // we do this by default in core, but for monitoring this isn't needed and only adds noise. .put("indices.lifecycle.history_index_enabled", false) - .put("xpack.profiling.templates.enabled", false) .put("index.store.mock.check_index_on_close", false); return builder.build(); From 0bce6ebd691d758914c38f94e62e9600325b8624 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Tue, 9 May 2023 16:25:59 +0200 Subject: [PATCH 19/36] Adjust tests --- .../xpack/monitoring/exporter/http/HttpExporterIT.java | 1 - x-pack/plugin/searchable-snapshots/qa/rest/build.gradle | 1 + x-pack/plugin/security/qa/operator-privileges-tests/build.gradle | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java index f1062d1054a85..e2e2cada89b48 100644 --- a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java +++ b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java @@ -111,7 +111,6 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .put(MonitoringService.INTERVAL.getKey(), MonitoringService.MIN_INTERVAL) // we do this by default in core, but for monitoring this isn't needed and only adds noise. .put("indices.lifecycle.history_index_enabled", false) - .put("xpack.profiling.templates.enabled", false) .put("index.store.mock.check_index_on_close", false); return builder.build(); } diff --git a/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle b/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle index b8c0127d0586a..7bd5e0eeaa50f 100644 --- a/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle @@ -22,6 +22,7 @@ testClusters.configureEach { testDistribution = 'DEFAULT' setting 'path.repo', repoDir.absolutePath setting 'xpack.license.self_generated.type', 'trial' + setting 'xpack.profiling.templates.enabled', 'false' setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle b/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle index c42b8583e648f..c1d1d7093068e 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle +++ b/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle @@ -36,6 +36,7 @@ testClusters.configureEach { setting 'xpack.security.enabled', 'true' setting 'xpack.security.http.ssl.enabled', 'false' setting 'xpack.security.operator_privileges.enabled', "true" + setting 'xpack.profiling.templates.enabled', 'false' setting 'path.repo', repoDir.absolutePath requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.3.0") From cc33ae1b24d169186c22e473b61d6a2bc1b8e4ea Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Tue, 9 May 2023 16:32:43 +0200 Subject: [PATCH 20/36] Amend docs about full cluster restore --- .../restore-snapshot.asciidoc | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/docs/reference/snapshot-restore/restore-snapshot.asciidoc b/docs/reference/snapshot-restore/restore-snapshot.asciidoc index 4fce5e99f4495..4880b6564779b 100644 --- a/docs/reference/snapshot-restore/restore-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/restore-snapshot.asciidoc @@ -232,7 +232,7 @@ snapshot as well as each feature's indices. To restore a specific feature state from the snapshot, specify the `feature_name` from the response in the restore snapshot API's -<> parameter. +<> parameter. NOTE: When you restore a feature state, {es} closes and overwrites the feature's existing indices. @@ -247,7 +247,7 @@ realm>> to ensure you'll still be able to access your cluster. POST _snapshot/my_repository/my_snapshot_2099.05.06/_restore { "feature_states": [ "geoip" ], - "include_global_state": false, <1> + "include_global_state": false, <1> "indices": "-*" <2> } ---- @@ -256,7 +256,7 @@ POST _snapshot/my_repository/my_snapshot_2099.05.06/_restore // TEST[s/_restore/_restore?wait_for_completion=true/] // TEST[s/"feature_states": \[ "geoip" \],//] -<1> Exclude the cluster state from the restore operation. +<1> Exclude the cluster state from the restore operation. <2> Exclude the other indices and data streams in the snapshot from the restore operation. [discrete] @@ -361,6 +361,17 @@ POST _watcher/_start //// -- +* Universal Profiling ++ +[source,console] +---- +PUT _cluster/settings +{ + "persistent": { + "xpack.profiling.templates.enabled": false + } +} + . {blank} + @@ -480,6 +491,17 @@ POST _watcher/_start ---- -- +* Universal Profiling ++ +[source,console] +---- +PUT _cluster/settings +{ + "persistent": { + "xpack.profiling.templates.enabled": true + } +} + . If wanted, reset the `action.destructive_requires_name` cluster setting. + [source,console] From d6655ad6ab245bc7fe5561ae5e19653e6bd26d1c Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Tue, 9 May 2023 17:09:51 +0200 Subject: [PATCH 21/36] Disable profiling only for 8.7.0+ --- x-pack/qa/mixed-tier-cluster/build.gradle | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/qa/mixed-tier-cluster/build.gradle b/x-pack/qa/mixed-tier-cluster/build.gradle index b546cfec8c2fb..1ea11aa56f5d5 100644 --- a/x-pack/qa/mixed-tier-cluster/build.gradle +++ b/x-pack/qa/mixed-tier-cluster/build.gradle @@ -19,7 +19,10 @@ BuildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter("7.9.0") && testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' + // profiling plugin has been added in 8.7.0 + if (bwcVersion.onOrAfter('8.7.0')) { + setting 'xpack.profiling.enabled', 'false' + } setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' nodes."${baseName}-0".setting 'node.roles', '["master"]' From 423d2d93cb6bc53c23e85bb735f2caa00e1eb0d7 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Tue, 9 May 2023 17:23:24 +0200 Subject: [PATCH 22/36] Adjust watcher tests --- .../xpack/watcher/test/AbstractWatcherIntegrationTestCase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java index 02bd510425b1c..8755ee282cb5e 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -117,7 +117,6 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .put("xpack.watcher.execution.scroll.size", randomIntBetween(1, 100)) .put("xpack.watcher.watch.scroll.size", randomIntBetween(1, 100)) .put("indices.lifecycle.history_index_enabled", false) - .put("xpack.profiling.templates.enabled", false) .build(); } From 62438919ea930aa9b7b8f072d9af0b4c83c7b63b Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Tue, 9 May 2023 17:42:18 +0200 Subject: [PATCH 23/36] Add missing snippet end in docs --- docs/reference/snapshot-restore/restore-snapshot.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/snapshot-restore/restore-snapshot.asciidoc b/docs/reference/snapshot-restore/restore-snapshot.asciidoc index 4880b6564779b..7e71c321f0565 100644 --- a/docs/reference/snapshot-restore/restore-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/restore-snapshot.asciidoc @@ -371,7 +371,7 @@ PUT _cluster/settings "xpack.profiling.templates.enabled": false } } - +---- . {blank} + @@ -501,6 +501,7 @@ PUT _cluster/settings "xpack.profiling.templates.enabled": true } } +---- . If wanted, reset the `action.destructive_requires_name` cluster setting. + From f7e9fec6c84daf52ce040bdab739ba22260e50b0 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Wed, 10 May 2023 09:54:23 +0200 Subject: [PATCH 24/36] More fixes --- docs/reference/snapshot-restore/restore-snapshot.asciidoc | 8 +++++--- x-pack/qa/runtime-fields/build.gradle | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/reference/snapshot-restore/restore-snapshot.asciidoc b/docs/reference/snapshot-restore/restore-snapshot.asciidoc index 7e71c321f0565..387baa1762447 100644 --- a/docs/reference/snapshot-restore/restore-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/restore-snapshot.asciidoc @@ -361,6 +361,10 @@ POST _watcher/_start //// -- + +. {blank} ++ +-- * Universal Profiling + [source,console] @@ -373,9 +377,6 @@ PUT _cluster/settings } ---- -. {blank} -+ --- [[restore-create-file-realm-user]] If you use {es} security features, log in to a node host, navigate to the {es} installation directory, and add a user with the `superuser` role to the file @@ -502,6 +503,7 @@ PUT _cluster/settings } } ---- +//TEST[s/true/false/] . If wanted, reset the `action.destructive_requires_name` cluster setting. + diff --git a/x-pack/qa/runtime-fields/build.gradle b/x-pack/qa/runtime-fields/build.gradle index 5bee9f08c2a73..058568362c998 100644 --- a/x-pack/qa/runtime-fields/build.gradle +++ b/x-pack/qa/runtime-fields/build.gradle @@ -43,6 +43,7 @@ subprojects { testDistribution = 'DEFAULT' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") } From 67488a96daa91f281ce036236d0488d1da187d8d Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Wed, 10 May 2023 11:10:31 +0200 Subject: [PATCH 25/36] Disable profiling only for 8.7.0+ --- x-pack/qa/rolling-upgrade-basic/build.gradle | 5 ++++- x-pack/qa/rolling-upgrade-multi-cluster/build.gradle | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/x-pack/qa/rolling-upgrade-basic/build.gradle b/x-pack/qa/rolling-upgrade-basic/build.gradle index 467f7886f00d0..dceb271f260c6 100644 --- a/x-pack/qa/rolling-upgrade-basic/build.gradle +++ b/x-pack/qa/rolling-upgrade-basic/build.gradle @@ -19,7 +19,10 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' + // profiling plugin has been added in 8.7.0 + if (bwcVersion.onOrAfter('8.7.0')) { + setting 'xpack.profiling.enabled', 'false' + } setting 'xpack.license.self_generated.type', 'basic' } diff --git a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle index 4f3440bdadd36..75b64cc1fc8a0 100644 --- a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle +++ b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle @@ -1,4 +1,4 @@ -import org.elasticsearch.gradle.internal.info.BuildParams + import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.internal-testclusters' @@ -25,7 +25,9 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' + if (bwcVersion.onOrAfter('8.7.0')) { + setting 'xpack.profiling.enabled', 'false' + } setting 'xpack.license.self_generated.type', 'trial' } From 7100f069ba7279e206ec100e86378e1d86528a33 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Wed, 10 May 2023 11:23:09 +0200 Subject: [PATCH 26/36] Wait for cluster state updates to finish --- .../java/org/elasticsearch/test/rest/ESRestTestCase.java | 2 +- .../elasticsearch/xpack/profiler/ProfilingIndexManager.java | 6 ++++++ .../AbstractSearchableSnapshotsRestTestCase.java | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 57c2ab1c64170..6e4f060f48920 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1245,7 +1245,7 @@ private void logIfThereAreRunningTasks() throws IOException { * Waits for the cluster state updates to have been processed, so that no cluster * state updates are still in-progress when the next test starts. */ - private void waitForClusterStateUpdatesToFinish() throws Exception { + protected final void waitForClusterStateUpdatesToFinish() throws Exception { assertBusy(() -> { try { Response response = adminClient().performRequest(new Request("GET", "/_cluster/pending_tasks")); diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java index acb87c8ebf562..14ddb142ff4c8 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java @@ -176,6 +176,12 @@ public void onResponse(CreateIndexResponse response) { index, ClientHelper.PROFILING_ORIGIN ); + } else if (response.isShardsAcknowledged() == false) { + logger.warn( + "adding index [{}] for [{}], shards were not acknowledged", + index, + ClientHelper.PROFILING_ORIGIN + ); } } diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java index 469bc33fcade5..7ff73c019667d 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java @@ -193,7 +193,7 @@ private void runSearchableSnapshotsTest( // Remove the snapshots, if a previous test failed to delete them. This is // useful for third party tests that runs the test against a real external service. deleteSnapshot(SNAPSHOT_NAME, true); - + waitForClusterStateUpdatesToFinish(); logger.info("creating snapshot [{}]", SNAPSHOT_NAME); createSnapshot(WRITE_REPOSITORY_NAME, SNAPSHOT_NAME, true); From bd7eee04e0006b51c222703d49421f5c7e02d11c Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Wed, 10 May 2023 11:48:38 +0200 Subject: [PATCH 27/36] Fix formatting --- .../elasticsearch/xpack/profiler/ProfilingIndexManager.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java index 14ddb142ff4c8..5e5dcfaeb8b4f 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java @@ -177,11 +177,7 @@ public void onResponse(CreateIndexResponse response) { ClientHelper.PROFILING_ORIGIN ); } else if (response.isShardsAcknowledged() == false) { - logger.warn( - "adding index [{}] for [{}], shards were not acknowledged", - index, - ClientHelper.PROFILING_ORIGIN - ); + logger.warn("adding index [{}] for [{}], shards were not acknowledged", index, ClientHelper.PROFILING_ORIGIN); } } From 66c4965c727b8f2f27998469e7ebd248a6e6f4a1 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Wed, 10 May 2023 13:57:02 +0200 Subject: [PATCH 28/36] More fixes --- .../java/org/elasticsearch/test/rest/ESRestTestCase.java | 2 +- x-pack/plugin/repositories-metering-api/build.gradle | 1 + x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle | 1 + x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle | 1 + x-pack/plugin/searchable-snapshots/qa/minio/build.gradle | 1 + x-pack/plugin/searchable-snapshots/qa/s3/build.gradle | 1 + x-pack/plugin/searchable-snapshots/qa/url/build.gradle | 1 + .../AbstractSearchableSnapshotsRestTestCase.java | 1 - x-pack/plugin/sql/qa/mixed-node/build.gradle | 5 ++++- 9 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 6e4f060f48920..57c2ab1c64170 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1245,7 +1245,7 @@ private void logIfThereAreRunningTasks() throws IOException { * Waits for the cluster state updates to have been processed, so that no cluster * state updates are still in-progress when the next test starts. */ - protected final void waitForClusterStateUpdatesToFinish() throws Exception { + private void waitForClusterStateUpdatesToFinish() throws Exception { assertBusy(() -> { try { Response response = adminClient().performRequest(new Request("GET", "/_cluster/pending_tasks")); diff --git a/x-pack/plugin/repositories-metering-api/build.gradle b/x-pack/plugin/repositories-metering-api/build.gradle index bfb6bf5364779..91ac21543b1aa 100644 --- a/x-pack/plugin/repositories-metering-api/build.gradle +++ b/x-pack/plugin/repositories-metering-api/build.gradle @@ -16,6 +16,7 @@ dependencies { testClusters.configureEach { setting 'xpack.security.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' } addQaCheckDependencies(project) diff --git a/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle b/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle index c9ce9c1e05a56..3444dec2a136a 100644 --- a/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle @@ -104,6 +104,7 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { } setting 'xpack.license.self_generated.type', 'trial' + setting 'xpack.profiling.templates.enabled', 'false' setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' diff --git a/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle b/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle index 94aa196f8e8e1..33e95d47f4f77 100644 --- a/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle @@ -131,6 +131,7 @@ testClusters.configureEach { testDistribution = 'DEFAULT' plugin(hdfsRepoPluginProject.path) setting 'xpack.license.self_generated.type', 'trial' + setting 'xpack.profiling.templates.enabled', 'false' setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' diff --git a/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle b/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle index 860e42378dcd9..b9972f4cd8fdb 100644 --- a/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle @@ -34,6 +34,7 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { keystore 's3.client.searchable_snapshots.access_key', 's3_test_access_key' keystore 's3.client.searchable_snapshots.secret_key', 's3_test_secret_key' setting 'xpack.license.self_generated.type', 'trial' + setting 'xpack.profiling.templates.enabled', 'false' setting 's3.client.searchable_snapshots.protocol', 'http' setting 's3.client.searchable_snapshots.endpoint', { "${-> fixtureAddress()}" }, IGNORE_VALUE diff --git a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle index 01a2dc065ec69..60fd097f66c47 100644 --- a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle @@ -49,6 +49,7 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { keystore 's3.client.searchable_snapshots.access_key', s3AccessKey keystore 's3.client.searchable_snapshots.secret_key', s3SecretKey setting 'xpack.license.self_generated.type', 'trial' + setting 'xpack.profiling.templates.enabled', 'false' if (useFixture) { def fixtureAddress = { fixtureName -> diff --git a/x-pack/plugin/searchable-snapshots/qa/url/build.gradle b/x-pack/plugin/searchable-snapshots/qa/url/build.gradle index 160eb8d28cbcc..ff4a5d6378a4d 100644 --- a/x-pack/plugin/searchable-snapshots/qa/url/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/url/build.gradle @@ -39,6 +39,7 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { setting 'repositories.url.allowed_urls', { "${-> fixtureAddress('nginx-fixture')}" }, IGNORE_VALUE setting 'xpack.license.self_generated.type', 'trial' + setting 'xpack.profiling.templates.enabled', 'false' setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java index 7ff73c019667d..d46c4f0c6e3f9 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java @@ -193,7 +193,6 @@ private void runSearchableSnapshotsTest( // Remove the snapshots, if a previous test failed to delete them. This is // useful for third party tests that runs the test against a real external service. deleteSnapshot(SNAPSHOT_NAME, true); - waitForClusterStateUpdatesToFinish(); logger.info("creating snapshot [{}]", SNAPSHOT_NAME); createSnapshot(WRITE_REPOSITORY_NAME, SNAPSHOT_NAME, true); diff --git a/x-pack/plugin/sql/qa/mixed-node/build.gradle b/x-pack/plugin/sql/qa/mixed-node/build.gradle index e05c7a944a1dd..935af1edb2653 100644 --- a/x-pack/plugin/sql/qa/mixed-node/build.gradle +++ b/x-pack/plugin/sql/qa/mixed-node/build.gradle @@ -28,7 +28,10 @@ BuildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter("7.10.3") && testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' + // profiling plugin has been added in 8.7.0 + if (bwcVersion.onOrAfter('8.7.0')) { + setting 'xpack.profiling.enabled', 'false' + } setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' // for debugging purposes From 896b7520de51add16690897f8e6d8dbc5ecbae66 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Wed, 10 May 2023 14:34:32 +0200 Subject: [PATCH 29/36] Disable profiling in repo metering QA tests --- x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle | 1 + x-pack/plugin/repositories-metering-api/qa/s3/build.gradle | 1 + 2 files changed, 2 insertions(+) diff --git a/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle b/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle index 2db3bf4f64cd4..5356ae1e915ce 100644 --- a/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle +++ b/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle @@ -49,6 +49,7 @@ if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) { testClusters.configureEach { setting 'xpack.security.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' } /** A service account file that points to the Google Cloud Storage service emulated by the fixture **/ diff --git a/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle b/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle index 05dbe3d6bd8c7..7ee1395544ba8 100644 --- a/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle +++ b/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle @@ -63,6 +63,7 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { println "Using an external service to test " + project.name } setting 'xpack.security.enabled', 'false' + setting 'xpack.profiling.enabled', 'false' } tasks.register("s3ThirdPartyTest").configure { From a9a1eb5e3cc1695a8917b1e64a1e6800e4109281 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Thu, 11 May 2023 07:32:02 +0200 Subject: [PATCH 30/36] Preliminary fix for #87001 --- .../xpack/core/DataTiersUsageTransportAction.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersUsageTransportAction.java index b7120f213cb15..92923ff99f29e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersUsageTransportAction.java @@ -206,7 +206,8 @@ private static void aggregateDataTierShardStats(NodeStats nodeStats, Index index accumulator.docCount += shardStat.getTotal().getDocs().getCount(); // Accumulate stats about started shards - if (node.getByShardId(shardStat.getShardId()).state() == ShardRoutingState.STARTED) { + ShardRouting shardRouting = node.getByShardId(shardStat.getShardId()); + if (shardRouting != null && shardRouting.state() == ShardRoutingState.STARTED) { accumulator.totalShardCount += 1; // Accumulate stats about started primary shards From 341c57fd734eb867227cf05180786eb15c3d2449 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Thu, 11 May 2023 11:02:02 +0200 Subject: [PATCH 31/36] Revert "Preliminary fix for #87001" This reverts commit a9a1eb5e3cc1695a8917b1e64a1e6800e4109281. --- .../xpack/core/DataTiersUsageTransportAction.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersUsageTransportAction.java index 92923ff99f29e..b7120f213cb15 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersUsageTransportAction.java @@ -206,8 +206,7 @@ private static void aggregateDataTierShardStats(NodeStats nodeStats, Index index accumulator.docCount += shardStat.getTotal().getDocs().getCount(); // Accumulate stats about started shards - ShardRouting shardRouting = node.getByShardId(shardStat.getShardId()); - if (shardRouting != null && shardRouting.state() == ShardRoutingState.STARTED) { + if (node.getByShardId(shardStat.getShardId()).state() == ShardRoutingState.STARTED) { accumulator.totalShardCount += 1; // Accumulate stats about started primary shards From f2e7c0261113f7ca80a73ac978132be66ce13f75 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Fri, 12 May 2023 07:38:35 +0200 Subject: [PATCH 32/36] Disable index template management by default --- docs/build.gradle | 1 - .../snapshot-restore/restore-snapshot.asciidoc | 13 +++++++++++++ modules/data-streams/build.gradle | 2 -- modules/dlm/build.gradle | 1 - plugins/mapper-murmur3/build.gradle | 1 - x-pack/docs/build.gradle | 2 -- x-pack/plugin/async-search/qa/rest/build.gradle | 1 - x-pack/plugin/async-search/qa/security/build.gradle | 1 - x-pack/plugin/autoscaling/qa/rest/build.gradle | 1 - x-pack/plugin/build.gradle | 1 - x-pack/plugin/ccr/qa/rest/build.gradle | 1 - x-pack/plugin/ccr/qa/restart/build.gradle | 2 -- x-pack/plugin/ccr/qa/security/build.gradle | 2 -- .../org/elasticsearch/xpack/CcrIntegTestCase.java | 1 - .../elasticsearch/xpack/CcrSingleNodeTestCase.java | 1 - x-pack/plugin/core/build.gradle | 2 -- .../qa/rest-with-advanced-security/build.gradle | 1 - .../enrich/qa/rest-with-security/build.gradle | 1 - x-pack/plugin/enrich/qa/rest/build.gradle | 1 - .../eql/qa/multi-cluster-with-security/build.gradle | 2 -- .../xpack/eql/action/AbstractEqlIntegTestCase.java | 1 - x-pack/plugin/ilm/qa/multi-cluster/build.gradle | 2 -- x-pack/plugin/ilm/qa/multi-node/build.gradle | 1 - x-pack/plugin/ilm/qa/rest/build.gradle | 1 - x-pack/plugin/ilm/qa/with-security/build.gradle | 1 - .../ilm/ClusterStateWaitThresholdBreachTests.java | 1 - .../xpack/ilm/DataAndIndexLifecycleMixingTests.java | 1 - .../xpack/ilm/DataTiersMigrationsTests.java | 1 - .../ilm/IndexLifecycleInitialisationTests.java | 1 - .../slm/SnapshotLifecycleInitialisationTests.java | 1 - x-pack/plugin/ml/qa/basic-multi-node/build.gradle | 1 - x-pack/plugin/ml/qa/disabled/build.gradle | 1 - .../multi-cluster-tests-with-security/build.gradle | 2 -- .../ml/qa/native-multi-node-tests/build.gradle | 1 - .../xpack/ml/integration/MlNativeIntegTestCase.java | 1 - .../xpack/ml/support/BaseMlIntegTestCase.java | 1 - .../xpack/monitoring/integration/MonitoringIT.java | 1 - ...TransportMonitoringMigrateAlertsActionTests.java | 1 - .../xpack/profiler/ProfilingPlugin.java | 3 +-- .../plugin/repositories-metering-api/build.gradle | 1 - .../repositories-metering-api/qa/gcs/build.gradle | 1 - .../repositories-metering-api/qa/s3/build.gradle | 1 - .../plugin/searchable-snapshots/qa/gcs/build.gradle | 1 - .../searchable-snapshots/qa/hdfs/build.gradle | 1 - .../searchable-snapshots/qa/minio/build.gradle | 1 - .../searchable-snapshots/qa/rest/build.gradle | 1 - .../plugin/searchable-snapshots/qa/s3/build.gradle | 1 - .../plugin/searchable-snapshots/qa/url/build.gradle | 1 - .../AbstractSearchableSnapshotsRestTestCase.java | 1 + .../security/EnableSecurityOnBasicLicenseIT.java | 1 - x-pack/plugin/security/qa/jwt-realm/build.gradle | 1 - .../qa/operator-privileges-tests/build.gradle | 1 - x-pack/plugin/security/qa/profile/build.gradle | 1 - .../plugin/security/qa/security-basic/build.gradle | 1 - .../security/qa/security-disabled/build.gradle | 1 - .../plugin/security/qa/security-trial/build.gradle | 1 - .../security/qa/smoke-test-all-realms/build.gradle | 1 - x-pack/plugin/security/qa/tls-basic/build.gradle | 1 - x-pack/plugin/sql/qa/jdbc/build.gradle | 1 - x-pack/plugin/sql/qa/mixed-node/build.gradle | 4 ---- x-pack/plugin/sql/qa/server/build.gradle | 1 - .../server/multi-cluster-with-security/build.gradle | 2 -- .../action/AbstractSqlBlockingIntegTestCase.java | 1 - .../xpack/sql/action/AbstractSqlIntegTestCase.java | 1 - x-pack/plugin/stack/qa/rest/build.gradle | 1 - .../multi-cluster-tests-with-security/build.gradle | 2 -- x-pack/plugin/watcher/qa/rest/build.gradle | 1 - .../plugin/watcher/qa/with-monitoring/build.gradle | 1 - x-pack/plugin/watcher/qa/with-security/build.gradle | 1 - .../CoreWithSecurityClientYamlTestSuiteIT.java | 1 - x-pack/qa/kerberos-tests/build.gradle | 1 - x-pack/qa/mixed-tier-cluster/build.gradle | 4 ---- .../multi_node/GlobalCheckpointSyncActionIT.java | 1 - .../java/org/elasticsearch/multi_node/RollupIT.java | 1 - x-pack/qa/reindex-tests-with-security/build.gradle | 1 - x-pack/qa/rolling-upgrade-basic/build.gradle | 4 ---- .../qa/rolling-upgrade-multi-cluster/build.gradle | 3 --- x-pack/qa/runtime-fields/build.gradle | 1 - x-pack/qa/runtime-fields/with-security/build.gradle | 1 - x-pack/qa/saml-idp-tests/build.gradle | 1 - .../qa/security-example-spi-extension/build.gradle | 1 - .../qa/security-setup-password-tests/build.gradle | 1 - x-pack/qa/smoke-test-plugins-ssl/build.gradle | 1 - .../smoke-test-security-with-mustache/build.gradle | 1 - x-pack/qa/third-party/jira/build.gradle | 1 - x-pack/qa/third-party/pagerduty/build.gradle | 1 - x-pack/qa/third-party/slack/build.gradle | 1 - x-pack/qa/xpack-prefix-rest-compat/build.gradle | 1 - 88 files changed, 15 insertions(+), 108 deletions(-) diff --git a/docs/build.gradle b/docs/build.gradle index 87eb4af808d8f..927419c6b3a69 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -61,7 +61,6 @@ restResources { testClusters.matching { it.name == "yamlRestTest"}.configureEach { if (singleNode().testDistribution == DEFAULT) { setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.profiling.templates.enabled', 'false' setting 'indices.lifecycle.history_index_enabled', 'false' keystorePassword 'keystore-password' } diff --git a/docs/reference/snapshot-restore/restore-snapshot.asciidoc b/docs/reference/snapshot-restore/restore-snapshot.asciidoc index 387baa1762447..48da6e7293fc5 100644 --- a/docs/reference/snapshot-restore/restore-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/restore-snapshot.asciidoc @@ -366,6 +366,17 @@ POST _watcher/_start + -- * Universal Profiling + +Check if Universal Profiling index template management is enabled: + ++ +[source,console] +---- +GET /_cluster/settings?filter_path=**.xpack.profiling.templates.enabled&include_defaults=true +---- + +If the value is `true`, disable Universal Profiling index template management: + + [source,console] ---- @@ -493,6 +504,8 @@ POST _watcher/_start -- * Universal Profiling + +If the value was `true` initially, enable Universal Profiling index template management again, otherwise skip this step: + [source,console] ---- diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index dff5b4a536e77..39d96c1955ab1 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -22,7 +22,6 @@ restResources { testClusters.configureEach { module ':modules:reindex' testDistribution = 'DEFAULT' - setting 'xpack.profiling.templates.enabled', 'false' // disable ILM history, since it disturbs tests using _all setting 'indices.lifecycle.history_index_enabled', 'false' setting 'xpack.security.enabled', 'true' @@ -34,7 +33,6 @@ testClusters.configureEach { testClusters.matching { it.name == "javaRestTest" }.configureEach { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' - setting 'xpack.profiling.templates.enabled', 'false' // disable ILM history, since it disturbs tests using _all setting 'indices.lifecycle.history_index_enabled', 'false' } diff --git a/modules/dlm/build.gradle b/modules/dlm/build.gradle index 74dce2c7deed2..20d738eabc03a 100644 --- a/modules/dlm/build.gradle +++ b/modules/dlm/build.gradle @@ -28,7 +28,6 @@ addQaCheckDependencies(project) testClusters.configureEach { module ':modules:reindex' testDistribution = 'DEFAULT' - setting 'xpack.profiling.templates.enabled', 'false' // disable ILM history, since it disturbs tests using _all setting 'indices.lifecycle.history_index_enabled', 'false' setting 'xpack.security.enabled', 'true' diff --git a/plugins/mapper-murmur3/build.gradle b/plugins/mapper-murmur3/build.gradle index e53f6eee0e022..7e93e6d295397 100644 --- a/plugins/mapper-murmur3/build.gradle +++ b/plugins/mapper-murmur3/build.gradle @@ -36,5 +36,4 @@ restResources { testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' } diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index 2cda8ded3887d..c834689a0d873 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -46,8 +46,6 @@ testClusters.matching { it.name == "yamlRestTest" }.configureEach { setting 'xpack.security.enabled', 'true' setting 'xpack.security.authc.api_key.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' - // disabling universal profiling template management for doc tests to avoid potential lingering tasks that'd cause test flakiness - setting 'xpack.profiling.templates.enabled', 'false' // disable the ILM history for doc tests to avoid potential lingering tasks that'd cause test flakiness setting 'indices.lifecycle.history_index_enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' diff --git a/x-pack/plugin/async-search/qa/rest/build.gradle b/x-pack/plugin/async-search/qa/rest/build.gradle index 79dd0c4b71e2a..b41600407be5c 100644 --- a/x-pack/plugin/async-search/qa/rest/build.gradle +++ b/x-pack/plugin/async-search/qa/rest/build.gradle @@ -18,7 +18,6 @@ restResources { testClusters.configureEach { testDistribution = 'DEFAULT' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'false' } diff --git a/x-pack/plugin/async-search/qa/security/build.gradle b/x-pack/plugin/async-search/qa/security/build.gradle index f464f03b04216..3e29a711b0a8c 100644 --- a/x-pack/plugin/async-search/qa/security/build.gradle +++ b/x-pack/plugin/async-search/qa/security/build.gradle @@ -11,7 +11,6 @@ testClusters.configureEach { testDistribution = 'DEFAULT' numberOfNodes = 2 setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'true' extraConfigFile 'roles.yml', file('roles.yml') user username: "test_kibana_user", password: "x-pack-test-password", role: "kibana_system" diff --git a/x-pack/plugin/autoscaling/qa/rest/build.gradle b/x-pack/plugin/autoscaling/qa/rest/build.gradle index c3c40bb42a101..19254880a7089 100644 --- a/x-pack/plugin/autoscaling/qa/rest/build.gradle +++ b/x-pack/plugin/autoscaling/qa/rest/build.gradle @@ -21,7 +21,6 @@ tasks.named("yamlRestTestV7CompatTest").configure { testClusters.configureEach { testDistribution = 'DEFAULT' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' extraConfigFile 'roles.yml', file('autoscaling-roles.yml') diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 3a28f3d6c3379..93aa30ec68f44 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -187,7 +187,6 @@ testClusters.configureEach { setting 'xpack.ml.enabled', 'true' setting 'xpack.security.enabled', 'true' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' // Integration tests are supposed to enable/disable exporters before/after each test setting 'xpack.security.authc.token.enabled', 'true' setting 'xpack.security.authc.api_key.enabled', 'true' diff --git a/x-pack/plugin/ccr/qa/rest/build.gradle b/x-pack/plugin/ccr/qa/rest/build.gradle index 7ba9597d9b1ce..5f173b8831df8 100644 --- a/x-pack/plugin/ccr/qa/rest/build.gradle +++ b/x-pack/plugin/ccr/qa/rest/build.gradle @@ -18,7 +18,6 @@ testClusters.configureEach { // that is thrown in production when indexing a document directly in a follower index. jvmArgs '-da:org.elasticsearch.xpack.ccr.index.engine.FollowingEngineAssertions' setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' // TODO: reduce the need for superuser here diff --git a/x-pack/plugin/ccr/qa/restart/build.gradle b/x-pack/plugin/ccr/qa/restart/build.gradle index 9ed3c5cbb10cf..d354cd911f2f8 100644 --- a/x-pack/plugin/ccr/qa/restart/build.gradle +++ b/x-pack/plugin/ccr/qa/restart/build.gradle @@ -12,7 +12,6 @@ def leaderCluster = testClusters.register('leader-cluster') { testDistribution = 'DEFAULT' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' - setting 'xpack.profiling.enabled', 'false' user username: 'admin', password: 'admin-password', role: 'superuser' } @@ -21,7 +20,6 @@ def followCluster = testClusters.register('follow-cluster') { setting 'xpack.monitoring.collection.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' - setting 'xpack.profiling.enabled', 'false' user username: 'admin', password: 'admin-password', role: 'superuser' setting 'cluster.remote.leader_cluster.seeds', { "\"${leaderCluster.get().getAllTransportPortURI().get(0)}\"" } diff --git a/x-pack/plugin/ccr/qa/security/build.gradle b/x-pack/plugin/ccr/qa/security/build.gradle index 02b8364b6f250..ff4a4f857fe32 100644 --- a/x-pack/plugin/ccr/qa/security/build.gradle +++ b/x-pack/plugin/ccr/qa/security/build.gradle @@ -13,7 +13,6 @@ def leadCluster = testClusters.register('leader-cluster') { testDistribution = 'DEFAULT' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' - setting 'xpack.profiling.enabled', 'false' extraConfigFile 'roles.yml', file('leader-roles.yml') user username: "test_admin", role: "superuser" user username: "test_ccr", role: "ccruser" @@ -26,7 +25,6 @@ testClusters.register('follow-cluster') { } setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.monitoring.collection.enabled', 'false' // will be enabled by tests extraConfigFile 'roles.yml', file('follower-roles.yml') user username: "test_admin", role: "superuser" diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java index 141e569738de2..d09b27cb06a58 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java @@ -304,7 +304,6 @@ private NodeConfigurationSource createNodeConfigurationSource(final String leade builder.put(XPackSettings.SECURITY_ENABLED.getKey(), false); builder.put(XPackSettings.WATCHER_ENABLED.getKey(), false); builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); - builder.put(XPackSettings.PROFILING_ENABLED.getKey(), false); builder.put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial"); // Let cluster state api return quickly in order to speed up auto follow tests: builder.put(CcrSettings.CCR_WAIT_FOR_METADATA_TIMEOUT.getKey(), TimeValue.timeValueMillis(100)); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java index d043b58094fe2..a540631dc040e 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java @@ -48,7 +48,6 @@ protected Settings nodeSettings() { builder.put(XPackSettings.SECURITY_ENABLED.getKey(), false); builder.put(XPackSettings.WATCHER_ENABLED.getKey(), false); builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); - builder.put(XPackSettings.PROFILING_ENABLED.getKey(), false); builder.put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial"); // Let cluster state api return quickly in order to speed up auto follow tests: builder.put(CcrSettings.CCR_WAIT_FOR_METADATA_TIMEOUT.getKey(), TimeValue.timeValueMillis(100)); diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle index 7fcf423838c30..38e255b6efc0d 100644 --- a/x-pack/plugin/core/build.gradle +++ b/x-pack/plugin/core/build.gradle @@ -140,8 +140,6 @@ testClusters.configureEach { testDistribution = 'default' setting 'xpack.security.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' - // disabling universal profiling template management as it disturbs testDSXpackUsage test - setting 'xpack.profiling.templates.enabled', 'false' //disabling ILM history as it disturbs testDSXpackUsage test setting 'indices.lifecycle.history_index_enabled', 'false' keystore 'bootstrap.password', 'x-pack-test-password' diff --git a/x-pack/plugin/enrich/qa/rest-with-advanced-security/build.gradle b/x-pack/plugin/enrich/qa/rest-with-advanced-security/build.gradle index eddaf8ca5208e..2e649e718b081 100644 --- a/x-pack/plugin/enrich/qa/rest-with-advanced-security/build.gradle +++ b/x-pack/plugin/enrich/qa/rest-with-advanced-security/build.gradle @@ -14,6 +14,5 @@ testClusters.configureEach { user username: "test_enrich", password: "x-pack-test-password", role: "integ_test_role" setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.monitoring.collection.enabled', 'true' } diff --git a/x-pack/plugin/enrich/qa/rest-with-security/build.gradle b/x-pack/plugin/enrich/qa/rest-with-security/build.gradle index f51349e54e0e1..69fec4ad32c74 100644 --- a/x-pack/plugin/enrich/qa/rest-with-security/build.gradle +++ b/x-pack/plugin/enrich/qa/rest-with-security/build.gradle @@ -19,6 +19,5 @@ testClusters.configureEach { user username: "test_enrich_no_privs", password: "x-pack-test-password", role: "enrich_no_privs" setting 'xpack.license.self_generated.type', 'basic' setting 'xpack.security.enabled', 'true' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.monitoring.collection.enabled', 'true' } diff --git a/x-pack/plugin/enrich/qa/rest/build.gradle b/x-pack/plugin/enrich/qa/rest/build.gradle index c9feca4a91247..e8473d15ed9ef 100644 --- a/x-pack/plugin/enrich/qa/rest/build.gradle +++ b/x-pack/plugin/enrich/qa/rest/build.gradle @@ -30,6 +30,5 @@ testClusters.configureEach { setting 'xpack.license.self_generated.type', 'basic' setting 'xpack.monitoring.collection.enabled', 'true' setting 'xpack.security.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.4.0") } diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle b/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle index abfb8c5d8ac36..5fc247693c453 100644 --- a/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle +++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle @@ -11,7 +11,6 @@ def remoteClusterReg = testClusters.register('remote-cluster') { numberOfNodes = 2 setting 'node.roles', '[data,ingest,master]' setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.watcher.enabled', 'false' setting 'xpack.security.enabled', 'true' setting 'xpack.security.autoconfiguration.enabled', 'false' @@ -23,7 +22,6 @@ def integTestClusterReg = testClusters.register('javaRestTest') { testDistribution = 'DEFAULT' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'cluster.remote.my_remote_cluster.seeds', { remoteClusterReg.get().getAllTransportPortURI().collect { "\"$it\"" }.toString() } diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java index d7f3460631cf7..329d0e3d0a049 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java @@ -28,7 +28,6 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); - settings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); settings.put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial"); return settings.build(); } diff --git a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle index bf54d13bb2c83..49c2629b58580 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle @@ -26,7 +26,6 @@ testClusters.matching { it.name == 'leader-cluster' }.configureEach { setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.poll_interval', '1000ms' } @@ -49,7 +48,6 @@ testClusters.matching{ it.name == 'follow-cluster' }.configureEach { setting 'xpack.ccr.enabled', 'true' setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.poll_interval', '1000ms' diff --git a/x-pack/plugin/ilm/qa/multi-node/build.gradle b/x-pack/plugin/ilm/qa/multi-node/build.gradle index f117bb646c50d..523bf5d04ae4f 100644 --- a/x-pack/plugin/ilm/qa/multi-node/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-node/build.gradle @@ -24,7 +24,6 @@ testClusters.configureEach { setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.poll_interval', '1000ms' diff --git a/x-pack/plugin/ilm/qa/rest/build.gradle b/x-pack/plugin/ilm/qa/rest/build.gradle index c6e4efb5ee456..a17c130d4bbb4 100644 --- a/x-pack/plugin/ilm/qa/rest/build.gradle +++ b/x-pack/plugin/ilm/qa/rest/build.gradle @@ -15,7 +15,6 @@ restResources { testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.autoconfiguration.enabled', 'false' } diff --git a/x-pack/plugin/ilm/qa/with-security/build.gradle b/x-pack/plugin/ilm/qa/with-security/build.gradle index 0a0ca062c0adf..01d48b8e1812b 100644 --- a/x-pack/plugin/ilm/qa/with-security/build.gradle +++ b/x-pack/plugin/ilm/qa/with-security/build.gradle @@ -9,7 +9,6 @@ dependencies { testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' rolesFile file('roles.yml') diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java index 78543938da86f..10e5cca64dc75 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java @@ -76,7 +76,6 @@ protected Collection> nodePlugins() { protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); - settings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataAndIndexLifecycleMixingTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataAndIndexLifecycleMixingTests.java index a1b507f5437e0..8ecd36fbd6139 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataAndIndexLifecycleMixingTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataAndIndexLifecycleMixingTests.java @@ -88,7 +88,6 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { settings.put(DataLifecycleService.DLM_POLL_INTERVAL, "1s"); settings.put(DataLifecycle.CLUSTER_DLM_DEFAULT_ROLLOVER_SETTING.getKey(), "min_docs=1,max_docs=1"); settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); - settings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java index 480e6356a0e54..0fb550e250312 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java @@ -69,7 +69,6 @@ protected Collection> nodePlugins() { protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); - settings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java index 70170001da82c..5696abe2f651c 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java @@ -99,7 +99,6 @@ public class IndexLifecycleInitialisationTests extends ESIntegTestCase { protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { Settings.Builder nodeSettings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); nodeSettings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); - nodeSettings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); nodeSettings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); nodeSettings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); nodeSettings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java index 2344c28ed90bf..fa312cdac8e14 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java @@ -46,7 +46,6 @@ public static void setupRepositoryPath() { protected Settings nodeSettings() { Settings.Builder settings = Settings.builder().put(super.nodeSettings()); settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); - settings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); diff --git a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle index 3cbede0af94ff..72793d1f9a414 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle +++ b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle @@ -8,7 +8,6 @@ testClusters.configureEach { setting 'xpack.security.enabled', 'false' setting 'xpack.monitoring.elasticsearch.collection.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.ml.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.history_index_enabled', 'false' diff --git a/x-pack/plugin/ml/qa/disabled/build.gradle b/x-pack/plugin/ml/qa/disabled/build.gradle index 01b67e288f285..97a7b0eed73ad 100644 --- a/x-pack/plugin/ml/qa/disabled/build.gradle +++ b/x-pack/plugin/ml/qa/disabled/build.gradle @@ -11,7 +11,6 @@ testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' } if (BuildParams.inFipsJvm){ diff --git a/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle b/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle index 945bf2c33efa8..368aab4615fc7 100644 --- a/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle @@ -25,7 +25,6 @@ def remoteCluster = testClusters.register('remote-cluster') { setting 'node.roles', '[data,ingest,master]' setting 'xpack.security.enabled', 'true' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' user username: "test_user", password: "x-pack-test-password" @@ -37,7 +36,6 @@ testClusters.register('mixed-cluster') { setting 'node.roles', '[data,ingest,master]' setting 'xpack.security.enabled', 'true' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'cluster.remote.my_remote_cluster.seeds', { remoteCluster.get().getAllTransportPortURI().collect { "\"$it\"" }.toString() diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle index 013df22a00925..c9643ce25dddb 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle @@ -42,7 +42,6 @@ testClusters.configureEach { setting 'xpack.security.enabled', 'true' setting 'xpack.ml.enabled', 'true' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.authc.token.enabled', 'true' setting 'xpack.security.transport.ssl.enabled', 'true' setting 'xpack.security.transport.ssl.key', nodeKey.name diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java index 1fc09cacd4c73..cda03265fec28 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java @@ -187,7 +187,6 @@ protected Settings externalClusterClientSettings() { builder.putList("node.roles", Collections.emptyList()); builder.put(NetworkModule.TRANSPORT_TYPE_KEY, SecurityField.NAME4); builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), true); - builder.put(XPackSettings.PROFILING_ENABLED.getKey(), false); builder.put(XPackSettings.SECURITY_ENABLED.getKey(), true); builder.put(MachineLearningField.AUTODETECT_PROCESS.getKey(), false); builder.put(XPackSettings.WATCHER_ENABLED.getKey(), false); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java index 51711e6324a45..8987df1ece6b9 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java @@ -130,7 +130,6 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); settings.put(MachineLearningField.AUTODETECT_PROCESS.getKey(), false); settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), true); - settings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); settings.put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial"); settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); diff --git a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java index a96ced9a3fb14..0b51e72846531 100644 --- a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java +++ b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java @@ -83,7 +83,6 @@ protected Settings nodeSettings() { return Settings.builder() .put(super.nodeSettings()) .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) - .put(XPackSettings.PROFILING_ENABLED.getKey(), false) .put("xpack.monitoring.collection.interval", MonitoringService.MIN_INTERVAL) .put("xpack.monitoring.exporters._local.type", "local") .put("xpack.monitoring.exporters._local.enabled", false) diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java index e4ce67e15e7c5..9d47fc3e973b3 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java @@ -91,7 +91,6 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { // X-Pack configuration .put("xpack.license.self_generated.type", "trial") .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) - .put(XPackSettings.PROFILING_ENABLED.getKey(), false) .build(); } diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java index 9a10f76dfcff0..471a843bc2b95 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java @@ -51,8 +51,7 @@ public class ProfilingPlugin extends Plugin implements ActionPlugin { private static final Logger logger = LogManager.getLogger(ProfilingPlugin.class); public static final Setting PROFILING_TEMPLATES_ENABLED = Setting.boolSetting( "xpack.profiling.templates.enabled", - // Enable by default iff the profiling plugin is enabled - (settings) -> String.valueOf(XPackSettings.PROFILING_ENABLED.get(settings)), + false, Setting.Property.NodeScope, Setting.Property.Dynamic ); diff --git a/x-pack/plugin/repositories-metering-api/build.gradle b/x-pack/plugin/repositories-metering-api/build.gradle index 91ac21543b1aa..bfb6bf5364779 100644 --- a/x-pack/plugin/repositories-metering-api/build.gradle +++ b/x-pack/plugin/repositories-metering-api/build.gradle @@ -16,7 +16,6 @@ dependencies { testClusters.configureEach { setting 'xpack.security.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' } addQaCheckDependencies(project) diff --git a/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle b/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle index 5356ae1e915ce..2db3bf4f64cd4 100644 --- a/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle +++ b/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle @@ -49,7 +49,6 @@ if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) { testClusters.configureEach { setting 'xpack.security.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' } /** A service account file that points to the Google Cloud Storage service emulated by the fixture **/ diff --git a/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle b/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle index 7ee1395544ba8..05dbe3d6bd8c7 100644 --- a/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle +++ b/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle @@ -63,7 +63,6 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { println "Using an external service to test " + project.name } setting 'xpack.security.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' } tasks.register("s3ThirdPartyTest").configure { diff --git a/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle b/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle index 3444dec2a136a..c9ce9c1e05a56 100644 --- a/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle @@ -104,7 +104,6 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { } setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.profiling.templates.enabled', 'false' setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' diff --git a/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle b/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle index 33e95d47f4f77..94aa196f8e8e1 100644 --- a/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle @@ -131,7 +131,6 @@ testClusters.configureEach { testDistribution = 'DEFAULT' plugin(hdfsRepoPluginProject.path) setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.profiling.templates.enabled', 'false' setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' diff --git a/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle b/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle index b9972f4cd8fdb..860e42378dcd9 100644 --- a/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle @@ -34,7 +34,6 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { keystore 's3.client.searchable_snapshots.access_key', 's3_test_access_key' keystore 's3.client.searchable_snapshots.secret_key', 's3_test_secret_key' setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.profiling.templates.enabled', 'false' setting 's3.client.searchable_snapshots.protocol', 'http' setting 's3.client.searchable_snapshots.endpoint', { "${-> fixtureAddress()}" }, IGNORE_VALUE diff --git a/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle b/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle index 7bd5e0eeaa50f..b8c0127d0586a 100644 --- a/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle @@ -22,7 +22,6 @@ testClusters.configureEach { testDistribution = 'DEFAULT' setting 'path.repo', repoDir.absolutePath setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.profiling.templates.enabled', 'false' setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' diff --git a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle index 60fd097f66c47..01a2dc065ec69 100644 --- a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle @@ -49,7 +49,6 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { keystore 's3.client.searchable_snapshots.access_key', s3AccessKey keystore 's3.client.searchable_snapshots.secret_key', s3SecretKey setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.profiling.templates.enabled', 'false' if (useFixture) { def fixtureAddress = { fixtureName -> diff --git a/x-pack/plugin/searchable-snapshots/qa/url/build.gradle b/x-pack/plugin/searchable-snapshots/qa/url/build.gradle index ff4a5d6378a4d..160eb8d28cbcc 100644 --- a/x-pack/plugin/searchable-snapshots/qa/url/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/url/build.gradle @@ -39,7 +39,6 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { setting 'repositories.url.allowed_urls', { "${-> fixtureAddress('nginx-fixture')}" }, IGNORE_VALUE setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.profiling.templates.enabled', 'false' setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java index d46c4f0c6e3f9..469bc33fcade5 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java @@ -193,6 +193,7 @@ private void runSearchableSnapshotsTest( // Remove the snapshots, if a previous test failed to delete them. This is // useful for third party tests that runs the test against a real external service. deleteSnapshot(SNAPSHOT_NAME, true); + logger.info("creating snapshot [{}]", SNAPSHOT_NAME); createSnapshot(WRITE_REPOSITORY_NAME, SNAPSHOT_NAME, true); diff --git a/x-pack/plugin/security/qa/basic-enable-security/src/javaRestTest/java/org/elasticsearch/xpack/security/EnableSecurityOnBasicLicenseIT.java b/x-pack/plugin/security/qa/basic-enable-security/src/javaRestTest/java/org/elasticsearch/xpack/security/EnableSecurityOnBasicLicenseIT.java index cba3c205e8e34..ec2a3e2b01803 100644 --- a/x-pack/plugin/security/qa/basic-enable-security/src/javaRestTest/java/org/elasticsearch/xpack/security/EnableSecurityOnBasicLicenseIT.java +++ b/x-pack/plugin/security/qa/basic-enable-security/src/javaRestTest/java/org/elasticsearch/xpack/security/EnableSecurityOnBasicLicenseIT.java @@ -46,7 +46,6 @@ public class EnableSecurityOnBasicLicenseIT extends ESRestTestCase { private static MutableSettingsProvider clusterSettings = new MutableSettingsProvider() { { put("xpack.ml.enabled", "false"); - put("xpack.profiling.enabled", "false"); put("xpack.security.enabled", "false"); put("xpack.license.self_generated.type", "basic"); } diff --git a/x-pack/plugin/security/qa/jwt-realm/build.gradle b/x-pack/plugin/security/qa/jwt-realm/build.gradle index 6afb12dcdcae7..d561f0c8ceeed 100644 --- a/x-pack/plugin/security/qa/jwt-realm/build.gradle +++ b/x-pack/plugin/security/qa/jwt-realm/build.gradle @@ -32,7 +32,6 @@ testClusters.matching { it.name == 'javaRestTest' }.configureEach { extraConfigFile 'rsa.jwkset', file('src/javaRestTest/resources/jwk/rsa-public-jwkset.json') setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' setting 'xpack.security.http.ssl.enabled', 'true' diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle b/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle index c1d1d7093068e..c42b8583e648f 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle +++ b/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle @@ -36,7 +36,6 @@ testClusters.configureEach { setting 'xpack.security.enabled', 'true' setting 'xpack.security.http.ssl.enabled', 'false' setting 'xpack.security.operator_privileges.enabled', "true" - setting 'xpack.profiling.templates.enabled', 'false' setting 'path.repo', repoDir.absolutePath requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.3.0") diff --git a/x-pack/plugin/security/qa/profile/build.gradle b/x-pack/plugin/security/qa/profile/build.gradle index 4398fb3fb1674..eae33a5ba5876 100644 --- a/x-pack/plugin/security/qa/profile/build.gradle +++ b/x-pack/plugin/security/qa/profile/build.gradle @@ -18,7 +18,6 @@ testClusters.matching { it.name == 'javaRestTest' }.configureEach { rolesFile file('src/javaRestTest/resources/roles.yml') setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' diff --git a/x-pack/plugin/security/qa/security-basic/build.gradle b/x-pack/plugin/security/qa/security-basic/build.gradle index a98bc715ca18a..10a8081789652 100644 --- a/x-pack/plugin/security/qa/security-basic/build.gradle +++ b/x-pack/plugin/security/qa/security-basic/build.gradle @@ -19,7 +19,6 @@ testClusters.configureEach { numberOfNodes = 2 setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'basic' setting 'xpack.security.enabled', 'true' setting 'xpack.security.ssl.diagnose.trust', 'true' diff --git a/x-pack/plugin/security/qa/security-disabled/build.gradle b/x-pack/plugin/security/qa/security-disabled/build.gradle index 36105365f77dc..e3d9c2f4f65ba 100644 --- a/x-pack/plugin/security/qa/security-disabled/build.gradle +++ b/x-pack/plugin/security/qa/security-disabled/build.gradle @@ -19,7 +19,6 @@ testClusters.configureEach { numberOfNodes = 2 setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' // We run with a trial license, but explicitly disable security. // This means the security plugin is loaded and all feature are permitted, but they are not enabled setting 'xpack.license.self_generated.type', 'trial' diff --git a/x-pack/plugin/security/qa/security-trial/build.gradle b/x-pack/plugin/security/qa/security-trial/build.gradle index e3c4f21e6b070..1b1d5495f1d37 100644 --- a/x-pack/plugin/security/qa/security-trial/build.gradle +++ b/x-pack/plugin/security/qa/security-trial/build.gradle @@ -15,7 +15,6 @@ testClusters.matching { it.name == 'javaRestTest' }.configureEach { numberOfNodes = 2 setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' setting 'xpack.security.ssl.diagnose.trust', 'true' diff --git a/x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle b/x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle index c6bc42911bf6c..ca33b61de4ca5 100644 --- a/x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle +++ b/x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle @@ -26,7 +26,6 @@ testClusters.matching { it.name == 'javaRestTest' }.configureEach { extraConfigFile 'oidc-jwkset.json', file('src/javaRestTest/resources/oidc-jwkset.json') setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' setting 'xpack.security.authc.api_key.enabled', 'true' diff --git a/x-pack/plugin/security/qa/tls-basic/build.gradle b/x-pack/plugin/security/qa/tls-basic/build.gradle index 327a99b94a4fd..fbe91009011e3 100644 --- a/x-pack/plugin/security/qa/tls-basic/build.gradle +++ b/x-pack/plugin/security/qa/tls-basic/build.gradle @@ -23,7 +23,6 @@ testClusters.matching { it.name == 'javaRestTest' }.configureEach { extraConfigFile 'ca.crt', file('src/javaRestTest/resources/ssl/ca.crt') setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'basic' setting 'xpack.security.enabled', 'true' setting 'xpack.security.http.ssl.enabled', 'true' diff --git a/x-pack/plugin/sql/qa/jdbc/build.gradle b/x-pack/plugin/sql/qa/jdbc/build.gradle index b79e3ffff74cd..65d074538361e 100644 --- a/x-pack/plugin/sql/qa/jdbc/build.gradle +++ b/x-pack/plugin/sql/qa/jdbc/build.gradle @@ -63,7 +63,6 @@ subprojects { testDistribution = 'DEFAULT' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' } tasks.named("javaRestTest").configure { diff --git a/x-pack/plugin/sql/qa/mixed-node/build.gradle b/x-pack/plugin/sql/qa/mixed-node/build.gradle index 935af1edb2653..13bf18e6c080d 100644 --- a/x-pack/plugin/sql/qa/mixed-node/build.gradle +++ b/x-pack/plugin/sql/qa/mixed-node/build.gradle @@ -28,10 +28,6 @@ BuildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter("7.10.3") && testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - // profiling plugin has been added in 8.7.0 - if (bwcVersion.onOrAfter('8.7.0')) { - setting 'xpack.profiling.enabled', 'false' - } setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' // for debugging purposes diff --git a/x-pack/plugin/sql/qa/server/build.gradle b/x-pack/plugin/sql/qa/server/build.gradle index 60c87efe27d8e..7105e71fc0544 100644 --- a/x-pack/plugin/sql/qa/server/build.gradle +++ b/x-pack/plugin/sql/qa/server/build.gradle @@ -47,7 +47,6 @@ subprojects { testDistribution = 'DEFAULT' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' } diff --git a/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle b/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle index 1d5073f13829e..b56d8ff211990 100644 --- a/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle +++ b/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle @@ -11,7 +11,6 @@ def remoteClusterReg = testClusters.register('remote-cluster') { setting 'node.roles', '[data,ingest,master]' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.autoconfiguration.enabled', 'false' @@ -24,7 +23,6 @@ def javaRestTestClusterReg = testClusters.register('javaRestTest') { testDistribution = 'DEFAULT' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'cluster.remote.my_remote_cluster.seeds', { remoteClusterReg.get().getAllTransportPortURI().collect { "\"$it\"" }.toString() } diff --git a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/AbstractSqlBlockingIntegTestCase.java b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/AbstractSqlBlockingIntegTestCase.java index 6e192af3eac93..4ac4827c9d520 100644 --- a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/AbstractSqlBlockingIntegTestCase.java +++ b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/AbstractSqlBlockingIntegTestCase.java @@ -61,7 +61,6 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); - settings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); settings.put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial"); return settings.build(); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/AbstractSqlIntegTestCase.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/AbstractSqlIntegTestCase.java index e5c770925e91f..4f7646127cf38 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/AbstractSqlIntegTestCase.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/AbstractSqlIntegTestCase.java @@ -27,7 +27,6 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); - settings.put(XPackSettings.PROFILING_ENABLED.getKey(), false); settings.put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial"); return settings.build(); } diff --git a/x-pack/plugin/stack/qa/rest/build.gradle b/x-pack/plugin/stack/qa/rest/build.gradle index 78e1d9a293935..a5bdb388d610f 100644 --- a/x-pack/plugin/stack/qa/rest/build.gradle +++ b/x-pack/plugin/stack/qa/rest/build.gradle @@ -15,6 +15,5 @@ restResources { testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' } diff --git a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle index 5fb75b2353870..7c0eaf1450567 100644 --- a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle +++ b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle @@ -26,7 +26,6 @@ def remoteCluster = testClusters.register('remote-cluster') { setting 'node.roles', '[data,ingest,master]' setting 'xpack.security.enabled', 'true' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' user username: "test_user", password: "x-pack-test-password" @@ -41,7 +40,6 @@ testClusters.register('mixed-cluster') { nodes."mixed-cluster-1".setting 'node.roles', '[data,ingest,master,transform,remote_cluster_client]' setting 'xpack.security.enabled', 'true' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'cluster.remote.my_remote_cluster.seeds', { remoteCluster.get().getAllTransportPortURI().collect { "\"$it\"" }.toString() diff --git a/x-pack/plugin/watcher/qa/rest/build.gradle b/x-pack/plugin/watcher/qa/rest/build.gradle index c2ccb61333b3f..3f61bdcb3c2ed 100644 --- a/x-pack/plugin/watcher/qa/rest/build.gradle +++ b/x-pack/plugin/watcher/qa/rest/build.gradle @@ -25,7 +25,6 @@ testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG' } diff --git a/x-pack/plugin/watcher/qa/with-monitoring/build.gradle b/x-pack/plugin/watcher/qa/with-monitoring/build.gradle index db2702b86b633..6a2ef6fbd7bd5 100644 --- a/x-pack/plugin/watcher/qa/with-monitoring/build.gradle +++ b/x-pack/plugin/watcher/qa/with-monitoring/build.gradle @@ -12,7 +12,6 @@ testClusters.configureEach { setting 'xpack.monitoring.collection.enabled', 'true' setting 'xpack.monitoring.collection.interval', '1s' setting 'xpack.watcher.enabled', 'true' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' diff --git a/x-pack/plugin/watcher/qa/with-security/build.gradle b/x-pack/plugin/watcher/qa/with-security/build.gradle index c4011f716ade7..5c01573e09417 100644 --- a/x-pack/plugin/watcher/qa/with-security/build.gradle +++ b/x-pack/plugin/watcher/qa/with-security/build.gradle @@ -22,7 +22,6 @@ restResources { testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'true' // settings to test settings filtering on setting 'xpack.notification.email.account._email.smtp.host', 'host.domain' diff --git a/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java b/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java index e680d0c3c39b9..172a776b61242 100644 --- a/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java +++ b/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java @@ -37,7 +37,6 @@ public class CoreWithSecurityClientYamlTestSuiteIT extends ESClientYamlSuiteTest .module("x-pack-stack") .setting("xpack.security.enabled", "true") .setting("xpack.watcher.enabled", "false") - .setting("xpack.profiling.enabled", "false") .setting("xpack.ml.enabled", "false") .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.autoconfiguration.enabled", "false") diff --git a/x-pack/qa/kerberos-tests/build.gradle b/x-pack/qa/kerberos-tests/build.gradle index 1300cbfc373dc..536758dcfb71f 100644 --- a/x-pack/qa/kerberos-tests/build.gradle +++ b/x-pack/qa/kerberos-tests/build.gradle @@ -21,7 +21,6 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { setting 'xpack.security.enabled', 'true' setting 'xpack.security.authc.realms.file.file1.order', '0' setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.audit.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' // Kerberos realm diff --git a/x-pack/qa/mixed-tier-cluster/build.gradle b/x-pack/qa/mixed-tier-cluster/build.gradle index 1ea11aa56f5d5..bf05be45e18a0 100644 --- a/x-pack/qa/mixed-tier-cluster/build.gradle +++ b/x-pack/qa/mixed-tier-cluster/build.gradle @@ -19,10 +19,6 @@ BuildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter("7.9.0") && testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - // profiling plugin has been added in 8.7.0 - if (bwcVersion.onOrAfter('8.7.0')) { - setting 'xpack.profiling.enabled', 'false' - } setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' nodes."${baseName}-0".setting 'node.roles', '["master"]' diff --git a/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/GlobalCheckpointSyncActionIT.java b/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/GlobalCheckpointSyncActionIT.java index 490f1c3983fc3..12cfa8e48b3e4 100644 --- a/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/GlobalCheckpointSyncActionIT.java +++ b/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/GlobalCheckpointSyncActionIT.java @@ -28,7 +28,6 @@ public class GlobalCheckpointSyncActionIT extends ESRestTestCase { .nodes(2) .setting("xpack.security.enabled", "true") .setting("xpack.watcher.enabled", "false") - .setting("xpack.profiling.enabled", "false") .setting("xpack.ml.enabled", "false") .setting("xpack.license.self_generated.type", "trial") .rolesFile(Resource.fromClasspath("roles.yml")) diff --git a/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/RollupIT.java b/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/RollupIT.java index 5d607c19ccee3..91e273f7e34b4 100644 --- a/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/RollupIT.java +++ b/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/RollupIT.java @@ -43,7 +43,6 @@ public class RollupIT extends ESRestTestCase { .module("x-pack-rollup") .setting("xpack.security.enabled", "true") .setting("xpack.watcher.enabled", "false") - .setting("xpack.profiling.enabled", "false") .setting("xpack.ml.enabled", "false") .setting("xpack.license.self_generated.type", "trial") .user("super-user", "x-pack-super-password") diff --git a/x-pack/qa/reindex-tests-with-security/build.gradle b/x-pack/qa/reindex-tests-with-security/build.gradle index 4af80e850a952..fdb61e02f8186 100644 --- a/x-pack/qa/reindex-tests-with-security/build.gradle +++ b/x-pack/qa/reindex-tests-with-security/build.gradle @@ -33,7 +33,6 @@ testClusters.matching { it.name == "yamlRestTest" }.configureEach { setting 'reindex.remote.whitelist', '127.0.0.1:*' setting 'xpack.security.enabled', 'true' setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.http.ssl.enabled', 'true' setting 'xpack.security.http.ssl.certificate', 'http.crt' diff --git a/x-pack/qa/rolling-upgrade-basic/build.gradle b/x-pack/qa/rolling-upgrade-basic/build.gradle index dceb271f260c6..a4a995d09c647 100644 --- a/x-pack/qa/rolling-upgrade-basic/build.gradle +++ b/x-pack/qa/rolling-upgrade-basic/build.gradle @@ -19,10 +19,6 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - // profiling plugin has been added in 8.7.0 - if (bwcVersion.onOrAfter('8.7.0')) { - setting 'xpack.profiling.enabled', 'false' - } setting 'xpack.license.self_generated.type', 'basic' } diff --git a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle index 75b64cc1fc8a0..8b8a45fa52ec9 100644 --- a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle +++ b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle @@ -25,9 +25,6 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.watcher.enabled', 'false' - if (bwcVersion.onOrAfter('8.7.0')) { - setting 'xpack.profiling.enabled', 'false' - } setting 'xpack.license.self_generated.type', 'trial' } diff --git a/x-pack/qa/runtime-fields/build.gradle b/x-pack/qa/runtime-fields/build.gradle index 058568362c998..5bee9f08c2a73 100644 --- a/x-pack/qa/runtime-fields/build.gradle +++ b/x-pack/qa/runtime-fields/build.gradle @@ -43,7 +43,6 @@ subprojects { testDistribution = 'DEFAULT' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") } diff --git a/x-pack/qa/runtime-fields/with-security/build.gradle b/x-pack/qa/runtime-fields/with-security/build.gradle index ce016fa5d3308..e664ee888a313 100644 --- a/x-pack/qa/runtime-fields/with-security/build.gradle +++ b/x-pack/qa/runtime-fields/with-security/build.gradle @@ -9,7 +9,6 @@ dependencies { testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' rolesFile file('roles.yml') diff --git a/x-pack/qa/saml-idp-tests/build.gradle b/x-pack/qa/saml-idp-tests/build.gradle index 8efe8840e00c5..859977f7b337c 100644 --- a/x-pack/qa/saml-idp-tests/build.gradle +++ b/x-pack/qa/saml-idp-tests/build.gradle @@ -81,7 +81,6 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { setting 'xpack.security.authc.realms.native.native.order', '4' setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'logger.org.elasticsearch.xpack.security', 'TRACE' extraConfigFile 'idp-metadata.xml', file(outputDir + "/idp-metadata.xml") diff --git a/x-pack/qa/security-example-spi-extension/build.gradle b/x-pack/qa/security-example-spi-extension/build.gradle index 8258891be7bbc..92e403d43b4f7 100644 --- a/x-pack/qa/security-example-spi-extension/build.gradle +++ b/x-pack/qa/security-example-spi-extension/build.gradle @@ -35,7 +35,6 @@ testClusters.configureEach { setting 'xpack.security.authc.realms.custom_role_mapping.role_map.order', '3' setting 'xpack.security.enabled', 'true' setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' user username: "test_user", password: "x-pack-test-password" diff --git a/x-pack/qa/security-setup-password-tests/build.gradle b/x-pack/qa/security-setup-password-tests/build.gradle index c4bdb32225980..8f17e6d92291a 100644 --- a/x-pack/qa/security-setup-password-tests/build.gradle +++ b/x-pack/qa/security-setup-password-tests/build.gradle @@ -13,7 +13,6 @@ tasks.named("javaRestTest").configure { testClusters.matching { it.name == "javaRestTest" }.configureEach { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'true' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' // Setup passwords doesn't work when there is a password auto-configured on first start setting 'xpack.security.autoconfiguration.enabled', 'false' diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle index 8a32db00f3fbe..1e8e88b91c330 100644 --- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle +++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle @@ -60,7 +60,6 @@ testClusters.matching { it.name == "yamlRestTest" }.configureEach { keystore 'xpack.security.http.ssl.secure_key_passphrase', 'testnode' setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' // copy keystores, keys and certificates into config/ extraConfigFile nodeKeystore.name, nodeKeystore extraConfigFile nodeKey.name, nodeKey diff --git a/x-pack/qa/smoke-test-security-with-mustache/build.gradle b/x-pack/qa/smoke-test-security-with-mustache/build.gradle index 02fd97ddbb5a0..373e6966df512 100644 --- a/x-pack/qa/smoke-test-security-with-mustache/build.gradle +++ b/x-pack/qa/smoke-test-security-with-mustache/build.gradle @@ -13,7 +13,6 @@ restResources { testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' user username: "test_admin", password: "x-pack-test-password" diff --git a/x-pack/qa/third-party/jira/build.gradle b/x-pack/qa/third-party/jira/build.gradle index ce16072bfe778..b7268af807535 100644 --- a/x-pack/qa/third-party/jira/build.gradle +++ b/x-pack/qa/third-party/jira/build.gradle @@ -43,7 +43,6 @@ if (!jiraUrl && !jiraUser && !jiraPassword && !jiraProject) { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG' setting 'xpack.notification.jira.account.test.issue_defaults.issuetype.name', 'Bug' diff --git a/x-pack/qa/third-party/pagerduty/build.gradle b/x-pack/qa/third-party/pagerduty/build.gradle index 8f21a7a3f34cf..4b5a0bbeeeb4a 100644 --- a/x-pack/qa/third-party/pagerduty/build.gradle +++ b/x-pack/qa/third-party/pagerduty/build.gradle @@ -22,7 +22,6 @@ if (!pagerDutyServiceKey) { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG' keystore 'xpack.notification.pagerduty.account.test_account.secure_service_api_key', pagerDutyServiceKey diff --git a/x-pack/qa/third-party/slack/build.gradle b/x-pack/qa/third-party/slack/build.gradle index 9b3e839156d57..b2b0478da0471 100644 --- a/x-pack/qa/third-party/slack/build.gradle +++ b/x-pack/qa/third-party/slack/build.gradle @@ -22,7 +22,6 @@ if (!slackUrl) { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.ml.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG' keystore 'xpack.notification.slack.account.test_account.secure_url', slackUrl diff --git a/x-pack/qa/xpack-prefix-rest-compat/build.gradle b/x-pack/qa/xpack-prefix-rest-compat/build.gradle index 465ebf31fb7c4..d609ede0efd2e 100644 --- a/x-pack/qa/xpack-prefix-rest-compat/build.gradle +++ b/x-pack/qa/xpack-prefix-rest-compat/build.gradle @@ -82,7 +82,6 @@ testClusters.configureEach { setting 'xpack.ml.enabled', 'true' setting 'xpack.security.enabled', 'true' setting 'xpack.watcher.enabled', 'false' - setting 'xpack.profiling.enabled', 'false' setting 'xpack.security.authc.token.enabled', 'true' setting 'xpack.security.authc.api_key.enabled', 'true' setting 'xpack.security.transport.ssl.enabled', 'true' From c9ffb25d28b4689a97b8fe383a51a3bd1dad6192 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Fri, 12 May 2023 07:49:03 +0200 Subject: [PATCH 33/36] Restore unintentional whitespace changes --- x-pack/plugin/ilm/qa/multi-cluster/build.gradle | 2 +- x-pack/plugin/ilm/qa/with-security/build.gradle | 1 + x-pack/qa/rolling-upgrade-basic/build.gradle | 2 +- x-pack/qa/rolling-upgrade-multi-cluster/build.gradle | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle index 49c2629b58580..a1a2a5129e3f7 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle @@ -59,4 +59,4 @@ tasks.named("check").configure { dependsOn 'follow-cluster' } // Security is explicitly disabled for follow-cluster and leader-cluster, do not run these in FIPS mode tasks.withType(Test).configureEach { enabled = BuildParams.inFipsJvm == false -} +} \ No newline at end of file diff --git a/x-pack/plugin/ilm/qa/with-security/build.gradle b/x-pack/plugin/ilm/qa/with-security/build.gradle index 01d48b8e1812b..7aaa43f14fe3f 100644 --- a/x-pack/plugin/ilm/qa/with-security/build.gradle +++ b/x-pack/plugin/ilm/qa/with-security/build.gradle @@ -9,6 +9,7 @@ dependencies { testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.watcher.enabled', 'false' + setting 'xpack.watcher.enabled', 'false' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' rolesFile file('roles.yml') diff --git a/x-pack/qa/rolling-upgrade-basic/build.gradle b/x-pack/qa/rolling-upgrade-basic/build.gradle index a4a995d09c647..a7ea1695c477a 100644 --- a/x-pack/qa/rolling-upgrade-basic/build.gradle +++ b/x-pack/qa/rolling-upgrade-basic/build.gradle @@ -79,4 +79,4 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> // Security is explicitly disabled, do not run tests in FIPS mode tasks.withType(Test).configureEach { enabled = BuildParams.inFipsJvm == false -} +} \ No newline at end of file diff --git a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle index 8b8a45fa52ec9..969ba23e19254 100644 --- a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle +++ b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle @@ -1,4 +1,4 @@ - import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.internal-testclusters' From d0fdd70c8a3ee3a4243267524abf2302b431dd79 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Fri, 12 May 2023 16:15:52 +0200 Subject: [PATCH 34/36] Use list continuation in docs --- docs/reference/snapshot-restore/restore-snapshot.asciidoc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/reference/snapshot-restore/restore-snapshot.asciidoc b/docs/reference/snapshot-restore/restore-snapshot.asciidoc index 48da6e7293fc5..0fca813143911 100644 --- a/docs/reference/snapshot-restore/restore-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/restore-snapshot.asciidoc @@ -366,17 +366,15 @@ POST _watcher/_start + -- * Universal Profiling - ++ Check if Universal Profiling index template management is enabled: - + [source,console] ---- GET /_cluster/settings?filter_path=**.xpack.profiling.templates.enabled&include_defaults=true ---- - ++ If the value is `true`, disable Universal Profiling index template management: - + [source,console] ---- @@ -504,7 +502,7 @@ POST _watcher/_start -- * Universal Profiling - ++ If the value was `true` initially, enable Universal Profiling index template management again, otherwise skip this step: + [source,console] From 7dc6f1ca856384d427785431685813a769ab0c8b Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Tue, 16 May 2023 13:11:40 +0200 Subject: [PATCH 35/36] Address review comments --- .../elasticsearch/xpack/profiler/ilm-policy/profiling.json | 7 +++---- .../elasticsearch/xpack/profiler/ProfilingTestCase.java | 2 -- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json index bacbf33ed41a6..9d01e97e430d2 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json @@ -5,7 +5,8 @@ "actions": { "rollover": { "max_primary_shard_size": "50gb", - "max_age": "7d" + "max_age": "7d", + "min_docs": 1 }, "set_priority": { "priority": 100 @@ -26,9 +27,7 @@ "delete": { "min_age": "60d", "actions": { - "delete": { - "delete_searchable_snapshot": true - } + "delete": {} } } }, diff --git a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java index c12a234bed4d1..14b624d7bc1ec 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java +++ b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java @@ -84,10 +84,8 @@ private void indexDoc(String index, String id, Map source) { protected abstract boolean useOnlyAllEvents(); protected void waitForIndices() throws Exception { - // block until the templates are installed assertBusy(() -> { ClusterState state = client().admin().cluster().prepareState().get().getState(); - // simple check whether at least one of the indices is present assertTrue( "Timed out waiting for the indices to be created", state.metadata().indices().keySet().containsAll(ProfilingIndexManager.INDICES_AND_ALIASES.keySet()) From f1dee21951b7bae18cd687618df69e5cd7987cd1 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Tue, 16 May 2023 15:49:24 +0200 Subject: [PATCH 36/36] Address review comments --- .../xpack/profiler/ProfilingIndexManager.java | 19 +++++++++---------- .../ProfilingIndexTemplateRegistry.java | 2 +- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java index 5e5dcfaeb8b4f..ea5748be4c334 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java @@ -28,7 +28,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; @@ -69,7 +68,7 @@ public class ProfilingIndexManager implements ClusterStateListener, Closeable { private final ThreadPool threadPool; private final Client client; private final ClusterService clusterService; - private final ConcurrentMap indexCreationInProgress = new ConcurrentHashMap<>(); + private final ConcurrentMap creationInProgressPerIndex = new ConcurrentHashMap<>(); private volatile boolean templatesEnabled; public ProfilingIndexManager(ThreadPool threadPool, Client client, ClusterService clusterService) { @@ -121,24 +120,23 @@ public void clusterChanged(ClusterChangedEvent event) { } protected boolean isAllTemplatesCreated(ClusterChangedEvent event) { - return ProfilingIndexTemplateRegistry.isAllTemplatesCreated(event.state()); + return ProfilingIndexTemplateRegistry.areAllTemplatesCreated(event.state()); } private void addIndicesIfMissing(ClusterState state) { - Optional> maybeMeta = Optional.ofNullable(state.metadata().indices()); + Map indicesMetadata = state.metadata().indices(); for (Map.Entry idxAlias : INDICES_AND_ALIASES.entrySet()) { String index = idxAlias.getKey(); String alias = idxAlias.getValue(); - final AtomicBoolean creationCheck = indexCreationInProgress.computeIfAbsent(index, key -> new AtomicBoolean(false)); - if (creationCheck.compareAndSet(false, true)) { - final boolean indexNeedsToBeCreated = maybeMeta.flatMap(idxMeta -> Optional.ofNullable(idxMeta.get(index))) - .isPresent() == false; + final AtomicBoolean creationInProgress = creationInProgressPerIndex.computeIfAbsent(index, key -> new AtomicBoolean(false)); + if (creationInProgress.compareAndSet(false, true)) { + final boolean indexNeedsToBeCreated = indicesMetadata == null || indicesMetadata.get(index) == null; if (indexNeedsToBeCreated) { logger.debug("adding index [{}], because it doesn't exist", index); - putIndex(index, alias, creationCheck); + putIndex(index, alias, creationInProgress); } else { logger.trace("not adding index [{}], because it already exists", index); - creationCheck.set(false); + creationInProgress.set(false); } } } @@ -159,6 +157,7 @@ private void putIndex(final String index, final String alias, final AtomicBoolea } catch (Exception ex) { creationCheck.set(false); onPutIndexFailure(index, ex); + return; } } request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java index 3019c1f5bb674..2a4e616001a0b 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java @@ -208,7 +208,7 @@ protected Map getComposableTemplateConfigs() { return templatesEnabled ? COMPOSABLE_INDEX_TEMPLATE_CONFIGS : Collections.emptyMap(); } - public static boolean isAllTemplatesCreated(ClusterState state) { + public static boolean areAllTemplatesCreated(ClusterState state) { for (String componentTemplate : COMPONENT_TEMPLATE_CONFIGS.keySet()) { if (state.metadata().componentTemplates().containsKey(componentTemplate) == false) { return false;