Skip to content

Commit 51cee9e

Browse files
original-brownbearoriginal-brownbear
authored
Fix IndexTemplateRegistry Deserializing Templates etc. During CS Updates (#80086)
This showed up pretty hot in profiling CS updates for many shards. We are loading+deserializing lifecycle policies and composable index templates on every CS update which gets expensive when you do a lot of them and needlessly duplicates the objects as well. Also, just throwing an unchecked exception on failure to deserialize one of these static resources seems broken as that's always a bug. This PR moves all these things to just be statically loaded on classload and makes the index template registry not show up in profiling CS updates any longer.
1 parent 06202d6 commit 51cee9e

File tree

30 files changed

+417
-460
lines changed

30 files changed

+417
-460
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java‎

Lines changed: 56 additions & 94 deletions
Large diffs are not rendered by default.

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,38 @@
88
package org.elasticsearch.xpack.core.template;
99

1010
import org.elasticsearch.xcontent.NamedXContentRegistry;
11+
import org.elasticsearch.xcontent.ParseField;
12+
import org.elasticsearch.xpack.core.ilm.DeleteAction;
13+
import org.elasticsearch.xpack.core.ilm.ForceMergeAction;
14+
import org.elasticsearch.xpack.core.ilm.LifecycleAction;
1115
import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
1216
import org.elasticsearch.xpack.core.ilm.LifecyclePolicyUtils;
17+
import org.elasticsearch.xpack.core.ilm.LifecycleType;
18+
import org.elasticsearch.xpack.core.ilm.RolloverAction;
19+
import org.elasticsearch.xpack.core.ilm.ShrinkAction;
20+
import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType;
21+
22+
import java.util.List;
1323

1424
/**
1525
* Describes an index lifecycle policy to be loaded from a resource file for use with an {@link IndexTemplateRegistry}.
1626
*/
1727
public class LifecyclePolicyConfig {
1828

29+
public static final NamedXContentRegistry DEFAULT_X_CONTENT_REGISTRY = new NamedXContentRegistry(
30+
List.of(
31+
new NamedXContentRegistry.Entry(
32+
LifecycleType.class,
33+
new ParseField(TimeseriesLifecycleType.TYPE),
34+
(p) -> TimeseriesLifecycleType.INSTANCE
35+
),
36+
new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse),
37+
new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse),
38+
new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse),
39+
new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse)
40+
)
41+
);
42+
1943
private final String policyName;
2044
private final String fileName;
2145

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java‎

Lines changed: 60 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,23 @@
88
package org.elasticsearch.xpack.deprecation.logging;
99

1010
import org.elasticsearch.client.Client;
11+
import org.elasticsearch.cluster.metadata.ComponentTemplate;
12+
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
1113
import org.elasticsearch.cluster.service.ClusterService;
1214
import org.elasticsearch.common.settings.Settings;
1315
import org.elasticsearch.threadpool.ThreadPool;
1416
import org.elasticsearch.xcontent.NamedXContentRegistry;
17+
import org.elasticsearch.xcontent.XContentParserConfiguration;
18+
import org.elasticsearch.xcontent.json.JsonXContent;
19+
import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
1520
import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
1621
import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
1722
import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig;
1823

24+
import java.io.IOException;
25+
import java.util.HashMap;
1926
import java.util.List;
27+
import java.util.Map;
2028

2129
import static org.elasticsearch.xpack.core.ClientHelper.DEPRECATION_ORIGIN;
2230

@@ -35,32 +43,6 @@ public class DeprecationIndexingTemplateRegistry extends IndexTemplateRegistry {
3543
public static final String DEPRECATION_INDEXING_TEMPLATE_NAME = ".deprecation-indexing-template";
3644
public static final String DEPRECATION_INDEXING_POLICY_NAME = ".deprecation-indexing-ilm-policy";
3745

38-
public static final IndexTemplateConfig DEPRECATION_INDEXING_MAPPINGS = new IndexTemplateConfig(
39-
DEPRECATION_INDEXING_MAPPINGS_NAME,
40-
"/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json",
41-
INDEX_TEMPLATE_VERSION,
42-
DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE
43-
);
44-
45-
public static final IndexTemplateConfig DEPRECATION_INDEXING_SETTINGS = new IndexTemplateConfig(
46-
DEPRECATION_INDEXING_SETTINGS_NAME,
47-
"/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json",
48-
INDEX_TEMPLATE_VERSION,
49-
DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE
50-
);
51-
52-
public static final IndexTemplateConfig DEPRECATION_INDEXING_INDEX_TEMPLATE = new IndexTemplateConfig(
53-
DEPRECATION_INDEXING_TEMPLATE_NAME,
54-
"/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json",
55-
INDEX_TEMPLATE_VERSION,
56-
DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE
57-
);
58-
59-
public static final LifecyclePolicyConfig DEPRECATION_INDEXING_HISTORY_POLICY = new LifecyclePolicyConfig(
60-
DEPRECATION_INDEXING_POLICY_NAME,
61-
"/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json"
62-
);
63-
6446
public DeprecationIndexingTemplateRegistry(
6547
Settings nodeSettings,
6648
ClusterService clusterService,
@@ -71,19 +53,65 @@ public DeprecationIndexingTemplateRegistry(
7153
super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
7254
}
7355

56+
private static final Map<String, ComponentTemplate> COMPONENT_TEMPLATE_CONFIGS;
57+
58+
static {
59+
final Map<String, ComponentTemplate> componentTemplates = new HashMap<>();
60+
for (IndexTemplateConfig config : List.of(
61+
new IndexTemplateConfig(
62+
DEPRECATION_INDEXING_MAPPINGS_NAME,
63+
"/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json",
64+
INDEX_TEMPLATE_VERSION,
65+
DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE
66+
),
67+
new IndexTemplateConfig(
68+
DEPRECATION_INDEXING_SETTINGS_NAME,
69+
"/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json",
70+
INDEX_TEMPLATE_VERSION,
71+
DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE
72+
)
73+
)) {
74+
try {
75+
componentTemplates.put(
76+
config.getTemplateName(),
77+
ComponentTemplate.parse(JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, config.loadBytes()))
78+
);
79+
} catch (IOException e) {
80+
throw new AssertionError(e);
81+
}
82+
}
83+
COMPONENT_TEMPLATE_CONFIGS = Map.copyOf(componentTemplates);
84+
}
85+
7486
@Override
75-
protected List<IndexTemplateConfig> getComponentTemplateConfigs() {
76-
return List.of(DEPRECATION_INDEXING_MAPPINGS, DEPRECATION_INDEXING_SETTINGS);
87+
protected Map<String, ComponentTemplate> getComponentTemplateConfigs() {
88+
return COMPONENT_TEMPLATE_CONFIGS;
7789
}
7890

91+
private static final Map<String, ComposableIndexTemplate> COMPOSABLE_INDEX_TEMPLATE_CONFIGS = parseComposableTemplates(
92+
new IndexTemplateConfig(
93+
DEPRECATION_INDEXING_TEMPLATE_NAME,
94+
"/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json",
95+
INDEX_TEMPLATE_VERSION,
96+
DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE
97+
)
98+
);
99+
79100
@Override
80-
protected List<IndexTemplateConfig> getComposableTemplateConfigs() {
81-
return List.of(DEPRECATION_INDEXING_INDEX_TEMPLATE);
101+
protected Map<String, ComposableIndexTemplate> getComposableTemplateConfigs() {
102+
return COMPOSABLE_INDEX_TEMPLATE_CONFIGS;
82103
}
83104

105+
private static final List<LifecyclePolicy> LIFECYCLE_POLICIES = List.of(
106+
new LifecyclePolicyConfig(
107+
DEPRECATION_INDEXING_POLICY_NAME,
108+
"/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json"
109+
).load(LifecyclePolicyConfig.DEFAULT_X_CONTENT_REGISTRY)
110+
);
111+
84112
@Override
85-
protected List<LifecyclePolicyConfig> getPolicyConfigs() {
86-
return List.of(DEPRECATION_INDEXING_HISTORY_POLICY);
113+
protected List<LifecyclePolicy> getPolicyConfigs() {
114+
return LIFECYCLE_POLICIES;
87115
}
88116

89117
@Override

‎x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/FleetTemplateRegistry.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,18 @@
1313
import org.elasticsearch.threadpool.ThreadPool;
1414
import org.elasticsearch.xcontent.NamedXContentRegistry;
1515
import org.elasticsearch.xpack.core.ClientHelper;
16+
import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
1617
import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
1718
import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig;
1819

1920
import java.util.List;
2021

2122
public class FleetTemplateRegistry extends IndexTemplateRegistry {
2223

23-
public static final LifecyclePolicyConfig FLEET_ACTIONS_RESULTS_POLICY = new LifecyclePolicyConfig(
24-
".fleet-actions-results-ilm-policy",
25-
"/fleet-actions-results-ilm-policy.json"
24+
private static final List<LifecyclePolicy> LIFECYCLE_POLICIES = List.of(
25+
new LifecyclePolicyConfig(".fleet-actions-results-ilm-policy", "/fleet-actions-results-ilm-policy.json").load(
26+
LifecyclePolicyConfig.DEFAULT_X_CONTENT_REGISTRY
27+
)
2628
);
2729

2830
public FleetTemplateRegistry(
@@ -41,7 +43,7 @@ protected String getOrigin() {
4143
}
4244

4345
@Override
44-
protected List<LifecyclePolicyConfig> getPolicyConfigs() {
45-
return List.of(FLEET_ACTIONS_RESULTS_POLICY);
46+
protected List<LifecyclePolicy> getPolicyConfigs() {
47+
return LIFECYCLE_POLICIES;
4648
}
4749
}

‎x-pack/plugin/ilm/qa/multi-node/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ apply plugin: 'elasticsearch.internal-java-rest-test'
55

66
dependencies {
77
javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
8+
javaRestTestImplementation project(xpackModule('ilm'))
89
javaRestTestImplementation project(":client:rest-high-level")
910
}
1011

‎x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,9 @@
5353
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
5454
import static org.elasticsearch.xpack.TimeSeriesRestDriver.createComposableTemplate;
5555
import static org.elasticsearch.xpack.TimeSeriesRestDriver.getStepKeyForIndex;
56-
import static org.elasticsearch.xpack.core.slm.history.SnapshotHistoryItem.CREATE_OPERATION;
57-
import static org.elasticsearch.xpack.core.slm.history.SnapshotHistoryItem.DELETE_OPERATION;
58-
import static org.elasticsearch.xpack.core.slm.history.SnapshotHistoryStore.SLM_HISTORY_DATA_STREAM;
56+
import static org.elasticsearch.xpack.slm.history.SnapshotHistoryItem.CREATE_OPERATION;
57+
import static org.elasticsearch.xpack.slm.history.SnapshotHistoryItem.DELETE_OPERATION;
58+
import static org.elasticsearch.xpack.slm.history.SnapshotHistoryStore.SLM_HISTORY_DATA_STREAM;
5959
import static org.hamcrest.Matchers.anyOf;
6060
import static org.hamcrest.Matchers.containsString;
6161
import static org.hamcrest.Matchers.empty;

‎x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@
5858
import java.util.concurrent.atomic.AtomicReference;
5959

6060
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
61-
import static org.elasticsearch.xpack.core.slm.history.SnapshotHistoryStore.SLM_HISTORY_DATA_STREAM;
61+
import static org.elasticsearch.xpack.slm.history.SnapshotHistoryStore.SLM_HISTORY_DATA_STREAM;
6262
import static org.hamcrest.Matchers.anyOf;
6363
import static org.hamcrest.Matchers.empty;
6464
import static org.hamcrest.Matchers.equalTo;

‎x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,8 +81,6 @@
8181
import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction;
8282
import org.elasticsearch.xpack.core.slm.action.StartSLMAction;
8383
import org.elasticsearch.xpack.core.slm.action.StopSLMAction;
84-
import org.elasticsearch.xpack.core.slm.history.SnapshotHistoryStore;
85-
import org.elasticsearch.xpack.core.slm.history.SnapshotLifecycleTemplateRegistry;
8684
import org.elasticsearch.xpack.ilm.action.RestDeleteLifecycleAction;
8785
import org.elasticsearch.xpack.ilm.action.RestExplainLifecycleAction;
8886
import org.elasticsearch.xpack.ilm.action.RestGetLifecycleAction;
@@ -131,6 +129,8 @@
131129
import org.elasticsearch.xpack.slm.action.TransportPutSnapshotLifecycleAction;
132130
import org.elasticsearch.xpack.slm.action.TransportStartSLMAction;
133131
import org.elasticsearch.xpack.slm.action.TransportStopSLMAction;
132+
import org.elasticsearch.xpack.slm.history.SnapshotHistoryStore;
133+
import org.elasticsearch.xpack.slm.history.SnapshotLifecycleTemplateRegistry;
134134

135135
import java.io.IOException;
136136
import java.time.Clock;
@@ -145,6 +145,9 @@
145145
import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN;
146146

147147
public class IndexLifecycle extends Plugin implements ActionPlugin {
148+
149+
public static final List<NamedXContentRegistry.Entry> NAMED_X_CONTENT_ENTRIES = xContentEntries();
150+
148151
private final SetOnce<IndexLifecycleService> indexLifecycleInitialisationService = new SetOnce<>();
149152
private final SetOnce<ILMHistoryStore> ilmHistoryStore = new SetOnce<>();
150153
private final SetOnce<SnapshotLifecycleService> snapshotLifecycleService = new SetOnce<>();
@@ -272,6 +275,10 @@ public List<Entry> getNamedWriteables() {
272275

273276
@Override
274277
public List<NamedXContentRegistry.Entry> getNamedXContent() {
278+
return NAMED_X_CONTENT_ENTRIES;
279+
}
280+
281+
private static List<NamedXContentRegistry.Entry> xContentEntries() {
275282
List<NamedXContentRegistry.Entry> entries = new ArrayList<>(
276283
Arrays.asList(
277284
// Custom Metadata
@@ -320,8 +327,7 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
320327
new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RollupILMAction.NAME), RollupILMAction::parse)
321328
);
322329
}
323-
324-
return entries;
330+
return List.copyOf(entries);
325331
}
326332

327333
@Override

‎x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java‎

Lines changed: 19 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,22 @@
88
package org.elasticsearch.xpack.ilm.history;
99

1010
import org.elasticsearch.client.Client;
11+
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
1112
import org.elasticsearch.cluster.service.ClusterService;
1213
import org.elasticsearch.common.settings.Settings;
1314
import org.elasticsearch.threadpool.ThreadPool;
1415
import org.elasticsearch.xcontent.NamedXContentRegistry;
1516
import org.elasticsearch.xpack.core.ClientHelper;
17+
import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
1618
import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
1719
import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
1820
import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
1921
import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig;
22+
import org.elasticsearch.xpack.ilm.IndexLifecycle;
2023

2124
import java.util.Collections;
2225
import java.util.List;
26+
import java.util.Map;
2327

2428
/**
2529
* The {@link ILMHistoryTemplateRegistry} class sets up and configures an ILM policy and index
@@ -44,18 +48,6 @@ protected boolean requiresMasterNode() {
4448
return true;
4549
}
4650

47-
public static final IndexTemplateConfig TEMPLATE_ILM_HISTORY = new IndexTemplateConfig(
48-
ILM_TEMPLATE_NAME,
49-
"/ilm-history.json",
50-
INDEX_TEMPLATE_VERSION,
51-
ILM_TEMPLATE_VERSION_VARIABLE
52-
);
53-
54-
public static final LifecyclePolicyConfig ILM_HISTORY_POLICY = new LifecyclePolicyConfig(
55-
ILM_POLICY_NAME,
56-
"/ilm-history-ilm-policy.json"
57-
);
58-
5951
private final boolean ilmHistoryEnabled;
6052

6153
public ILMHistoryTemplateRegistry(
@@ -69,22 +61,28 @@ public ILMHistoryTemplateRegistry(
6961
this.ilmHistoryEnabled = LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING.get(nodeSettings);
7062
}
7163

64+
private static final Map<String, ComposableIndexTemplate> COMPOSABLE_INDEX_TEMPLATE_CONFIGS = parseComposableTemplates(
65+
new IndexTemplateConfig(ILM_TEMPLATE_NAME, "/ilm-history.json", INDEX_TEMPLATE_VERSION, ILM_TEMPLATE_VERSION_VARIABLE)
66+
);
67+
7268
@Override
73-
protected List<IndexTemplateConfig> getComposableTemplateConfigs() {
69+
protected Map<String, ComposableIndexTemplate> getComposableTemplateConfigs() {
7470
if (this.ilmHistoryEnabled) {
75-
return Collections.singletonList(TEMPLATE_ILM_HISTORY);
71+
return COMPOSABLE_INDEX_TEMPLATE_CONFIGS;
7672
} else {
77-
return Collections.emptyList();
73+
return Map.of();
7874
}
7975
}
8076

77+
private static final List<LifecyclePolicy> LIFECYCLE_POLICIES = List.of(
78+
new LifecyclePolicyConfig(ILM_POLICY_NAME, "/ilm-history-ilm-policy.json").load(
79+
new NamedXContentRegistry(IndexLifecycle.NAMED_X_CONTENT_ENTRIES)
80+
)
81+
);
82+
8183
@Override
82-
protected List<LifecyclePolicyConfig> getPolicyConfigs() {
83-
if (this.ilmHistoryEnabled) {
84-
return Collections.singletonList(ILM_HISTORY_POLICY);
85-
} else {
86-
return Collections.emptyList();
87-
}
84+
protected List<LifecyclePolicy> getPolicyConfigs() {
85+
return this.ilmHistoryEnabled ? LIFECYCLE_POLICIES : Collections.emptyList();
8886
}
8987

9088
@Override

‎x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,9 @@
3333
import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
3434
import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata;
3535
import org.elasticsearch.xpack.core.slm.SnapshotLifecycleStats;
36-
import org.elasticsearch.xpack.core.slm.history.SnapshotHistoryItem;
37-
import org.elasticsearch.xpack.core.slm.history.SnapshotHistoryStore;
3836
import org.elasticsearch.xpack.ilm.LifecyclePolicySecurityClient;
37+
import org.elasticsearch.xpack.slm.history.SnapshotHistoryItem;
38+
import org.elasticsearch.xpack.slm.history.SnapshotHistoryStore;
3939

4040
import java.io.IOException;
4141
import java.time.Instant;

0 commit comments

Comments
 (0)