Skip to content

Commit d43cbda

Browse files
davidkyledavidkyle
authored
[ML] ensure the ml-config index (#36792) (#36832)
1 parent f2a5373 commit d43cbda

File tree

10 files changed

+305
-70
lines changed

10 files changed

+305
-70
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
*/
1111
public final class AnomalyDetectorsIndex {
1212

13+
public static final int CONFIG_INDEX_MAX_RESULTS_WINDOW = 10_000;
14+
1315
private AnomalyDetectorsIndex() {
1416
}
1517

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -670,7 +670,9 @@ public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDat
670670
// least possible burden on Elasticsearch
671671
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
672672
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
673-
.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delayedNodeTimeOutSetting))
673+
.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delayedNodeTimeOutSetting)
674+
.put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(),
675+
AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW))
674676
.version(Version.CURRENT.id)
675677
.putMapping(ElasticsearchMappings.DOC_TYPE, Strings.toString(configMapping))
676678
.build();

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheck.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,15 @@
77

88
import org.elasticsearch.Version;
99
import org.elasticsearch.cluster.ClusterState;
10+
import org.elasticsearch.cluster.routing.IndexRoutingTable;
1011
import org.elasticsearch.cluster.service.ClusterService;
1112
import org.elasticsearch.common.settings.Setting;
1213
import org.elasticsearch.common.settings.Settings;
1314
import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
1415
import org.elasticsearch.xpack.core.ml.MlMetadata;
1516
import org.elasticsearch.xpack.core.ml.MlTasks;
1617
import org.elasticsearch.xpack.core.ml.job.config.Job;
18+
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
1719

1820
/**
1921
* Checks whether migration can start and whether ML resources (e.g. jobs, datafeeds)
@@ -37,10 +39,12 @@ private void setConfigMigrationEnabled(boolean configMigrationEnabled) {
3739
this.isConfigMigrationEnabled = configMigrationEnabled;
3840
}
3941

42+
4043
/**
4144
* Can migration start? Returns:
4245
* False if config migration is disabled via the setting {@link #ENABLE_CONFIG_MIGRATION}
4346
* False if the min node version of the cluster is before {@link #MIN_NODE_VERSION}
47+
* False if the .ml-config index shards are not active
4448
* True otherwise
4549
* @param clusterState The cluster state
4650
* @return A boolean that dictates if config migration can start
@@ -54,12 +58,26 @@ public boolean canStartMigration(ClusterState clusterState) {
5458
if (minNodeVersion.before(MIN_NODE_VERSION)) {
5559
return false;
5660
}
61+
62+
return mlConfigIndexIsAllocated(clusterState);
63+
}
64+
65+
static boolean mlConfigIndexIsAllocated(ClusterState clusterState) {
66+
if (clusterState.metaData().hasIndex(AnomalyDetectorsIndex.configIndexName()) == false) {
67+
return false;
68+
}
69+
70+
IndexRoutingTable routingTable = clusterState.getRoutingTable().index(AnomalyDetectorsIndex.configIndexName());
71+
if (routingTable == null || routingTable.allPrimaryShardsActive() == false) {
72+
return false;
73+
}
5774
return true;
5875
}
5976

6077
/**
6178
* Is the job a eligible for migration? Returns:
6279
* False if {@link #canStartMigration(ClusterState)} returns {@code false}
80+
* False if the job is not in the cluster state
6381
* False if the {@link Job#isDeleting()}
6482
* False if the job has a persistent task
6583
* True otherwise i.e. the job is present, not deleting

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java‎

Lines changed: 57 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
import org.elasticsearch.action.ActionListener;
1212
import org.elasticsearch.action.DocWriteRequest;
1313
import org.elasticsearch.action.DocWriteResponse;
14+
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
15+
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
1416
import org.elasticsearch.action.bulk.BulkItemResponse;
1517
import org.elasticsearch.action.bulk.BulkRequestBuilder;
1618
import org.elasticsearch.action.bulk.BulkResponse;
@@ -21,6 +23,7 @@
2123
import org.elasticsearch.client.Client;
2224
import org.elasticsearch.cluster.ClusterState;
2325
import org.elasticsearch.cluster.ClusterStateUpdateTask;
26+
import org.elasticsearch.cluster.metadata.IndexMetaData;
2427
import org.elasticsearch.cluster.metadata.MetaData;
2528
import org.elasticsearch.cluster.service.ClusterService;
2629
import org.elasticsearch.common.settings.Settings;
@@ -29,6 +32,7 @@
2932
import org.elasticsearch.common.xcontent.ToXContentObject;
3033
import org.elasticsearch.common.xcontent.XContentBuilder;
3134
import org.elasticsearch.common.xcontent.XContentFactory;
35+
import org.elasticsearch.index.IndexSettings;
3236
import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
3337
import org.elasticsearch.xpack.core.ml.MlMetadata;
3438
import org.elasticsearch.xpack.core.ml.MlTasks;
@@ -126,19 +130,11 @@ public MlConfigMigrator(Settings settings, Client client, ClusterService cluster
126130
* @param listener The success listener
127131
*/
128132
public void migrateConfigsWithoutTasks(ClusterState clusterState, ActionListener<Boolean> listener) {
129-
130-
if (migrationEligibilityCheck.canStartMigration(clusterState) == false) {
131-
listener.onResponse(false);
132-
return;
133-
}
134-
135133
if (migrationInProgress.compareAndSet(false, true) == false) {
136134
listener.onResponse(Boolean.FALSE);
137135
return;
138136
}
139137

140-
logger.debug("migrating ml configurations");
141-
142138
ActionListener<Boolean> unMarkMigrationInProgress = ActionListener.wrap(
143139
response -> {
144140
migrationInProgress.set(false);
@@ -150,19 +146,34 @@ public void migrateConfigsWithoutTasks(ClusterState clusterState, ActionListener
150146
}
151147
);
152148

149+
List<JobsAndDatafeeds> batches = splitInBatches(clusterState);
150+
if (batches.isEmpty()) {
151+
unMarkMigrationInProgress.onResponse(Boolean.FALSE);
152+
return;
153+
}
154+
155+
if (clusterState.metaData().hasIndex(AnomalyDetectorsIndex.configIndexName()) == false) {
156+
createConfigIndex(ActionListener.wrap(
157+
response -> {
158+
unMarkMigrationInProgress.onResponse(Boolean.FALSE);
159+
},
160+
unMarkMigrationInProgress::onFailure
161+
));
162+
return;
163+
}
164+
165+
if (migrationEligibilityCheck.canStartMigration(clusterState) == false) {
166+
unMarkMigrationInProgress.onResponse(Boolean.FALSE);
167+
return;
168+
}
169+
153170
snapshotMlMeta(MlMetadata.getMlMetadata(clusterState), ActionListener.wrap(
154-
response -> {
155-
// We have successfully snapshotted the ML configs so we don't need to try again
156-
tookConfigSnapshot.set(true);
157-
158-
List<JobsAndDatafeeds> batches = splitInBatches(clusterState);
159-
if (batches.isEmpty()) {
160-
unMarkMigrationInProgress.onResponse(Boolean.FALSE);
161-
return;
162-
}
163-
migrateBatches(batches, unMarkMigrationInProgress);
164-
},
165-
unMarkMigrationInProgress::onFailure
171+
response -> {
172+
// We have successfully snapshotted the ML configs so we don't need to try again
173+
tookConfigSnapshot.set(true);
174+
migrateBatches(batches, unMarkMigrationInProgress);
175+
},
176+
unMarkMigrationInProgress::onFailure
166177
));
167178
}
168179

@@ -296,13 +307,15 @@ static RemovalResult removeJobsAndDatafeeds(List<String> jobsToRemove, List<Stri
296307
private void addJobIndexRequests(Collection<Job> jobs, BulkRequestBuilder bulkRequestBuilder) {
297308
ToXContent.Params params = new ToXContent.MapParams(JobConfigProvider.TO_XCONTENT_PARAMS);
298309
for (Job job : jobs) {
310+
logger.debug("adding job to migrate: " + job.getId());
299311
bulkRequestBuilder.add(indexRequest(job, Job.documentId(job.getId()), params));
300312
}
301313
}
302314

303315
private void addDatafeedIndexRequests(Collection<DatafeedConfig> datafeedConfigs, BulkRequestBuilder bulkRequestBuilder) {
304316
ToXContent.Params params = new ToXContent.MapParams(DatafeedConfigProvider.TO_XCONTENT_PARAMS);
305317
for (DatafeedConfig datafeedConfig : datafeedConfigs) {
318+
logger.debug("adding datafeed to migrate: " + datafeedConfig.getId());
306319
bulkRequestBuilder.add(indexRequest(datafeedConfig, DatafeedConfig.documentId(datafeedConfig.getId()), params));
307320
}
308321
}
@@ -318,7 +331,6 @@ private IndexRequest indexRequest(ToXContentObject source, String documentId, To
318331
return indexRequest;
319332
}
320333

321-
322334
// public for testing
323335
public void snapshotMlMeta(MlMetadata mlMetadata, ActionListener<Boolean> listener) {
324336

@@ -361,6 +373,30 @@ public void snapshotMlMeta(MlMetadata mlMetadata, ActionListener<Boolean> listen
361373
);
362374
}
363375

376+
private void createConfigIndex(ActionListener<Boolean> listener) {
377+
logger.info("creating the .ml-config index");
378+
CreateIndexRequest createIndexRequest = new CreateIndexRequest(AnomalyDetectorsIndex.configIndexName());
379+
try
380+
{
381+
createIndexRequest.settings(
382+
Settings.builder()
383+
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
384+
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
385+
.put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
386+
);
387+
createIndexRequest.mapping(ElasticsearchMappings.DOC_TYPE, ElasticsearchMappings.configMapping());
388+
} catch (Exception e) {
389+
logger.error("error writing the .ml-config mappings", e);
390+
listener.onFailure(e);
391+
return;
392+
}
393+
394+
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest,
395+
ActionListener.<CreateIndexResponse>wrap(
396+
r -> listener.onResponse(r.isAcknowledged()),
397+
listener::onFailure
398+
), client.admin().indices()::create);
399+
}
364400

365401
public static Job updateJobForMigration(Job job) {
366402
Job.Builder builder = new Job.Builder(job);

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,15 @@
7373
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
7474
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
7575

76+
/**
77+
* This class implements CRUD operation for the
78+
* datafeed configuration document
79+
*
80+
* The number of datafeeds returned in a search it limited to
81+
* {@link AnomalyDetectorsIndex#CONFIG_INDEX_MAX_RESULTS_WINDOW}.
82+
* In most cases we expect 10s or 100s of datafeeds to be defined and
83+
* a search for all datafeeds should return all.
84+
*/
7685
public class DatafeedConfigProvider {
7786

7887
private static final Logger logger = LogManager.getLogger(DatafeedConfigProvider.class);
@@ -87,13 +96,6 @@ public class DatafeedConfigProvider {
8796
TO_XCONTENT_PARAMS = Collections.unmodifiableMap(modifiable);
8897
}
8998

90-
/**
91-
* In most cases we expect 10s or 100s of datafeeds to be defined and
92-
* a search for all datafeeds should return all.
93-
* TODO this is a temporary fix
94-
*/
95-
public int searchSize = 1000;
96-
9799
public DatafeedConfigProvider(Client client, NamedXContentRegistry xContentRegistry) {
98100
this.client = client;
99101
this.xContentRegistry = xContentRegistry;
@@ -368,7 +370,9 @@ public void expandDatafeedIds(String expression, boolean allowNoDatafeeds, Actio
368370

369371
SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName())
370372
.setIndicesOptions(IndicesOptions.lenientExpandOpen())
371-
.setSource(sourceBuilder).request();
373+
.setSource(sourceBuilder)
374+
.setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
375+
.request();
372376

373377
ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoDatafeeds);
374378

@@ -407,7 +411,6 @@ public void expandDatafeedIds(String expression, boolean allowNoDatafeeds, Actio
407411
* wildcard then setting this true will not suppress the exception
408412
* @param listener The expanded datafeed config listener
409413
*/
410-
// NORELEASE datafeed configs should be paged or have a mechanism to return all jobs if there are many of them
411414
public void expandDatafeedConfigs(String expression, boolean allowNoDatafeeds, ActionListener<List<DatafeedConfig.Builder>> listener) {
412415
String [] tokens = ExpandedIdsMatcher.tokenizeExpression(expression);
413416
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildDatafeedIdQuery(tokens));
@@ -416,7 +419,7 @@ public void expandDatafeedConfigs(String expression, boolean allowNoDatafeeds, A
416419
SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName())
417420
.setIndicesOptions(IndicesOptions.lenientExpandOpen())
418421
.setSource(sourceBuilder)
419-
.setSize(searchSize)
422+
.setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
420423
.request();
421424

422425
ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoDatafeeds);

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java‎

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,11 @@
8989
/**
9090
* This class implements CRUD operation for the
9191
* anomaly detector job configuration document
92+
*
93+
* The number of jobs returned in a search it limited to
94+
* {@link AnomalyDetectorsIndex#CONFIG_INDEX_MAX_RESULTS_WINDOW}.
95+
* In most cases we expect 10s or 100s of jobs to be defined and
96+
* a search for all jobs should return all.
9297
*/
9398
public class JobConfigProvider {
9499

@@ -101,13 +106,6 @@ public class JobConfigProvider {
101106
TO_XCONTENT_PARAMS = Collections.unmodifiableMap(modifiable);
102107
}
103108

104-
/**
105-
* In most cases we expect 10s or 100s of jobs to be defined and
106-
* a search for all jobs should return all.
107-
* TODO this is a temporary fix
108-
*/
109-
private int searchSize = 1000;
110-
111109
private final Client client;
112110

113111
public JobConfigProvider(Client client) {
@@ -565,7 +563,7 @@ public void expandJobsIds(String expression, boolean allowNoJobs, boolean exclud
565563
SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName())
566564
.setIndicesOptions(IndicesOptions.lenientExpandOpen())
567565
.setSource(sourceBuilder)
568-
.setSize(searchSize)
566+
.setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
569567
.request();
570568

571569
ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoJobs);
@@ -599,6 +597,21 @@ public void expandJobsIds(String expression, boolean allowNoJobs, boolean exclud
599597

600598
}
601599

600+
private SearchRequest makeExpandIdsSearchRequest(String expression, boolean excludeDeleting) {
601+
String [] tokens = ExpandedIdsMatcher.tokenizeExpression(expression);
602+
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens, excludeDeleting));
603+
sourceBuilder.sort(Job.ID.getPreferredName());
604+
sourceBuilder.fetchSource(false);
605+
sourceBuilder.docValueField(Job.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT);
606+
sourceBuilder.docValueField(Job.GROUPS.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT);
607+
608+
return client.prepareSearch(AnomalyDetectorsIndex.configIndexName())
609+
.setIndicesOptions(IndicesOptions.lenientExpandOpen())
610+
.setSource(sourceBuilder)
611+
.setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
612+
.request();
613+
}
614+
602615
/**
603616
* The same logic as {@link #expandJobsIds(String, boolean, boolean, ActionListener)} but
604617
* the full anomaly detector job configuration is returned.
@@ -612,7 +625,6 @@ public void expandJobsIds(String expression, boolean allowNoJobs, boolean exclud
612625
* @param excludeDeleting If true exclude jobs marked as deleting
613626
* @param listener The expanded jobs listener
614627
*/
615-
// NORELEASE jobs should be paged or have a mechanism to return all jobs if there are many of them
616628
public void expandJobs(String expression, boolean allowNoJobs, boolean excludeDeleting, ActionListener<List<Job.Builder>> listener) {
617629
String [] tokens = ExpandedIdsMatcher.tokenizeExpression(expression);
618630
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens, excludeDeleting));
@@ -621,7 +633,7 @@ public void expandJobs(String expression, boolean allowNoJobs, boolean excludeDe
621633
SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName())
622634
.setIndicesOptions(IndicesOptions.lenientExpandOpen())
623635
.setSource(sourceBuilder)
624-
.setSize(searchSize)
636+
.setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
625637
.request();
626638

627639
ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoJobs);
@@ -679,7 +691,7 @@ public void expandGroupIds(List<String> groupIds, ActionListener<SortedSet<Strin
679691
SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName())
680692
.setIndicesOptions(IndicesOptions.lenientExpandOpen())
681693
.setSource(sourceBuilder)
682-
.setSize(searchSize)
694+
.setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
683695
.request();
684696

685697
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest,
@@ -741,7 +753,7 @@ public void findJobsWithCustomRules(ActionListener<List<Job>> listener) {
741753
SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName())
742754
.setIndicesOptions(IndicesOptions.lenientExpandOpen())
743755
.setSource(sourceBuilder)
744-
.setSize(searchSize)
756+
.setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
745757
.request();
746758

747759
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest,

0 commit comments

Comments
 (0)