Skip to content

Commit 66a582a

Browse files
davidkyledavidkyle
authored
[ML] Merge the Jindex 6x feature branch (#36698)
Store anomaly detector jobs and datafeed configurations in the .ml-config index. Existing configurations for closed jobs and datafeeds are migrated after upgrade. For #32905
1 parent c0d1597 commit 66a582a

File tree

158 files changed

+12199
-2856
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

158 files changed

+12199
-2856
lines changed

‎qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ public void testIndexTemplatesCreated() throws Exception {
8989
if (masterIsNewVersion()) {
9090
// Everything else waits until the master is upgraded to create its templates
9191
expectedTemplates.add(".ml-anomalies-");
92+
expectedTemplates.add(".ml-config");
9293
expectedTemplates.add(".ml-meta");
9394
expectedTemplates.add(".ml-notifications");
9495
expectedTemplates.add(".ml-state");

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
import org.elasticsearch.xpack.core.logstash.LogstashFeatureSetUsage;
6565
import org.elasticsearch.xpack.core.ml.MachineLearningFeatureSetUsage;
6666
import org.elasticsearch.xpack.core.ml.MlMetadata;
67+
import org.elasticsearch.xpack.core.ml.MlTasks;
6768
import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
6869
import org.elasticsearch.xpack.core.ml.action.DeleteCalendarAction;
6970
import org.elasticsearch.xpack.core.ml.action.DeleteCalendarEventAction;
@@ -363,9 +364,9 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
363364
new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new),
364365
new NamedWriteableRegistry.Entry(NamedDiff.class, "ml", MlMetadata.MlMetadataDiff::new),
365366
// ML - Persistent action requests
366-
new NamedWriteableRegistry.Entry(PersistentTaskParams.class, StartDatafeedAction.TASK_NAME,
367+
new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.DATAFEED_TASK_NAME,
367368
StartDatafeedAction.DatafeedParams::new),
368-
new NamedWriteableRegistry.Entry(PersistentTaskParams.class, OpenJobAction.TASK_NAME,
369+
new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.JOB_TASK_NAME,
369370
OpenJobAction.JobParams::new),
370371
// ML - Task states
371372
new NamedWriteableRegistry.Entry(PersistentTaskState.class, JobTaskState.NAME, JobTaskState::new),
@@ -433,9 +434,9 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
433434
new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField("ml"),
434435
parser -> MlMetadata.LENIENT_PARSER.parse(parser, null).build()),
435436
// ML - Persistent action requests
436-
new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(StartDatafeedAction.TASK_NAME),
437+
new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(MlTasks.DATAFEED_TASK_NAME),
437438
StartDatafeedAction.DatafeedParams::fromXContent),
438-
new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(OpenJobAction.TASK_NAME),
439+
new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(MlTasks.JOB_TASK_NAME),
439440
OpenJobAction.JobParams::fromXContent),
440441
// ML - Task states
441442
new NamedXContentRegistry.Entry(PersistentTaskState.class, new ParseField(DatafeedState.NAME), DatafeedState::fromXContent),

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,6 @@ public final class MlMetaIndex {
2121
*/
2222
public static final String INDEX_NAME = ".ml-meta";
2323

24-
public static final String INCLUDE_TYPE_KEY = "include_type";
25-
2624
public static final String TYPE = "doc";
2725

2826
private MlMetaIndex() {}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java‎

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
*/
66
package org.elasticsearch.xpack.core.ml;
77

8-
import org.elasticsearch.ResourceAlreadyExistsException;
98
import org.elasticsearch.ResourceNotFoundException;
109
import org.elasticsearch.Version;
1110
import org.elasticsearch.cluster.AbstractDiffable;
@@ -87,8 +86,13 @@ public boolean isGroupOrJob(String id) {
8786
return groupOrJobLookup.isGroupOrJob(id);
8887
}
8988

90-
public Set<String> expandJobIds(String expression, boolean allowNoJobs) {
91-
return groupOrJobLookup.expandJobIds(expression, allowNoJobs);
89+
public Set<String> expandJobIds(String expression) {
90+
return groupOrJobLookup.expandJobIds(expression);
91+
}
92+
93+
// Matches only groups
94+
public Set<String> expandGroupIds(String expression) {
95+
return groupOrJobLookup.expandGroupIds(expression);
9296
}
9397

9498
public boolean isJobDeleting(String jobId) {
@@ -108,9 +112,9 @@ public Optional<DatafeedConfig> getDatafeedByJobId(String jobId) {
108112
return datafeeds.values().stream().filter(s -> s.getJobId().equals(jobId)).findFirst();
109113
}
110114

111-
public Set<String> expandDatafeedIds(String expression, boolean allowNoDatafeeds) {
112-
return NameResolver.newUnaliased(datafeeds.keySet(), ExceptionsHelper::missingDatafeedException)
113-
.expand(expression, allowNoDatafeeds);
115+
public Set<String> expandDatafeedIds(String expression) {
116+
return NameResolver.newUnaliased(datafeeds.keySet())
117+
.expand(expression);
114118
}
115119

116120
@Override
@@ -146,7 +150,6 @@ public MlMetadata(StreamInput in) throws IOException {
146150
datafeeds.put(in.readString(), new DatafeedConfig(in));
147151
}
148152
this.datafeeds = datafeeds;
149-
150153
this.groupOrJobLookup = new GroupOrJobLookup(jobs.values());
151154
}
152155

@@ -167,7 +170,7 @@ private static <T extends Writeable> void writeMap(Map<String, T> map, StreamOut
167170
@Override
168171
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
169172
DelegatingMapParams extendedParams =
170-
new DelegatingMapParams(Collections.singletonMap(ToXContentParams.FOR_CLUSTER_STATE, "true"), params);
173+
new DelegatingMapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true"), params);
171174
mapValuesToXContent(JOBS_FIELD, jobs, builder, extendedParams);
172175
mapValuesToXContent(DATAFEEDS_FIELD, datafeeds, builder, extendedParams);
173176
return builder;
@@ -196,9 +199,14 @@ public MlMetadataDiff(StreamInput in) throws IOException {
196199
this.jobs = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), Job::new,
197200
MlMetadataDiff::readJobDiffFrom);
198201
this.datafeeds = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), DatafeedConfig::new,
199-
MlMetadataDiff::readSchedulerDiffFrom);
202+
MlMetadataDiff::readDatafeedDiffFrom);
200203
}
201204

205+
/**
206+
* Merge the diff with the ML metadata.
207+
* @param part The current ML metadata.
208+
* @return The new ML metadata.
209+
*/
202210
@Override
203211
public MetaData.Custom apply(MetaData.Custom part) {
204212
TreeMap<String, Job> newJobs = new TreeMap<>(jobs.apply(((MlMetadata) part).jobs));
@@ -221,7 +229,7 @@ static Diff<Job> readJobDiffFrom(StreamInput in) throws IOException {
221229
return AbstractDiffable.readDiffFrom(Job::new, in);
222230
}
223231

224-
static Diff<DatafeedConfig> readSchedulerDiffFrom(StreamInput in) throws IOException {
232+
static Diff<DatafeedConfig> readDatafeedDiffFrom(StreamInput in) throws IOException {
225233
return AbstractDiffable.readDiffFrom(DatafeedConfig::new, in);
226234
}
227235
}
@@ -295,7 +303,7 @@ public Builder deleteJob(String jobId, PersistentTasksCustomMetaData tasks) {
295303

296304
public Builder putDatafeed(DatafeedConfig datafeedConfig, Map<String, String> headers) {
297305
if (datafeeds.containsKey(datafeedConfig.getId())) {
298-
throw new ResourceAlreadyExistsException("A datafeed with id [" + datafeedConfig.getId() + "] already exists");
306+
throw ExceptionsHelper.datafeedAlreadyExists(datafeedConfig.getId());
299307
}
300308
String jobId = datafeedConfig.getJobId();
301309
checkJobIsAvailableForDatafeed(jobId);
@@ -369,14 +377,14 @@ private void checkDatafeedIsStopped(Supplier<String> msg, String datafeedId, Per
369377
}
370378
}
371379

372-
private Builder putJobs(Collection<Job> jobs) {
380+
public Builder putJobs(Collection<Job> jobs) {
373381
for (Job job : jobs) {
374382
putJob(job, true);
375383
}
376384
return this;
377385
}
378386

379-
private Builder putDatafeeds(Collection<DatafeedConfig> datafeeds) {
387+
public Builder putDatafeeds(Collection<DatafeedConfig> datafeeds) {
380388
for (DatafeedConfig datafeed : datafeeds) {
381389
this.datafeeds.put(datafeed.getId(), datafeed);
382390
}
@@ -421,8 +429,6 @@ void checkJobHasNoDatafeed(String jobId) {
421429
}
422430
}
423431

424-
425-
426432
public static MlMetadata getMlMetadata(ClusterState state) {
427433
MlMetadata mlMetadata = (state == null) ? null : state.getMetaData().custom(TYPE);
428434
if (mlMetadata == null) {

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java‎

Lines changed: 73 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,19 @@
1212
import org.elasticsearch.xpack.core.ml.job.config.JobState;
1313
import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
1414

15+
import java.util.Collections;
16+
import java.util.List;
17+
import java.util.Set;
18+
import java.util.stream.Collectors;
19+
1520
public final class MlTasks {
1621

22+
public static final String JOB_TASK_NAME = "xpack/ml/job";
23+
public static final String DATAFEED_TASK_NAME = "xpack/ml/datafeed";
24+
25+
private static final String JOB_TASK_ID_PREFIX = "job-";
26+
private static final String DATAFEED_TASK_ID_PREFIX = "datafeed-";
27+
1728
private MlTasks() {
1829
}
1930

@@ -22,15 +33,15 @@ private MlTasks() {
2233
* A datafeed id can be used as a job id, because they are stored separately in cluster state.
2334
*/
2435
public static String jobTaskId(String jobId) {
25-
return "job-" + jobId;
36+
return JOB_TASK_ID_PREFIX + jobId;
2637
}
2738

2839
/**
2940
* Namespaces the task ids for datafeeds.
3041
* A job id can be used as a datafeed id, because they are stored separately in cluster state.
3142
*/
3243
public static String datafeedTaskId(String datafeedId) {
33-
return "datafeed-" + datafeedId;
44+
return DATAFEED_TASK_ID_PREFIX + datafeedId;
3445
}
3546

3647
@Nullable
@@ -67,4 +78,64 @@ public static DatafeedState getDatafeedState(String datafeedId, @Nullable Persis
6778
return DatafeedState.STOPPED;
6879
}
6980
}
81+
82+
/**
83+
* The job Ids of anomaly detector job tasks.
84+
* All anomaly detector jobs are returned regardless of the status of the
85+
* task (OPEN, CLOSED, FAILED etc).
86+
*
87+
* @param tasks Persistent tasks. If null an empty set is returned.
88+
* @return The job Ids of anomaly detector job tasks
89+
*/
90+
public static Set<String> openJobIds(@Nullable PersistentTasksCustomMetaData tasks) {
91+
if (tasks == null) {
92+
return Collections.emptySet();
93+
}
94+
95+
return tasks.findTasks(JOB_TASK_NAME, task -> true)
96+
.stream()
97+
.map(t -> t.getId().substring(JOB_TASK_ID_PREFIX.length()))
98+
.collect(Collectors.toSet());
99+
}
100+
101+
/**
102+
* The datafeed Ids of started datafeed tasks
103+
*
104+
* @param tasks Persistent tasks. If null an empty set is returned.
105+
* @return The Ids of running datafeed tasks
106+
*/
107+
public static Set<String> startedDatafeedIds(@Nullable PersistentTasksCustomMetaData tasks) {
108+
if (tasks == null) {
109+
return Collections.emptySet();
110+
}
111+
112+
return tasks.findTasks(DATAFEED_TASK_NAME, task -> true)
113+
.stream()
114+
.map(t -> t.getId().substring(DATAFEED_TASK_ID_PREFIX.length()))
115+
.collect(Collectors.toSet());
116+
}
117+
118+
/**
119+
* Is there an ml anomaly detector job task for the job {@code jobId}?
120+
* @param jobId The job id
121+
* @param tasks Persistent tasks
122+
* @return True if the job has a task
123+
*/
124+
public static boolean taskExistsForJob(String jobId, PersistentTasksCustomMetaData tasks) {
125+
return openJobIds(tasks).contains(jobId);
126+
}
127+
128+
/**
129+
* Read the active anomaly detector job tasks.
130+
* Active tasks are not {@code JobState.CLOSED} or {@code JobState.FAILED}.
131+
*
132+
* @param tasks Persistent tasks
133+
* @return The job tasks excluding closed and failed jobs
134+
*/
135+
public static List<PersistentTasksCustomMetaData.PersistentTask<?>> activeJobTasks(PersistentTasksCustomMetaData tasks) {
136+
return tasks.findTasks(JOB_TASK_NAME, task -> true)
137+
.stream()
138+
.filter(task -> ((JobTaskState) task.getState()).getState().isAnyOf(JobState.CLOSED, JobState.FAILED) == false)
139+
.collect(Collectors.toList());
140+
}
70141
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.action.support.master.MasterNodeRequest;
1414
import org.elasticsearch.client.ElasticsearchClient;
1515
import org.elasticsearch.cluster.metadata.MetaData;
16+
import org.elasticsearch.common.Nullable;
1617
import org.elasticsearch.common.ParseField;
1718
import org.elasticsearch.common.Strings;
1819
import org.elasticsearch.common.io.stream.StreamInput;
@@ -25,6 +26,7 @@
2526
import org.elasticsearch.tasks.Task;
2627
import org.elasticsearch.xpack.core.XPackPlugin;
2728
import org.elasticsearch.xpack.core.ml.MachineLearningField;
29+
import org.elasticsearch.xpack.core.ml.MlTasks;
2830
import org.elasticsearch.xpack.core.ml.job.config.Job;
2931
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
3032

@@ -35,7 +37,7 @@ public class OpenJobAction extends Action<OpenJobAction.Request, AcknowledgedRes
3537

3638
public static final OpenJobAction INSTANCE = new OpenJobAction();
3739
public static final String NAME = "cluster:admin/xpack/ml/job/open";
38-
public static final String TASK_NAME = "xpack/ml/job";
40+
3941

4042
private OpenJobAction() {
4143
super(NAME);
@@ -136,15 +138,16 @@ public static class JobParams implements XPackPlugin.XPackPersistentTaskParams {
136138

137139
/** TODO Remove in 7.0.0 */
138140
public static final ParseField IGNORE_DOWNTIME = new ParseField("ignore_downtime");
139-
140141
public static final ParseField TIMEOUT = new ParseField("timeout");
141-
public static ObjectParser<JobParams, Void> PARSER = new ObjectParser<>(TASK_NAME, true, JobParams::new);
142+
public static final ParseField JOB = new ParseField("job");
142143

144+
public static ObjectParser<JobParams, Void> PARSER = new ObjectParser<>(MlTasks.JOB_TASK_NAME, true, JobParams::new);
143145
static {
144146
PARSER.declareString(JobParams::setJobId, Job.ID);
145147
PARSER.declareBoolean((p, v) -> {}, IGNORE_DOWNTIME);
146148
PARSER.declareString((params, val) ->
147149
params.setTimeout(TimeValue.parseTimeValue(val, TIMEOUT.getPreferredName())), TIMEOUT);
150+
PARSER.declareObject(JobParams::setJob, (p, c) -> Job.LENIENT_PARSER.apply(p, c).build(), JOB);
148151
}
149152

150153
public static JobParams fromXContent(XContentParser parser) {
@@ -163,6 +166,7 @@ public static JobParams parseRequest(String jobId, XContentParser parser) {
163166
// A big state can take a while to restore. For symmetry with the _close endpoint any
164167
// changes here should be reflected there too.
165168
private TimeValue timeout = MachineLearningField.STATE_PERSIST_RESTORE_TIMEOUT;
169+
private Job job;
166170

167171
JobParams() {
168172
}
@@ -178,6 +182,9 @@ public JobParams(StreamInput in) throws IOException {
178182
in.readBoolean();
179183
}
180184
timeout = TimeValue.timeValueMillis(in.readVLong());
185+
if (in.getVersion().onOrAfter(Version.V_6_6_0)) {
186+
job = in.readOptionalWriteable(Job::new);
187+
}
181188
}
182189

183190
public String getJobId() {
@@ -196,9 +203,18 @@ public void setTimeout(TimeValue timeout) {
196203
this.timeout = timeout;
197204
}
198205

206+
@Nullable
207+
public Job getJob() {
208+
return job;
209+
}
210+
211+
public void setJob(Job job) {
212+
this.job = job;
213+
}
214+
199215
@Override
200216
public String getWriteableName() {
201-
return TASK_NAME;
217+
return MlTasks.JOB_TASK_NAME;
202218
}
203219

204220
@Override
@@ -209,20 +225,27 @@ public void writeTo(StreamOutput out) throws IOException {
209225
out.writeBoolean(true);
210226
}
211227
out.writeVLong(timeout.millis());
228+
if (out.getVersion().onOrAfter(Version.V_6_6_0)) {
229+
out.writeOptionalWriteable(job);
230+
}
212231
}
213232

214233
@Override
215234
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
216235
builder.startObject();
217236
builder.field(Job.ID.getPreferredName(), jobId);
218237
builder.field(TIMEOUT.getPreferredName(), timeout.getStringRep());
238+
if (job != null) {
239+
builder.field("job", job);
240+
}
219241
builder.endObject();
242+
// The job field is streamed but not persisted
220243
return builder;
221244
}
222245

223246
@Override
224247
public int hashCode() {
225-
return Objects.hash(jobId, timeout);
248+
return Objects.hash(jobId, timeout, job);
226249
}
227250

228251
@Override
@@ -235,7 +258,8 @@ public boolean equals(Object obj) {
235258
}
236259
OpenJobAction.JobParams other = (OpenJobAction.JobParams) obj;
237260
return Objects.equals(jobId, other.jobId) &&
238-
Objects.equals(timeout, other.timeout);
261+
Objects.equals(timeout, other.timeout) &&
262+
Objects.equals(job, other.job);
239263
}
240264

241265
@Override

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -143,9 +143,7 @@ public void writeTo(StreamOutput out) throws IOException {
143143

144144
@Override
145145
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
146-
builder.startObject();
147-
datafeed.doXContentBody(builder, params);
148-
builder.endObject();
146+
datafeed.toXContent(builder, params);
149147
return builder;
150148
}
151149

0 commit comments

Comments
 (0)