Skip to content

Commit 625e675

Browse files
davidkyledavidkyle
authored
[ML] Job in Index: Enable integ tests (#34851)
Enables the ml integration tests excluding the rolling upgrade tests and a lot of fixes to make the tests pass again.
1 parent 344b2ab commit 625e675

File tree

27 files changed

+313
-106
lines changed

27 files changed

+313
-106
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,13 +139,15 @@ public static class JobParams implements XPackPlugin.XPackPersistentTaskParams {
139139
/** TODO Remove in 7.0.0 */
140140
public static final ParseField IGNORE_DOWNTIME = new ParseField("ignore_downtime");
141141
public static final ParseField TIMEOUT = new ParseField("timeout");
142+
public static final ParseField JOB = new ParseField("job");
142143

143144
public static ObjectParser<JobParams, Void> PARSER = new ObjectParser<>(MlTasks.JOB_TASK_NAME, true, JobParams::new);
144145
static {
145146
PARSER.declareString(JobParams::setJobId, Job.ID);
146147
PARSER.declareBoolean((p, v) -> {}, IGNORE_DOWNTIME);
147148
PARSER.declareString((params, val) ->
148149
params.setTimeout(TimeValue.parseTimeValue(val, TIMEOUT.getPreferredName())), TIMEOUT);
150+
PARSER.declareObject(JobParams::setJob, (p, c) -> Job.LENIENT_PARSER.apply(p, c).build(), JOB);
149151
}
150152

151153
public static JobParams fromXContent(XContentParser parser) {
@@ -233,6 +235,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
233235
builder.startObject();
234236
builder.field(Job.ID.getPreferredName(), jobId);
235237
builder.field(TIMEOUT.getPreferredName(), timeout.getStringRep());
238+
if (job != null) {
239+
builder.field("job", job);
240+
}
236241
builder.endObject();
237242
// The job field is streamed but not persisted
238243
return builder;

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
*/
66
package org.elasticsearch.xpack.core.ml.job.config;
77

8+
import org.elasticsearch.ResourceAlreadyExistsException;
89
import org.elasticsearch.Version;
910
import org.elasticsearch.cluster.AbstractDiffable;
1011
import org.elasticsearch.common.Nullable;
@@ -1084,6 +1085,10 @@ private void validateGroups() {
10841085
if (MlStrings.isValidId(group) == false) {
10851086
throw new IllegalArgumentException(Messages.getMessage(Messages.INVALID_GROUP, group));
10861087
}
1088+
if (this.id.equals(group)) {
1089+
// cannot have a group name the same as the job id
1090+
throw new ResourceAlreadyExistsException(Messages.getMessage(Messages.JOB_AND_GROUP_NAMES_MUST_BE_UNIQUE, group));
1091+
}
10871092
}
10881093
}
10891094

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java‎

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
1212
import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
1313
import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
14-
import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig;
1514
import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
1615
import org.elasticsearch.xpack.core.ml.job.config.DetectionRule;
1716
import org.elasticsearch.xpack.core.ml.job.config.Detector;
@@ -156,21 +155,7 @@ public static void addJobConfigFields(XContentBuilder builder) throws IOExceptio
156155
.field(TYPE, KEYWORD)
157156
.endObject()
158157
.startObject(AnalysisConfig.CATEGORIZATION_ANALYZER.getPreferredName())
159-
.startObject(PROPERTIES)
160-
.startObject(CategorizationAnalyzerConfig.CATEGORIZATION_ANALYZER.getPreferredName())
161-
.field(TYPE, KEYWORD)
162-
.endObject()
163-
// TOKENIZER, TOKEN_FILTERS and CHAR_FILTERS are complex types, don't parse or index
164-
.startObject(CategorizationAnalyzerConfig.TOKENIZER.getPreferredName())
165-
.field(ENABLED, false)
166-
.endObject()
167-
.startObject(CategorizationAnalyzerConfig.TOKEN_FILTERS.getPreferredName())
168-
.field(ENABLED, false)
169-
.endObject()
170-
.startObject(CategorizationAnalyzerConfig.CHAR_FILTERS.getPreferredName())
171-
.field(ENABLED, false)
172-
.endObject()
173-
.endObject()
158+
.field(ENABLED, false)
174159
.endObject()
175160
.startObject(AnalysisConfig.LATENCY.getPreferredName())
176161
.field(TYPE, KEYWORD)

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
1010
import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
1111
import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
12-
import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig;
1312
import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
1413
import org.elasticsearch.xpack.core.ml.job.config.DetectionRule;
1514
import org.elasticsearch.xpack.core.ml.job.config.Detector;
@@ -214,10 +213,6 @@ public final class ReservedFieldNames {
214213
AnalysisLimits.MODEL_MEMORY_LIMIT.getPreferredName(),
215214
AnalysisLimits.CATEGORIZATION_EXAMPLES_LIMIT.getPreferredName(),
216215

217-
CategorizationAnalyzerConfig.CHAR_FILTERS.getPreferredName(),
218-
CategorizationAnalyzerConfig.TOKENIZER.getPreferredName(),
219-
CategorizationAnalyzerConfig.TOKEN_FILTERS.getPreferredName(),
220-
221216
Detector.DETECTOR_DESCRIPTION_FIELD.getPreferredName(),
222217
Detector.FUNCTION_FIELD.getPreferredName(),
223218
Detector.FIELD_NAME_FIELD.getPreferredName(),

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/JobParamsTests.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@
1010
import org.elasticsearch.common.unit.TimeValue;
1111
import org.elasticsearch.common.xcontent.XContentParser;
1212
import org.elasticsearch.test.AbstractSerializingTestCase;
13+
import org.elasticsearch.xpack.core.ml.job.config.JobTests;
1314

1415
import java.io.IOException;
16+
import java.util.function.Predicate;
1517

1618
public class JobParamsTests extends AbstractSerializingTestCase<OpenJobAction.JobParams> {
1719

@@ -25,6 +27,9 @@ public static OpenJobAction.JobParams createJobParams() {
2527
if (randomBoolean()) {
2628
params.setTimeout(TimeValue.timeValueMillis(randomNonNegativeLong()));
2729
}
30+
if (randomBoolean()) {
31+
params.setJob(JobTests.createRandomizedJob());
32+
}
2833
return params;
2934
}
3035

@@ -42,4 +47,12 @@ protected Writeable.Reader<OpenJobAction.JobParams> instanceReader() {
4247
protected boolean supportsUnknownFields() {
4348
return true;
4449
}
50+
51+
@Override
52+
protected Predicate<String> getRandomFieldsExcludeFilter() {
53+
// Don't insert random fields into the job object as the
54+
// custom_fields member accepts arbitrary fields and new
55+
// fields inserted there will result in object inequality
56+
return path -> path.startsWith(OpenJobAction.JobParams.JOB.getPreferredName());
57+
}
4558
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
99
import org.elasticsearch.ElasticsearchStatusException;
10+
import org.elasticsearch.ResourceAlreadyExistsException;
1011
import org.elasticsearch.Version;
1112
import org.elasticsearch.common.bytes.BytesReference;
1213
import org.elasticsearch.common.io.stream.Writeable;
@@ -523,6 +524,13 @@ public void testInvalidGroup() {
523524
assertThat(e.getMessage(), containsString("Invalid group id '$$$'"));
524525
}
525526

527+
public void testInvalidGroup_matchesJobId() {
528+
Job.Builder builder = buildJobBuilder("foo");
529+
builder.setGroups(Collections.singletonList("foo"));
530+
ResourceAlreadyExistsException e = expectThrows(ResourceAlreadyExistsException.class, builder::build);
531+
assertEquals(e.getMessage(), "job and group names must be unique but job [foo] and group [foo] have the same name");
532+
}
533+
526534
public void testEstimateMemoryFootprint_GivenEstablished() {
527535
Job.Builder builder = buildJobBuilder("established");
528536
long establishedModelMemory = randomIntBetween(10_000, 2_000_000_000);

‎x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -704,7 +704,7 @@ public void testRealtime() throws Exception {
704704
response = e.getResponse();
705705
assertThat(response.getStatusLine().getStatusCode(), equalTo(409));
706706
assertThat(EntityUtils.toString(response.getEntity()),
707-
containsString("Cannot delete job [" + jobId + "] because datafeed [" + datafeedId + "] refers to it"));
707+
containsString("Cannot delete job [" + jobId + "] because the job is opened"));
708708

709709
response = client().performRequest(new Request("POST", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_stop"));
710710
assertThat(response.getStatusLine().getStatusCode(), equalTo(200));

‎x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,8 @@ public void testDeleteExpiredDataGivenNothingToDelete() throws Exception {
9595
}
9696

9797
public void testDeleteExpiredData() throws Exception {
98-
registerJob(newJobBuilder("no-retention").setResultsRetentionDays(null).setModelSnapshotRetentionDays(null));
99-
registerJob(newJobBuilder("results-retention").setResultsRetentionDays(1L).setModelSnapshotRetentionDays(null));
98+
registerJob(newJobBuilder("no-retention").setResultsRetentionDays(null).setModelSnapshotRetentionDays(1000L));
99+
registerJob(newJobBuilder("results-retention").setResultsRetentionDays(1L).setModelSnapshotRetentionDays(1000L));
100100
registerJob(newJobBuilder("snapshots-retention").setResultsRetentionDays(null).setModelSnapshotRetentionDays(2L));
101101
registerJob(newJobBuilder("snapshots-retention-with-retain").setResultsRetentionDays(null).setModelSnapshotRetentionDays(2L));
102102
registerJob(newJobBuilder("results-and-snapshots-retention").setResultsRetentionDays(1L).setModelSnapshotRetentionDays(2L));

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,7 @@ private void deleteJobDocuments(ParentTaskAssigningClient parentTaskClient, Stri
331331
builder -> {
332332
Job job = builder.build();
333333
indexName.set(job.getResultsIndexName());
334-
if (indexName.equals(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX +
334+
if (indexName.get().equals(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX +
335335
AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT)) {
336336
//don't bother searching the index any further, we are on the default shared
337337
customIndexSearchHandler.onResponse(null);

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -98,12 +98,15 @@ Map<String, DatafeedConfig> expandClusterStateDatafeeds(String datafeedExpressio
9898
ClusterState clusterState) {
9999

100100
Map<String, DatafeedConfig> configById = new HashMap<>();
101-
102-
MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState);
103-
Set<String> expandedDatafeedIds = mlMetadata.expandDatafeedIds(datafeedExpression, allowNoDatafeeds);
104-
105-
for (String expandedDatafeedId : expandedDatafeedIds) {
106-
configById.put(expandedDatafeedId, mlMetadata.getDatafeed(expandedDatafeedId));
101+
try {
102+
MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState);
103+
Set<String> expandedDatafeedIds = mlMetadata.expandDatafeedIds(datafeedExpression, allowNoDatafeeds);
104+
105+
for (String expandedDatafeedId : expandedDatafeedIds) {
106+
configById.put(expandedDatafeedId, mlMetadata.getDatafeed(expandedDatafeedId));
107+
}
108+
} catch (Exception e){
109+
// ignore
107110
}
108111

109112
return configById;

0 commit comments

Comments
 (0)