Skip to content

Commit 64ba62a

Browse files
davidkyledavidkyle
authored
[ML] Job In Index: Enable GET APIS in mixed state (#35344)
This enables calls to the job and datafeed APIs in a mixed cluster state before jobs have been migrated
1 parent 2b6cd7a commit 64ba62a

File tree

21 files changed

+1194
-274
lines changed

21 files changed

+1194
-274
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,13 @@ public boolean isGroupOrJob(String id) {
9090
return groupOrJobLookup.isGroupOrJob(id);
9191
}
9292

93-
public Set<String> expandJobIds(String expression, boolean allowNoJobs) {
94-
return groupOrJobLookup.expandJobIds(expression, allowNoJobs);
93+
public Set<String> expandJobIds(String expression) {
94+
return groupOrJobLookup.expandJobIds(expression);
95+
}
96+
97+
// Matches only groups
98+
public Set<String> expandGroupIds(String expression) {
99+
return groupOrJobLookup.expandGroupIds(expression);
95100
}
96101

97102
public boolean isJobDeleting(String jobId) {
@@ -111,9 +116,9 @@ public Optional<DatafeedConfig> getDatafeedByJobId(String jobId) {
111116
return datafeeds.values().stream().filter(s -> s.getJobId().equals(jobId)).findFirst();
112117
}
113118

114-
public Set<String> expandDatafeedIds(String expression, boolean allowNoDatafeeds) {
115-
return NameResolver.newUnaliased(datafeeds.keySet(), ExceptionsHelper::missingDatafeedException)
116-
.expand(expression, allowNoDatafeeds);
119+
public Set<String> expandDatafeedIds(String expression) {
120+
return NameResolver.newUnaliased(datafeeds.keySet())
121+
.expand(expression);
117122
}
118123

119124
public Long getLastMemoryRefreshVersion() {

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/groups/GroupOrJobLookup.java‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
import org.elasticsearch.ResourceAlreadyExistsException;
99
import org.elasticsearch.xpack.core.ml.job.config.Job;
1010
import org.elasticsearch.xpack.core.ml.job.messages.Messages;
11-
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
1211
import org.elasticsearch.xpack.core.ml.utils.NameResolver;
1312

1413
import java.util.ArrayList;
@@ -55,8 +54,12 @@ private void put(Job job) {
5554
}
5655
}
5756

58-
public Set<String> expandJobIds(String expression, boolean allowNoJobs) {
59-
return new GroupOrJobResolver().expand(expression, allowNoJobs);
57+
public Set<String> expandJobIds(String expression) {
58+
return new GroupOrJobResolver().expand(expression);
59+
}
60+
61+
public Set<String> expandGroupIds(String expression) {
62+
return new GroupResolver().expand(expression);
6063
}
6164

6265
public boolean isGroupOrJob(String id) {
@@ -66,7 +69,6 @@ public boolean isGroupOrJob(String id) {
6669
private class GroupOrJobResolver extends NameResolver {
6770

6871
private GroupOrJobResolver() {
69-
super(ExceptionsHelper::missingJobException);
7072
}
7173

7274
@Override
@@ -88,4 +90,33 @@ protected List<String> lookup(String key) {
8890
return groupOrJob == null ? Collections.emptyList() : groupOrJob.jobs().stream().map(Job::getId).collect(Collectors.toList());
8991
}
9092
}
93+
94+
private class GroupResolver extends NameResolver {
95+
96+
private GroupResolver() {
97+
}
98+
99+
@Override
100+
protected Set<String> keys() {
101+
return nameSet();
102+
}
103+
104+
@Override
105+
protected Set<String> nameSet() {
106+
return groupOrJobLookup.entrySet().stream()
107+
.filter(entry -> entry.getValue().isGroup())
108+
.map(entry -> entry.getKey())
109+
.collect(Collectors.toSet());
110+
}
111+
112+
@Override
113+
protected List<String> lookup(String key) {
114+
GroupOrJob groupOrJob = groupOrJobLookup.get(key);
115+
if (groupOrJob == null || groupOrJob.isGroup() == false) {
116+
return Collections.emptyList();
117+
} else {
118+
return Collections.singletonList(key);
119+
}
120+
}
121+
}
91122
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/NameResolver.java‎

Lines changed: 3 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,15 @@
55
*/
66
package org.elasticsearch.xpack.core.ml.utils;
77

8-
import org.elasticsearch.ResourceNotFoundException;
98
import org.elasticsearch.cluster.metadata.MetaData;
109
import org.elasticsearch.common.Strings;
1110
import org.elasticsearch.common.regex.Regex;
1211

1312
import java.util.Collections;
1413
import java.util.List;
15-
import java.util.Objects;
1614
import java.util.Set;
1715
import java.util.SortedSet;
1816
import java.util.TreeSet;
19-
import java.util.function.Function;
2017
import java.util.stream.Collectors;
2118

2219
/**
@@ -25,12 +22,6 @@
2522
*/
2623
public abstract class NameResolver {
2724

28-
private final Function<String, ResourceNotFoundException> notFoundExceptionSupplier;
29-
30-
protected NameResolver(Function<String, ResourceNotFoundException> notFoundExceptionSupplier) {
31-
this.notFoundExceptionSupplier = Objects.requireNonNull(notFoundExceptionSupplier);
32-
}
33-
3425
/**
3526
* Expands an expression into the set of matching names.
3627
* For example, given a set of names ["foo-1", "foo-2", "bar-1", bar-2"],
@@ -46,12 +37,9 @@ protected NameResolver(Function<String, ResourceNotFoundException> notFoundExcep
4637
* </ul>
4738
*
4839
* @param expression the expression to resolve
49-
* @param allowNoMatch if {@code false}, an error is thrown when no name matches the {@code expression}.
50-
* This only applies to wild card expressions, if {@code expression} is not a
51-
* wildcard then setting this true will not suppress the exception
5240
* @return the sorted set of matching names
5341
*/
54-
public SortedSet<String> expand(String expression, boolean allowNoMatch) {
42+
public SortedSet<String> expand(String expression) {
5543
SortedSet<String> result = new TreeSet<>();
5644
if (MetaData.ALL.equals(expression) || Regex.isMatchAllPattern(expression)) {
5745
result.addAll(nameSet());
@@ -64,24 +52,13 @@ public SortedSet<String> expand(String expression, boolean allowNoMatch) {
6452
.map(this::lookup)
6553
.flatMap(List::stream)
6654
.collect(Collectors.toList());
67-
if (expanded.isEmpty() && allowNoMatch == false) {
68-
throw notFoundExceptionSupplier.apply(token);
69-
}
7055
result.addAll(expanded);
7156
} else {
7257
List<String> matchingNames = lookup(token);
73-
// allowNoMatch only applies to wildcard expressions,
74-
// this isn't so don't check the allowNoMatch here
75-
if (matchingNames.isEmpty()) {
76-
throw notFoundExceptionSupplier.apply(token);
77-
}
7858
result.addAll(matchingNames);
7959
}
8060
}
8161
}
82-
if (result.isEmpty() && allowNoMatch == false) {
83-
throw notFoundExceptionSupplier.apply(expression);
84-
}
8562
return result;
8663
}
8764

@@ -105,11 +82,10 @@ public SortedSet<String> expand(String expression, boolean allowNoMatch) {
10582
/**
10683
* Creates a {@code NameResolver} that has no aliases
10784
* @param nameSet the set of all names
108-
* @param notFoundExceptionSupplier a supplier of {@link ResourceNotFoundException} to be used when an expression matches no name
10985
* @return the unaliased {@code NameResolver}
11086
*/
111-
public static NameResolver newUnaliased(Set<String> nameSet, Function<String, ResourceNotFoundException> notFoundExceptionSupplier) {
112-
return new NameResolver(notFoundExceptionSupplier) {
87+
public static NameResolver newUnaliased(Set<String> nameSet) {
88+
return new NameResolver() {
11389
@Override
11490
protected Set<String> keys() {
11591
return nameSet;

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/groups/GroupOrJobLookupTests.java‎

Lines changed: 33 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -6,46 +6,34 @@
66
package org.elasticsearch.xpack.core.ml.job.groups;
77

88
import org.elasticsearch.ResourceAlreadyExistsException;
9-
import org.elasticsearch.ResourceNotFoundException;
109
import org.elasticsearch.test.ESTestCase;
1110
import org.elasticsearch.xpack.core.ml.job.config.Job;
12-
import org.elasticsearch.xpack.core.ml.job.groups.GroupOrJobLookup;
1311

1412
import java.util.ArrayList;
1513
import java.util.Arrays;
1614
import java.util.Collections;
1715
import java.util.List;
1816

17+
import static org.hamcrest.Matchers.contains;
1918
import static org.hamcrest.Matchers.equalTo;
2019
import static org.hamcrest.Matchers.is;
21-
import static org.hamcrest.Matchers.contains;
2220
import static org.mockito.Mockito.mock;
2321
import static org.mockito.Mockito.when;
2422

2523
public class GroupOrJobLookupTests extends ESTestCase {
2624

27-
public void testEmptyLookup_GivenAllowNoJobs() {
28-
GroupOrJobLookup lookup = new GroupOrJobLookup(Collections.emptyList());
29-
30-
assertThat(lookup.expandJobIds("_all", true).isEmpty(), is(true));
31-
assertThat(lookup.expandJobIds("*", true).isEmpty(), is(true));
32-
assertThat(lookup.expandJobIds("foo*", true).isEmpty(), is(true));
33-
expectThrows(ResourceNotFoundException.class, () -> lookup.expandJobIds("foo", true));
34-
}
35-
36-
public void testEmptyLookup_GivenNotAllowNoJobs() {
25+
public void testEmptyLookup() {
3726
GroupOrJobLookup lookup = new GroupOrJobLookup(Collections.emptyList());
3827

39-
expectThrows(ResourceNotFoundException.class, () -> lookup.expandJobIds("_all", false));
40-
expectThrows(ResourceNotFoundException.class, () -> lookup.expandJobIds("*", false));
41-
expectThrows(ResourceNotFoundException.class, () -> lookup.expandJobIds("foo*", false));
42-
expectThrows(ResourceNotFoundException.class, () -> lookup.expandJobIds("foo", true));
28+
assertThat(lookup.expandJobIds("_all").isEmpty(), is(true));
29+
assertThat(lookup.expandJobIds("*").isEmpty(), is(true));
30+
assertThat(lookup.expandJobIds("foo*").isEmpty(), is(true));
31+
assertThat(lookup.expandJobIds("foo").isEmpty(), is(true));
4332
}
4433

4534
public void testAllIsNotExpandedInCommaSeparatedExpression() {
4635
GroupOrJobLookup lookup = new GroupOrJobLookup(Collections.emptyList());
47-
ResourceNotFoundException e = expectThrows(ResourceNotFoundException.class, () -> lookup.expandJobIds("foo-*,_all", true));
48-
assertThat(e.getMessage(), equalTo("No known job with id '_all'"));
36+
assertThat(lookup.expandJobIds("foo*,_all").isEmpty(), is(true));
4937
}
5038

5139
public void testConstructor_GivenJobWithSameIdAsPreviousGroupName() {
@@ -75,19 +63,19 @@ public void testLookup() {
7563
jobs.add(mockJob("nogroup", Collections.emptyList()));
7664
GroupOrJobLookup groupOrJobLookup = new GroupOrJobLookup(jobs);
7765

78-
assertThat(groupOrJobLookup.expandJobIds("_all", false), contains("bar-1", "bar-2", "foo-1", "foo-2", "nogroup"));
79-
assertThat(groupOrJobLookup.expandJobIds("*", false), contains("bar-1", "bar-2", "foo-1", "foo-2", "nogroup"));
80-
assertThat(groupOrJobLookup.expandJobIds("bar-1", false), contains("bar-1"));
81-
assertThat(groupOrJobLookup.expandJobIds("foo-1", false), contains("foo-1"));
82-
assertThat(groupOrJobLookup.expandJobIds("foo-2, bar-1", false), contains("bar-1", "foo-2"));
83-
assertThat(groupOrJobLookup.expandJobIds("foo-group", false), contains("foo-1", "foo-2"));
84-
assertThat(groupOrJobLookup.expandJobIds("bar-group", false), contains("bar-1", "bar-2"));
85-
assertThat(groupOrJobLookup.expandJobIds("ones", false), contains("bar-1", "foo-1"));
86-
assertThat(groupOrJobLookup.expandJobIds("twos", false), contains("bar-2", "foo-2"));
87-
assertThat(groupOrJobLookup.expandJobIds("foo-group, nogroup", false), contains("foo-1", "foo-2", "nogroup"));
88-
assertThat(groupOrJobLookup.expandJobIds("*-group", false), contains("bar-1", "bar-2", "foo-1", "foo-2"));
89-
assertThat(groupOrJobLookup.expandJobIds("foo-group,foo-1,foo-2", false), contains("foo-1", "foo-2"));
90-
assertThat(groupOrJobLookup.expandJobIds("foo-group,*-2", false), contains("bar-2", "foo-1", "foo-2"));
66+
assertThat(groupOrJobLookup.expandJobIds("_all"), contains("bar-1", "bar-2", "foo-1", "foo-2", "nogroup"));
67+
assertThat(groupOrJobLookup.expandJobIds("*"), contains("bar-1", "bar-2", "foo-1", "foo-2", "nogroup"));
68+
assertThat(groupOrJobLookup.expandJobIds("bar-1"), contains("bar-1"));
69+
assertThat(groupOrJobLookup.expandJobIds("foo-1"), contains("foo-1"));
70+
assertThat(groupOrJobLookup.expandJobIds("foo-2, bar-1"), contains("bar-1", "foo-2"));
71+
assertThat(groupOrJobLookup.expandJobIds("foo-group"), contains("foo-1", "foo-2"));
72+
assertThat(groupOrJobLookup.expandJobIds("bar-group"), contains("bar-1", "bar-2"));
73+
assertThat(groupOrJobLookup.expandJobIds("ones"), contains("bar-1", "foo-1"));
74+
assertThat(groupOrJobLookup.expandJobIds("twos"), contains("bar-2", "foo-2"));
75+
assertThat(groupOrJobLookup.expandJobIds("foo-group, nogroup"), contains("foo-1", "foo-2", "nogroup"));
76+
assertThat(groupOrJobLookup.expandJobIds("*-group"), contains("bar-1", "bar-2", "foo-1", "foo-2"));
77+
assertThat(groupOrJobLookup.expandJobIds("foo-group,foo-1,foo-2"), contains("foo-1", "foo-2"));
78+
assertThat(groupOrJobLookup.expandJobIds("foo-group,*-2"), contains("bar-2", "foo-1", "foo-2"));
9179
}
9280

9381
public void testIsGroupOrJob() {
@@ -104,6 +92,19 @@ public void testIsGroupOrJob() {
10492
assertFalse(groupOrJobLookup.isGroupOrJob("missing"));
10593
}
10694

95+
public void testExpandGroupIds() {
96+
List<Job> jobs = new ArrayList<>();
97+
jobs.add(mockJob("foo-1", Arrays.asList("foo-group")));
98+
jobs.add(mockJob("foo-2", Arrays.asList("foo-group")));
99+
jobs.add(mockJob("bar-1", Arrays.asList("bar-group")));
100+
jobs.add(mockJob("nogroup", Collections.emptyList()));
101+
102+
GroupOrJobLookup groupOrJobLookup = new GroupOrJobLookup(jobs);
103+
assertThat(groupOrJobLookup.expandGroupIds("foo*"), contains("foo-group"));
104+
assertThat(groupOrJobLookup.expandGroupIds("bar-group,nogroup"), contains("bar-group"));
105+
assertThat(groupOrJobLookup.expandGroupIds("*"), contains("bar-group", "foo-group"));
106+
}
107+
107108
private static Job mockJob(String jobId, List<String> groups) {
108109
Job job = mock(Job.class);
109110
when(job.getId()).thenReturn(jobId);

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
import org.elasticsearch.action.TaskOperationFailure;
1313
import org.elasticsearch.action.support.ActionFilters;
1414
import org.elasticsearch.action.support.tasks.TransportTasksAction;
15-
import org.elasticsearch.client.Client;
1615
import org.elasticsearch.cluster.ClusterState;
1716
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
1817
import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -37,7 +36,7 @@
3736
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
3837
import org.elasticsearch.xpack.ml.MachineLearning;
3938
import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider;
40-
import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider;
39+
import org.elasticsearch.xpack.ml.job.JobManager;
4140
import org.elasticsearch.xpack.ml.notifications.Auditor;
4241

4342
import java.io.IOException;
@@ -52,28 +51,25 @@
5251
public class TransportCloseJobAction extends TransportTasksAction<TransportOpenJobAction.JobTask, CloseJobAction.Request,
5352
CloseJobAction.Response, CloseJobAction.Response> {
5453

55-
private final Client client;
5654
private final ClusterService clusterService;
5755
private final Auditor auditor;
5856
private final PersistentTasksService persistentTasksService;
59-
private final JobConfigProvider jobConfigProvider;
6057
private final DatafeedConfigProvider datafeedConfigProvider;
58+
private final JobManager jobManager;
6159

6260
@Inject
6361
public TransportCloseJobAction(Settings settings, TransportService transportService, ThreadPool threadPool,
6462
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
65-
ClusterService clusterService, Client client, Auditor auditor,
66-
PersistentTasksService persistentTasksService, JobConfigProvider jobConfigProvider,
67-
DatafeedConfigProvider datafeedConfigProvider) {
63+
ClusterService clusterService, Auditor auditor, PersistentTasksService persistentTasksService,
64+
DatafeedConfigProvider datafeedConfigProvider, JobManager jobManager) {
6865
// We fork in innerTaskOperation(...), so we can use ThreadPool.Names.SAME here:
6966
super(settings, CloseJobAction.NAME, threadPool, clusterService, transportService, actionFilters,
7067
indexNameExpressionResolver, CloseJobAction.Request::new, CloseJobAction.Response::new, ThreadPool.Names.SAME);
71-
this.client = client;
7268
this.clusterService = clusterService;
7369
this.auditor = auditor;
7470
this.persistentTasksService = persistentTasksService;
75-
this.jobConfigProvider = jobConfigProvider;
7671
this.datafeedConfigProvider = datafeedConfigProvider;
72+
this.jobManager = jobManager;
7773
}
7874

7975
@Override
@@ -107,7 +103,7 @@ protected void doExecute(Task task, CloseJobAction.Request request, ActionListen
107103
*/
108104

109105
PersistentTasksCustomMetaData tasksMetaData = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
110-
jobConfigProvider.expandJobsIds(request.getJobId(), request.allowNoJobs(), true, ActionListener.wrap(
106+
jobManager.expandJobIds(request.getJobId(), request.allowNoJobs(), ActionListener.wrap(
111107
expandedJobIds -> {
112108
validate(expandedJobIds, request.isForce(), tasksMetaData, ActionListener.wrap(
113109
response -> {

0 commit comments

Comments
 (0)