Skip to content

Commit 30d8fa4

Browse files
przemekwitekprzemekwitek
authored
[ML] Add _cat/ml/data_frame/analytics API (#52260)
1 parent 1aeda20 commit 30d8fa4

File tree

7 files changed

+386
-21
lines changed

7 files changed

+386
-21
lines changed

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,7 @@
250250
import org.elasticsearch.xpack.ml.rest.calendar.RestPostCalendarEventAction;
251251
import org.elasticsearch.xpack.ml.rest.calendar.RestPutCalendarAction;
252252
import org.elasticsearch.xpack.ml.rest.calendar.RestPutCalendarJobAction;
253+
import org.elasticsearch.xpack.ml.rest.cat.RestCatDataFrameAnalyticsAction;
253254
import org.elasticsearch.xpack.ml.rest.cat.RestCatDatafeedsAction;
254255
import org.elasticsearch.xpack.ml.rest.cat.RestCatJobsAction;
255256
import org.elasticsearch.xpack.ml.rest.cat.RestCatTrainedModelsAction;
@@ -770,7 +771,8 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
770771
// CAT Handlers
771772
new RestCatJobsAction(),
772773
new RestCatTrainedModelsAction(),
773-
new RestCatDatafeedsAction()
774+
new RestCatDatafeedsAction(),
775+
new RestCatDataFrameAnalyticsAction()
774776
);
775777
}
776778

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDataFrameAnalyticsAction.java‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.ml.rest.cat;
7+
8+
import org.elasticsearch.client.node.NodeClient;
9+
import org.elasticsearch.cluster.metadata.MetaData;
10+
import org.elasticsearch.cluster.node.DiscoveryNode;
11+
import org.elasticsearch.common.Strings;
12+
import org.elasticsearch.common.Table;
13+
import org.elasticsearch.rest.RestRequest;
14+
import org.elasticsearch.rest.RestResponse;
15+
import org.elasticsearch.rest.action.RestActionListener;
16+
import org.elasticsearch.rest.action.RestResponseListener;
17+
import org.elasticsearch.rest.action.cat.AbstractCatAction;
18+
import org.elasticsearch.rest.action.cat.RestTable;
19+
import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction;
20+
import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction;
21+
import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction.Response.Stats;
22+
import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
23+
import org.elasticsearch.xpack.core.ml.utils.PhaseProgress;
24+
25+
import java.util.List;
26+
import java.util.Map;
27+
import java.util.function.Function;
28+
29+
import static java.util.Arrays.asList;
30+
import static java.util.Collections.unmodifiableList;
31+
import static java.util.stream.Collectors.joining;
32+
import static java.util.stream.Collectors.toMap;
33+
import static org.elasticsearch.rest.RestRequest.Method.GET;
34+
35+
public class RestCatDataFrameAnalyticsAction extends AbstractCatAction {
36+
37+
@Override
38+
public List<Route> routes() {
39+
return unmodifiableList(asList(
40+
new Route(GET, "_cat/ml/data_frame/analytics/{" + DataFrameAnalyticsConfig.ID.getPreferredName() + "}"),
41+
new Route(GET, "_cat/ml/data_frame/analytics")));
42+
}
43+
44+
@Override
45+
public String getName() {
46+
return "cat_ml_get_data_frame_analytics_action";
47+
}
48+
49+
@Override
50+
protected RestChannelConsumer doCatRequest(RestRequest restRequest, NodeClient client) {
51+
String dataFrameAnalyticsId = restRequest.param(DataFrameAnalyticsConfig.ID.getPreferredName());
52+
if (Strings.isNullOrEmpty(dataFrameAnalyticsId)) {
53+
dataFrameAnalyticsId = MetaData.ALL;
54+
}
55+
56+
GetDataFrameAnalyticsAction.Request getRequest = new GetDataFrameAnalyticsAction.Request(dataFrameAnalyticsId);
57+
getRequest.setAllowNoResources(
58+
restRequest.paramAsBoolean(
59+
GetDataFrameAnalyticsAction.Request.ALLOW_NO_MATCH.getPreferredName(), getRequest.isAllowNoResources()));
60+
61+
GetDataFrameAnalyticsStatsAction.Request getStatsRequest = new GetDataFrameAnalyticsStatsAction.Request(dataFrameAnalyticsId);
62+
getStatsRequest.setAllowNoMatch(true);
63+
64+
return channel -> client.execute(GetDataFrameAnalyticsAction.INSTANCE, getRequest, new RestActionListener<>(channel) {
65+
@Override
66+
public void processResponse(GetDataFrameAnalyticsAction.Response getResponse) {
67+
client.execute(GetDataFrameAnalyticsStatsAction.INSTANCE, getStatsRequest, new RestResponseListener<>(channel) {
68+
@Override
69+
public RestResponse buildResponse(GetDataFrameAnalyticsStatsAction.Response getStatsResponse) throws Exception {
70+
return RestTable.buildResponse(buildTable(getResponse, getStatsResponse), channel);
71+
}
72+
});
73+
}
74+
});
75+
}
76+
77+
@Override
78+
protected void documentation(StringBuilder sb) {
79+
sb.append("/_cat/ml/data_frame/analytics\n");
80+
sb.append("/_cat/ml/data_frame/analytics/{").append(DataFrameAnalyticsConfig.ID.getPreferredName()).append("}\n");
81+
}
82+
83+
@Override
84+
protected Table getTableWithHeader(RestRequest unused) {
85+
return getTableWithHeader();
86+
}
87+
88+
private static Table getTableWithHeader() {
89+
return new Table()
90+
.startHeaders()
91+
// DFA config info
92+
.addCell("id", TableColumnAttributeBuilder.builder("the id").build())
93+
.addCell("type",
94+
TableColumnAttributeBuilder.builder("analysis type")
95+
.setAliases("t")
96+
.build())
97+
.addCell("create_time",
98+
TableColumnAttributeBuilder.builder("job creation time")
99+
.setAliases("ct", "createTime")
100+
.build())
101+
.addCell("version",
102+
TableColumnAttributeBuilder.builder("the version of Elasticsearch when the analytics was created", false)
103+
.setAliases("v")
104+
.build())
105+
.addCell("source_index",
106+
TableColumnAttributeBuilder.builder("source index", false)
107+
.setAliases("si", "sourceIndex")
108+
.build())
109+
.addCell("dest_index",
110+
TableColumnAttributeBuilder.builder("destination index", false)
111+
.setAliases("di", "destIndex")
112+
.build())
113+
.addCell("description",
114+
TableColumnAttributeBuilder.builder("description", false)
115+
.setAliases("d")
116+
.build())
117+
.addCell("model_memory_limit",
118+
TableColumnAttributeBuilder.builder("model memory limit", false)
119+
.setAliases("mml", "modelMemoryLimit")
120+
.build())
121+
// DFA stats info
122+
.addCell("state",
123+
TableColumnAttributeBuilder.builder("job state")
124+
.setAliases("s")
125+
.setTextAlignment(TableColumnAttributeBuilder.TextAlign.RIGHT)
126+
.build())
127+
.addCell("failure_reason",
128+
TableColumnAttributeBuilder.builder("failure reason", false)
129+
.setAliases("fr", "failureReason")
130+
.build())
131+
.addCell("progress",
132+
TableColumnAttributeBuilder.builder("progress", false)
133+
.setAliases("p")
134+
.build())
135+
.addCell("assignment_explanation",
136+
TableColumnAttributeBuilder.builder("why the job is or is not assigned to a node", false)
137+
.setAliases("ae", "assignmentExplanation")
138+
.build())
139+
// Node info
140+
.addCell("node.id",
141+
TableColumnAttributeBuilder.builder("id of the assigned node", false)
142+
.setAliases("ni", "nodeId")
143+
.build())
144+
.addCell("node.name",
145+
TableColumnAttributeBuilder.builder("name of the assigned node", false)
146+
.setAliases("nn", "nodeName")
147+
.build())
148+
.addCell("node.ephemeral_id",
149+
TableColumnAttributeBuilder.builder("ephemeral id of the assigned node", false)
150+
.setAliases("ne", "nodeEphemeralId")
151+
.build())
152+
.addCell("node.address",
153+
TableColumnAttributeBuilder.builder("network address of the assigned node", false)
154+
.setAliases("na", "nodeAddress")
155+
.build())
156+
.endHeaders();
157+
}
158+
159+
private static Table buildTable(GetDataFrameAnalyticsAction.Response getResponse,
160+
GetDataFrameAnalyticsStatsAction.Response getStatsResponse) {
161+
Map<String, Stats> statsById = getStatsResponse.getResponse().results().stream().collect(toMap(Stats::getId, Function.identity()));
162+
Table table = getTableWithHeader();
163+
for (DataFrameAnalyticsConfig config : getResponse.getResources().results()) {
164+
Stats stats = statsById.get(config.getId());
165+
DiscoveryNode node = stats == null ? null : stats.getNode();
166+
table
167+
.startRow()
168+
.addCell(config.getId())
169+
.addCell(config.getAnalysis().getWriteableName())
170+
.addCell(config.getCreateTime())
171+
.addCell(config.getVersion())
172+
.addCell(String.join(",", config.getSource().getIndex()))
173+
.addCell(config.getDest().getIndex())
174+
.addCell(config.getDescription())
175+
.addCell(config.getModelMemoryLimit())
176+
.addCell(stats == null ? null : stats.getState())
177+
.addCell(stats == null ? null : stats.getFailureReason())
178+
.addCell(stats == null ? null : progressToString(stats.getProgress()))
179+
.addCell(stats == null ? null : stats.getAssignmentExplanation())
180+
.addCell(node == null ? null : node.getId())
181+
.addCell(node == null ? null : node.getName())
182+
.addCell(node == null ? null : node.getEphemeralId())
183+
.addCell(node == null ? null : node.getAddress().toString())
184+
.endRow();
185+
}
186+
return table;
187+
}
188+
189+
private static String progressToString(List<PhaseProgress> phases) {
190+
return phases.stream().map(p -> p.getPhase() + ":" + p.getProgressPercent()).collect(joining(","));
191+
}
192+
}

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -66,12 +66,12 @@ protected Table getTableWithHeader(RestRequest request) {
6666
table.startHeaders();
6767

6868
// Datafeed Info
69-
table.addCell("id", TableColumnAttributeBuilder.builder().setDescription("the datafeed_id").build());
70-
table.addCell("state", TableColumnAttributeBuilder.builder()
71-
.setDescription("the datafeed state")
72-
.setAliases("s")
73-
.setTextAlignment(TableColumnAttributeBuilder.TextAlign.RIGHT)
74-
.build());
69+
table.addCell("id", TableColumnAttributeBuilder.builder("the datafeed_id").build());
70+
table.addCell("state",
71+
TableColumnAttributeBuilder.builder("the datafeed state")
72+
.setAliases("s")
73+
.setTextAlignment(TableColumnAttributeBuilder.TextAlign.RIGHT)
74+
.build());
7575
table.addCell("assignment_explanation",
7676
TableColumnAttributeBuilder.builder("why the datafeed is or is not assigned to a node", false)
7777
.setAliases("ae")

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -71,17 +71,15 @@ protected Table getTableWithHeader(RestRequest request) {
7171
table.startHeaders();
7272

7373
// Job Info
74-
table.addCell("id", TableColumnAttributeBuilder.builder().setDescription("the job_id").build());
75-
table.addCell("state", TableColumnAttributeBuilder.builder()
76-
.setDescription("the job state")
77-
.setAliases("s")
78-
.setTextAlignment(TableColumnAttributeBuilder.TextAlign.RIGHT)
79-
.build());
74+
table.addCell("id", TableColumnAttributeBuilder.builder("the job_id").build());
75+
table.addCell("state",
76+
TableColumnAttributeBuilder.builder("the job state")
77+
.setAliases("s")
78+
.setTextAlignment(TableColumnAttributeBuilder.TextAlign.RIGHT)
79+
.build());
8080
table.addCell("opened_time",
81-
TableColumnAttributeBuilder.builder()
82-
.setDescription("the amount of time the job has been opened")
81+
TableColumnAttributeBuilder.builder("the amount of time the job has been opened", false)
8382
.setAliases("ot")
84-
.setDisplayByDefault(false)
8583
.build());
8684
table.addCell("assignment_explanation",
8785
TableColumnAttributeBuilder.builder("why the job is or is not assigned to a node", false)

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -128,17 +128,15 @@ protected Table getTableWithHeader(RestRequest request) {
128128
table.startHeaders();
129129

130130
// Trained Model Info
131-
table.addCell("id", TableColumnAttributeBuilder.builder().setDescription("the trained model id").build());
131+
table.addCell("id", TableColumnAttributeBuilder.builder("the trained model id").build());
132132
table.addCell("created_by", TableColumnAttributeBuilder.builder("who created the model", false)
133133
.setAliases("c", "createdBy")
134134
.setTextAlignment(TableColumnAttributeBuilder.TextAlign.RIGHT)
135135
.build());
136-
table.addCell("heap_size", TableColumnAttributeBuilder.builder()
137-
.setDescription("the estimated heap size to keep the model in memory")
136+
table.addCell("heap_size", TableColumnAttributeBuilder.builder("the estimated heap size to keep the model in memory")
138137
.setAliases("hs","modelHeapSize")
139138
.build());
140-
table.addCell("operations", TableColumnAttributeBuilder.builder()
141-
.setDescription("the estimated number of operations to use the model")
139+
table.addCell("operations", TableColumnAttributeBuilder.builder("the estimated number of operations to use the model")
142140
.setAliases("o", "modelOperations")
143141
.build());
144142
table.addCell("license", TableColumnAttributeBuilder.builder("The license level of the model", false)

‎x-pack/plugin/src/test/resources/rest-api-spec/api/cat.ml_data_frame_analytics.json‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
{
2+
"cat.ml_data_frame_analytics":{
3+
"documentation":{
4+
"url":"http://www.elastic.co/guide/en/elasticsearch/reference/current/get-dfanalytics-stats.html"
5+
},
6+
"stability":"stable",
7+
"url":{
8+
"paths":[
9+
{
10+
"path":"/_cat/ml/data_frame/analytics",
11+
"methods":[
12+
"GET"
13+
]
14+
},
15+
{
16+
"path":"/_cat/ml/data_frame/analytics/{id}",
17+
"methods":[
18+
"GET"
19+
],
20+
"parts":{
21+
"id":{
22+
"type":"string",
23+
"description":"The ID of the data frame analytics to fetch"
24+
}
25+
}
26+
}
27+
]
28+
},
29+
"params":{
30+
"allow_no_match":{
31+
"type":"boolean",
32+
"required":false,
33+
"description":"Whether to ignore if a wildcard expression matches no configs. (This includes `_all` string or when no configs have been specified)"
34+
},
35+
"bytes":{
36+
"type":"enum",
37+
"description":"The unit in which to display byte values",
38+
"options":[
39+
"b",
40+
"k",
41+
"kb",
42+
"m",
43+
"mb",
44+
"g",
45+
"gb",
46+
"t",
47+
"tb",
48+
"p",
49+
"pb"
50+
]
51+
},
52+
"format":{
53+
"type":"string",
54+
"description":"a short version of the Accept header, e.g. json, yaml"
55+
},
56+
"h":{
57+
"type":"list",
58+
"description":"Comma-separated list of column names to display"
59+
},
60+
"help":{
61+
"type":"boolean",
62+
"description":"Return help information",
63+
"default":false
64+
},
65+
"s":{
66+
"type":"list",
67+
"description":"Comma-separated list of column names or column aliases to sort by"
68+
},
69+
"time":{
70+
"type":"enum",
71+
"description":"The unit in which to display time values",
72+
"options":[
73+
"d (Days)",
74+
"h (Hours)",
75+
"m (Minutes)",
76+
"s (Seconds)",
77+
"ms (Milliseconds)",
78+
"micros (Microseconds)",
79+
"nanos (Nanoseconds)"
80+
]
81+
},
82+
"v":{
83+
"type":"boolean",
84+
"description":"Verbose mode. Display column headers",
85+
"default":false
86+
}
87+
}
88+
}
89+
}

0 commit comments

Comments
 (0)