Skip to content

Commit 82c1ddc

Browse files
benwtrentbenwtrent
authored
[7.x] [ML][Data Frame] Add deduced mappings to _preview response payload (#43742) (#43849)
* [ML][Data Frame] Add deduced mappings to _preview response payload (#43742) * [ML][Data Frame] Add deduced mappings to _preview response payload * updating preview docs * fixing code for backport
1 parent b977f01 commit 82c1ddc

File tree

9 files changed

+130
-29
lines changed

9 files changed

+130
-29
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponse.java‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,23 +29,32 @@
2929
public class PreviewDataFrameTransformResponse {
3030

3131
private static final String PREVIEW = "preview";
32+
private static final String MAPPINGS = "mappings";
3233

3334
@SuppressWarnings("unchecked")
3435
public static PreviewDataFrameTransformResponse fromXContent(final XContentParser parser) throws IOException {
35-
Object previewDocs = parser.map().get(PREVIEW);
36-
return new PreviewDataFrameTransformResponse((List<Map<String, Object>>) previewDocs);
36+
Map<String, Object> previewMap = parser.mapOrdered();
37+
Object previewDocs = previewMap.get(PREVIEW);
38+
Object mappings = previewMap.get(MAPPINGS);
39+
return new PreviewDataFrameTransformResponse((List<Map<String, Object>>) previewDocs, (Map<String, Object>) mappings);
3740
}
3841

3942
private List<Map<String, Object>> docs;
43+
private Map<String, Object> mappings;
4044

41-
public PreviewDataFrameTransformResponse(List<Map<String, Object>> docs) {
45+
public PreviewDataFrameTransformResponse(List<Map<String, Object>> docs, Map<String, Object> mappings) {
4246
this.docs = docs;
47+
this.mappings = mappings;
4348
}
4449

4550
public List<Map<String, Object>> getDocs() {
4651
return docs;
4752
}
4853

54+
public Map<String, Object> getMappings() {
55+
return mappings;
56+
}
57+
4958
@Override
5059
public boolean equals(Object obj) {
5160
if (obj == this) {
@@ -57,12 +66,12 @@ public boolean equals(Object obj) {
5766
}
5867

5968
PreviewDataFrameTransformResponse other = (PreviewDataFrameTransformResponse) obj;
60-
return Objects.equals(other.docs, docs);
69+
return Objects.equals(other.docs, docs) && Objects.equals(other.mappings, mappings);
6170
}
6271

6372
@Override
6473
public int hashCode() {
65-
return Objects.hashCode(docs);
74+
return Objects.hash(docs, mappings);
6675
}
6776

6877
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060

6161
import java.io.IOException;
6262
import java.util.ArrayList;
63+
import java.util.Collections;
6364
import java.util.HashMap;
6465
import java.util.List;
6566
import java.util.Map;
@@ -71,6 +72,7 @@
7172
import static org.hamcrest.Matchers.empty;
7273
import static org.hamcrest.Matchers.equalTo;
7374
import static org.hamcrest.Matchers.greaterThan;
75+
import static org.hamcrest.Matchers.hasKey;
7476
import static org.hamcrest.Matchers.hasSize;
7577
import static org.hamcrest.Matchers.is;
7678
import static org.hamcrest.Matchers.oneOf;
@@ -277,6 +279,7 @@ public void testStartStop() throws IOException {
277279
assertThat(taskState, is(DataFrameTransformTaskState.STOPPED));
278280
}
279281

282+
@SuppressWarnings("unchecked")
280283
public void testPreview() throws IOException {
281284
String sourceIndex = "transform-source";
282285
createIndex(sourceIndex);
@@ -298,6 +301,12 @@ public void testPreview() throws IOException {
298301
Optional<Map<String, Object>> michel = docs.stream().filter(doc -> "michel".equals(doc.get("reviewer"))).findFirst();
299302
assertTrue(michel.isPresent());
300303
assertEquals(3.6d, (double) michel.get().get("avg_rating"), 0.1d);
304+
305+
Map<String, Object> mappings = preview.getMappings();
306+
assertThat(mappings, hasKey("properties"));
307+
Map<String, Object> fields = (Map<String, Object>)mappings.get("properties");
308+
assertThat(fields.get("reviewer"), equalTo(Collections.singletonMap("type", "keyword")));
309+
assertThat(fields.get("avg_rating"), equalTo(Collections.singletonMap("type", "double")));
301310
}
302311

303312
private DataFrameTransformConfig validDataFrameTransformConfig(String id, String source, String destination) {

‎client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponseTests.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424

2525
import java.io.IOException;
2626
import java.util.ArrayList;
27+
import java.util.Collections;
2728
import java.util.HashMap;
2829
import java.util.List;
2930
import java.util.Map;
@@ -53,8 +54,13 @@ private PreviewDataFrameTransformResponse createTestInstance() {
5354
}
5455
docs.add(doc);
5556
}
57+
int numMappingEntries = randomIntBetween(5, 10);
58+
Map<String, Object> mappings = new HashMap<>(numMappingEntries);
59+
for (int i = 0; i < numMappingEntries; i++) {
60+
mappings.put(randomAlphaOfLength(10), Collections.singletonMap("type", randomAlphaOfLength(10)));
61+
}
5662

57-
return new PreviewDataFrameTransformResponse(docs);
63+
return new PreviewDataFrameTransformResponse(docs, mappings);
5864
}
5965

6066
private void toXContent(PreviewDataFrameTransformResponse response, XContentBuilder builder) throws IOException {
@@ -64,6 +70,7 @@ private void toXContent(PreviewDataFrameTransformResponse response, XContentBuil
6470
builder.map(doc);
6571
}
6672
builder.endArray();
73+
builder.field("mappings", response.getMappings());
6774
builder.endObject();
6875
}
6976
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -447,6 +447,7 @@ public void testPreview() throws IOException, InterruptedException {
447447
// end::preview-data-frame-transform-execute
448448

449449
assertNotNull(response.getDocs());
450+
assertNotNull(response.getMappings());
450451
}
451452
{
452453
// tag::preview-data-frame-transform-execute-listener

‎docs/reference/data-frames/apis/preview-transform.asciidoc‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,17 @@ The data that is returned for this example is as follows:
9090
"customer_id" : "12"
9191
}
9292
...
93-
]
93+
],
94+
"mappings": {
95+
"properties": {
96+
"max_price": {
97+
"type": "double"
98+
},
99+
"customer_id": {
100+
"type": "keyword"
101+
}
102+
}
103+
}
94104
}
95105
----
96106
// NOTCONSOLE

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformAction.java‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
package org.elasticsearch.xpack.core.dataframe.action;
88

9+
import org.elasticsearch.Version;
910
import org.elasticsearch.action.ActionType;
1011
import org.elasticsearch.action.ActionRequestValidationException;
1112
import org.elasticsearch.action.ActionResponse;
@@ -28,6 +29,7 @@
2829

2930
import java.io.IOException;
3031
import java.util.ArrayList;
32+
import java.util.Collections;
3133
import java.util.HashMap;
3234
import java.util.List;
3335
import java.util.Map;
@@ -137,11 +139,14 @@ public boolean equals(Object obj) {
137139
public static class Response extends ActionResponse implements ToXContentObject {
138140

139141
private List<Map<String, Object>> docs;
142+
private Map<String, Object> mappings;
140143
public static ParseField PREVIEW = new ParseField("preview");
144+
public static ParseField MAPPINGS = new ParseField("mappings");
141145

142146
static ObjectParser<Response, Void> PARSER = new ObjectParser<>("data_frame_transform_preview", Response::new);
143147
static {
144148
PARSER.declareObjectArray(Response::setDocs, (p, c) -> p.mapOrdered(), PREVIEW);
149+
PARSER.declareObject(Response::setMappings, (p, c) -> p.mapOrdered(), MAPPINGS);
145150
}
146151
public Response() {}
147152

@@ -151,6 +156,10 @@ public Response(StreamInput in) throws IOException {
151156
for (int i = 0; i < size; i++) {
152157
this.docs.add(in.readMap());
153158
}
159+
if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
160+
Map<String, Object> objectMap = in.readMap();
161+
this.mappings = objectMap == null ? null : Collections.unmodifiableMap(objectMap);
162+
}
154163
}
155164

156165
public Response(List<Map<String, Object>> docs) {
@@ -161,18 +170,56 @@ public void setDocs(List<Map<String, Object>> docs) {
161170
this.docs = new ArrayList<>(docs);
162171
}
163172

173+
public void setMappings(Map<String, Object> mappings) {
174+
this.mappings = Collections.unmodifiableMap(mappings);
175+
}
176+
177+
/**
178+
* This takes the a {@code Map<String, String>} of the type "fieldname: fieldtype" and transforms it into the
179+
* typical mapping format.
180+
*
181+
* Example:
182+
*
183+
* input:
184+
* {"field1.subField1": "long", "field2": "keyword"}
185+
*
186+
* output:
187+
* {
188+
* "properties": {
189+
* "field1.subField1": {
190+
* "type": "long"
191+
* },
192+
* "field2": {
193+
* "type": "keyword"
194+
* }
195+
* }
196+
* }
197+
* @param mappings A Map of the form {"fieldName": "fieldType"}
198+
*/
199+
public void setMappingsFromStringMap(Map<String, String> mappings) {
200+
Map<String, Object> fieldMappings = new HashMap<>();
201+
mappings.forEach((k, v) -> fieldMappings.put(k, Collections.singletonMap("type", v)));
202+
this.mappings = Collections.singletonMap("properties", fieldMappings);
203+
}
204+
164205
@Override
165206
public void writeTo(StreamOutput out) throws IOException {
166207
out.writeInt(docs.size());
167208
for (Map<String, Object> doc : docs) {
168209
out.writeMapWithConsistentOrder(doc);
169210
}
211+
if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
212+
out.writeMap(mappings);
213+
}
170214
}
171215

172216
@Override
173217
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
174218
builder.startObject();
175219
builder.field(PREVIEW.getPreferredName(), docs);
220+
if (mappings != null) {
221+
builder.field(MAPPINGS.getPreferredName(), mappings);
222+
}
176223
builder.endObject();
177224
return builder;
178225
}
@@ -188,12 +235,12 @@ public boolean equals(Object obj) {
188235
}
189236

190237
Response other = (Response) obj;
191-
return Objects.equals(other.docs, docs);
238+
return Objects.equals(other.docs, docs) && Objects.equals(other.mappings, mappings);
192239
}
193240

194241
@Override
195242
public int hashCode() {
196-
return Objects.hashCode(docs);
243+
return Objects.hash(docs, mappings);
197244
}
198245

199246
public static Response fromXContent(final XContentParser parser) throws IOException {

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformsActionResponseTests.java‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
import java.io.IOException;
1515
import java.util.ArrayList;
16+
import java.util.Collections;
1617
import java.util.HashMap;
1718
import java.util.List;
1819
import java.util.Map;
@@ -35,13 +36,28 @@ protected Response createTestInstance() {
3536
int size = randomIntBetween(0, 10);
3637
List<Map<String, Object>> data = new ArrayList<>(size);
3738
for (int i = 0; i < size; i++) {
38-
Map<String, Object> datum = new HashMap<>();
39-
Map<String, Object> entry = new HashMap<>();
40-
entry.put("value1", randomIntBetween(1, 100));
41-
datum.put(randomAlphaOfLength(10), entry);
42-
data.add(datum);
39+
data.add(Collections.singletonMap(randomAlphaOfLength(10), Collections.singletonMap("value1", randomIntBetween(1, 100))));
4340
}
44-
return new Response(data);
41+
42+
Response response = new Response(data);
43+
if (randomBoolean()) {
44+
size = randomIntBetween(0, 10);
45+
if (randomBoolean()) {
46+
Map<String, Object> mappings = new HashMap<>(size);
47+
for (int i = 0; i < size; i++) {
48+
mappings.put(randomAlphaOfLength(10), Collections.singletonMap("type", randomAlphaOfLength(10)));
49+
}
50+
response.setMappings(mappings);
51+
} else {
52+
Map<String, String> mappings = new HashMap<>(size);
53+
for (int i = 0; i < size; i++) {
54+
mappings.put(randomAlphaOfLength(10), randomAlphaOfLength(10));
55+
}
56+
response.setMappingsFromStringMap(mappings);
57+
}
58+
}
59+
60+
return response;
4561
}
4662

4763
@Override

‎x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -105,25 +105,16 @@ protected void doExecute(Task task,
105105

106106
Pivot pivot = new Pivot(config.getPivotConfig());
107107

108-
getPreview(pivot,
109-
config.getSource(),
110-
config.getDestination().getPipeline(),
111-
config.getDestination().getIndex(),
112-
ActionListener.wrap(
113-
previewResponse -> listener.onResponse(new PreviewDataFrameTransformAction.Response(previewResponse)),
114-
error -> {
115-
logger.error("Failure gathering preview", error);
116-
listener.onFailure(error);
117-
}
118-
));
108+
getPreview(pivot, config.getSource(), config.getDestination().getPipeline(), config.getDestination().getIndex(), listener);
119109
}
120110

121111
@SuppressWarnings("unchecked")
122112
private void getPreview(Pivot pivot,
123113
SourceConfig source,
124114
String pipeline,
125115
String dest,
126-
ActionListener<List<Map<String, Object>>> listener) {
116+
ActionListener<PreviewDataFrameTransformAction.Response> listener) {
117+
final PreviewDataFrameTransformAction.Response previewResponse = new PreviewDataFrameTransformAction.Response();
127118
ActionListener<SimulatePipelineResponse> pipelineResponseActionListener = ActionListener.wrap(
128119
simulatePipelineResponse -> {
129120
List<Map<String, Object>> response = new ArrayList<>(simulatePipelineResponse.getResults().size());
@@ -136,12 +127,14 @@ private void getPreview(Pivot pivot,
136127
response.add((Map<String, Object>)XContentMapValues.extractValue("doc._source", tempMap));
137128
}
138129
}
139-
listener.onResponse(response);
130+
previewResponse.setDocs(response);
131+
listener.onResponse(previewResponse);
140132
},
141133
listener::onFailure
142134
);
143135
pivot.deduceMappings(client, source, ActionListener.wrap(
144136
deducedMappings -> {
137+
previewResponse.setMappingsFromStringMap(deducedMappings);
145138
ClientHelper.executeWithHeadersAsync(threadPool.getThreadContext().getHeaders(),
146139
ClientHelper.DATA_FRAME_ORIGIN,
147140
client,
@@ -158,7 +151,8 @@ private void getPreview(Pivot pivot,
158151
List<Map<String, Object>> results = pivot.extractResults(agg, deducedMappings, stats)
159152
.peek(doc -> doc.keySet().removeIf(k -> k.startsWith("_")))
160153
.collect(Collectors.toList());
161-
listener.onResponse(results);
154+
previewResponse.setDocs(results);
155+
listener.onResponse(previewResponse);
162156
} else {
163157
List<Map<String, Object>> results = pivot.extractResults(agg, deducedMappings, stats)
164158
.map(doc -> {

‎x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/preview_transforms.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,11 @@ setup:
9898
- match: { preview.2.avg_response: 42.0 }
9999
- match: { preview.2.time.max: "2017-02-18T01:01:00.000Z" }
100100
- match: { preview.2.time.min: "2017-02-18T01:01:00.000Z" }
101+
- match: { mappings.properties.airline.type: "keyword" }
102+
- match: { mappings.properties.by-hour.type: "date" }
103+
- match: { mappings.properties.avg_response.type: "double" }
104+
- match: { mappings.properties.time\.max.type: "date" }
105+
- match: { mappings.properties.time\.min.type: "date" }
101106

102107
- do:
103108
ingest.put_pipeline:
@@ -141,6 +146,9 @@ setup:
141146
- match: { preview.2.by-hour: 1487379600000 }
142147
- match: { preview.2.avg_response: 42.0 }
143148
- match: { preview.2.my_field: 42 }
149+
- match: { mappings.properties.airline.type: "keyword" }
150+
- match: { mappings.properties.by-hour.type: "date" }
151+
- match: { mappings.properties.avg_response.type: "double" }
144152

145153
---
146154
"Test preview transform with invalid config":

0 commit comments

Comments
 (0)