Skip to content

Commit f45a85e

Browse files
przemekwitekprzemekwitek
authored
[7.x] [Transform] Handle multi-fields properly when creating destination index. (#66273) (#66430)
1 parent e991068 commit f45a85e

File tree

5 files changed

+208
-37
lines changed

5 files changed

+208
-37
lines changed

‎x-pack/plugin/src/test/resources/rest-api-spec/test/transform/preview_transforms.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,8 @@ setup:
101101
- match: { generated_dest_index.mappings.properties.airline.type: "keyword" }
102102
- match: { generated_dest_index.mappings.properties.by-hour.type: "date" }
103103
- match: { generated_dest_index.mappings.properties.avg_response.type: "double" }
104-
- match: { generated_dest_index.mappings.properties.time\.max.type: "date" }
105-
- match: { generated_dest_index.mappings.properties.time\.min.type: "date" }
104+
- match: { generated_dest_index.mappings.properties.time.properties.max.type: "date" }
105+
- match: { generated_dest_index.mappings.properties.time.properties.min.type: "date" }
106106

107107
- do:
108108
ingest.put_pipeline:

‎x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/LatestIT.java‎

Lines changed: 51 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@
2626
import java.util.Map;
2727
import java.util.stream.Stream;
2828

29+
import static java.util.Collections.singletonMap;
2930
import static java.util.stream.Collectors.toList;
31+
import static java.util.stream.Collectors.toMap;
3032
import static org.hamcrest.Matchers.containsInAnyOrder;
3133
import static org.hamcrest.Matchers.equalTo;
3234
import static org.hamcrest.Matchers.hasSize;
@@ -55,8 +57,10 @@ private static final String getDateStringForRow(int row) {
5557
private static final String BUSINESS_ID = "business_id";
5658
private static final String COUNT = "count";
5759
private static final String STARS = "stars";
60+
private static final String COMMENT = "comment";
5861

59-
private static final Map<String, Object> row(String userId, String businessId, int count, int stars, String timestamp) {
62+
private static final Map<String, Object> row(
63+
String userId, String businessId, int count, int stars, String timestamp, String comment) {
6064
return new HashMap<String, Object>() {{
6165
if (userId != null) {
6266
put(USER_ID, userId);
@@ -65,40 +69,43 @@ private static final Map<String, Object> row(String userId, String businessId, i
6569
put(COUNT, count);
6670
put(STARS, stars);
6771
put(TIMESTAMP, timestamp);
72+
put(COMMENT, comment);
73+
put("regular_object", singletonMap("foo", 42));
74+
put("nested_object", singletonMap("bar", 43));
6875
}};
6976
}
7077

7178
private static final Object[] EXPECTED_DEST_INDEX_ROWS =
7279
new Object[] {
73-
row("user_0", "business_37", 87, 2, "2017-04-04T12:30:00Z"),
74-
row("user_1", "business_38", 88, 3, "2017-04-05T12:30:00Z"),
75-
row("user_2", "business_39", 89, 4, "2017-04-06T12:30:00Z"),
76-
row("user_3", "business_40", 90, 0, "2017-04-07T12:30:00Z"),
77-
row("user_4", "business_41", 91, 1, "2017-04-08T12:30:00Z"),
78-
row("user_5", "business_42", 92, 2, "2017-04-09T12:30:00Z"),
79-
row("user_6", "business_43", 93, 3, "2017-04-10T12:30:00Z"),
80-
row("user_7", "business_44", 94, 4, "2017-04-11T12:30:00Z"),
81-
row("user_8", "business_45", 95, 0, "2017-04-12T12:30:00Z"),
82-
row("user_9", "business_46", 96, 1, "2017-04-13T12:30:00Z"),
83-
row("user_10", "business_47", 97, 2, "2017-04-14T12:30:00Z"),
84-
row("user_11", "business_48", 98, 3, "2017-04-15T12:30:00Z"),
85-
row("user_12", "business_49", 99, 4, "2017-04-16T12:30:00Z"),
86-
row("user_13", "business_21", 71, 1, "2017-03-16T12:30:00Z"),
87-
row("user_14", "business_22", 72, 2, "2017-03-17T12:30:00Z"),
88-
row("user_15", "business_23", 73, 3, "2017-03-18T12:30:00Z"),
89-
row("user_16", "business_24", 74, 4, "2017-03-19T12:30:00Z"),
90-
row("user_17", "business_25", 75, 0, "2017-03-20T12:30:00Z"),
91-
row("user_18", "business_26", 76, 1, "2017-03-21T12:30:00Z"),
92-
row("user_19", "business_27", 77, 2, "2017-03-22T12:30:00Z"),
93-
row("user_20", "business_28", 78, 3, "2017-03-23T12:30:00Z"),
94-
row("user_21", "business_29", 79, 4, "2017-03-24T12:30:00Z"),
95-
row("user_22", "business_30", 80, 0, "2017-03-25T12:30:00Z"),
96-
row("user_23", "business_31", 81, 1, "2017-03-26T12:30:00Z"),
97-
row("user_24", "business_32", 82, 2, "2017-03-27T12:30:00Z"),
98-
row("user_25", "business_33", 83, 3, "2017-03-28T12:30:00Z"),
99-
row("user_26", "business_34", 84, 4, "2017-04-01T12:30:00Z"),
100-
row("user_27", "business_35", 85, 0, "2017-04-02T12:30:00Z"),
101-
row(null, "business_36", 86, 1, "2017-04-03T12:30:00Z")
80+
row("user_0", "business_37", 87, 2, "2017-04-04T12:30:00Z", "Great stuff, deserves 2 stars"),
81+
row("user_1", "business_38", 88, 3, "2017-04-05T12:30:00Z", "Great stuff, deserves 3 stars"),
82+
row("user_2", "business_39", 89, 4, "2017-04-06T12:30:00Z", "Great stuff, deserves 4 stars"),
83+
row("user_3", "business_40", 90, 0, "2017-04-07T12:30:00Z", "Great stuff, deserves 0 stars"),
84+
row("user_4", "business_41", 91, 1, "2017-04-08T12:30:00Z", "Great stuff, deserves 1 stars"),
85+
row("user_5", "business_42", 92, 2, "2017-04-09T12:30:00Z", "Great stuff, deserves 2 stars"),
86+
row("user_6", "business_43", 93, 3, "2017-04-10T12:30:00Z", "Great stuff, deserves 3 stars"),
87+
row("user_7", "business_44", 94, 4, "2017-04-11T12:30:00Z", "Great stuff, deserves 4 stars"),
88+
row("user_8", "business_45", 95, 0, "2017-04-12T12:30:00Z", "Great stuff, deserves 0 stars"),
89+
row("user_9", "business_46", 96, 1, "2017-04-13T12:30:00Z", "Great stuff, deserves 1 stars"),
90+
row("user_10", "business_47", 97, 2, "2017-04-14T12:30:00Z", "Great stuff, deserves 2 stars"),
91+
row("user_11", "business_48", 98, 3, "2017-04-15T12:30:00Z", "Great stuff, deserves 3 stars"),
92+
row("user_12", "business_49", 99, 4, "2017-04-16T12:30:00Z", "Great stuff, deserves 4 stars"),
93+
row("user_13", "business_21", 71, 1, "2017-03-16T12:30:00Z", "Great stuff, deserves 1 stars"),
94+
row("user_14", "business_22", 72, 2, "2017-03-17T12:30:00Z", "Great stuff, deserves 2 stars"),
95+
row("user_15", "business_23", 73, 3, "2017-03-18T12:30:00Z", "Great stuff, deserves 3 stars"),
96+
row("user_16", "business_24", 74, 4, "2017-03-19T12:30:00Z", "Great stuff, deserves 4 stars"),
97+
row("user_17", "business_25", 75, 0, "2017-03-20T12:30:00Z", "Great stuff, deserves 0 stars"),
98+
row("user_18", "business_26", 76, 1, "2017-03-21T12:30:00Z", "Great stuff, deserves 1 stars"),
99+
row("user_19", "business_27", 77, 2, "2017-03-22T12:30:00Z", "Great stuff, deserves 2 stars"),
100+
row("user_20", "business_28", 78, 3, "2017-03-23T12:30:00Z", "Great stuff, deserves 3 stars"),
101+
row("user_21", "business_29", 79, 4, "2017-03-24T12:30:00Z", "Great stuff, deserves 4 stars"),
102+
row("user_22", "business_30", 80, 0, "2017-03-25T12:30:00Z", "Great stuff, deserves 0 stars"),
103+
row("user_23", "business_31", 81, 1, "2017-03-26T12:30:00Z", "Great stuff, deserves 1 stars"),
104+
row("user_24", "business_32", 82, 2, "2017-03-27T12:30:00Z", "Great stuff, deserves 2 stars"),
105+
row("user_25", "business_33", 83, 3, "2017-03-28T12:30:00Z", "Great stuff, deserves 3 stars"),
106+
row("user_26", "business_34", 84, 4, "2017-04-01T12:30:00Z", "Great stuff, deserves 4 stars"),
107+
row("user_27", "business_35", 85, 0, "2017-04-02T12:30:00Z", "Great stuff, deserves 0 stars"),
108+
row(null, "business_36", 86, 1, "2017-04-03T12:30:00Z", "Great stuff, deserves 1 stars")
102109
};
103110

104111
@After
@@ -162,11 +169,24 @@ public void testLatestPreview() throws Exception {
162169
GetMappingsResponse sourceIndexMapping =
163170
restClient.indices().getMapping(new GetMappingsRequest().indices(SOURCE_INDEX_NAME), RequestOptions.DEFAULT);
164171
assertThat(
165-
previewResponse.getMappings().get("properties"),
172+
// Mappings we get from preview sometimes contain redundant { "type": "object" } entries.
173+
// We clear them here to be able to compare with the GetMappingsAction output.
174+
clearDefaultObjectType(previewResponse.getMappings().get("properties")),
166175
is(equalTo(sourceIndexMapping.mappings().get(SOURCE_INDEX_NAME).sourceAsMap().get("properties"))));
167176
// Verify preview contents
168177
assertThat(previewResponse.getDocs(), hasSize(NUM_USERS + 1));
169178
assertThat(previewResponse.getDocs(), containsInAnyOrder(EXPECTED_DEST_INDEX_ROWS));
170179
}
171180
}
181+
182+
private static Object clearDefaultObjectType(Object obj) {
183+
if (obj instanceof Map == false) {
184+
return obj;
185+
}
186+
@SuppressWarnings("unchecked")
187+
Map<String, Object> map = (Map<String, Object>) obj;
188+
return map.entrySet().stream()
189+
.filter(entry -> (entry.getKey().equals("type") && entry.getValue().equals("object")) == false)
190+
.collect(toMap(entry -> entry.getKey(), entry -> clearDefaultObjectType(entry.getValue())));
191+
}
172192
}

‎x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIntegTestCase.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,20 @@ protected void createReviewsIndex(String indexName,
342342
.startObject("stars")
343343
.field("type", "integer")
344344
.endObject()
345+
.startObject("regular_object")
346+
.field("type", "object")
347+
.endObject()
348+
.startObject("nested_object")
349+
.field("type", "nested")
350+
.endObject()
351+
.startObject("comment")
352+
.field("type", "text")
353+
.startObject("fields")
354+
.startObject("keyword")
355+
.field("type", "keyword")
356+
.endObject()
357+
.endObject()
358+
.endObject()
345359
.endObject();
346360
}
347361
builder.endObject();
@@ -374,6 +388,10 @@ protected void createReviewsIndex(String indexName,
374388
.append(business)
375389
.append("\",\"stars\":")
376390
.append(stars)
391+
.append(",\"comment\":")
392+
.append("\"Great stuff, deserves " + stars + " stars\"")
393+
.append(",\"regular_object\":{\"foo\": 42}")
394+
.append(",\"nested_object\":{\"bar\": 43}")
377395
.append(",\"timestamp\":\"")
378396
.append(dateString)
379397
.append("\"}");

‎x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformIndex.java‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,24 +15,39 @@
1515
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
1616
import org.elasticsearch.client.Client;
1717
import org.elasticsearch.cluster.metadata.IndexMetadata;
18+
import org.elasticsearch.common.Strings;
1819
import org.elasticsearch.common.settings.Settings;
1920
import org.elasticsearch.index.mapper.MapperService;
21+
import org.elasticsearch.index.mapper.ObjectMapper;
2022
import org.elasticsearch.xpack.core.transform.TransformField;
2123
import org.elasticsearch.xpack.core.transform.TransformMessages;
2224
import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
2325
import org.elasticsearch.xpack.core.transform.transforms.TransformDestIndexSettings;
2426

2527
import java.time.Clock;
28+
import java.util.ArrayList;
29+
import java.util.Arrays;
2630
import java.util.Collections;
2731
import java.util.HashMap;
32+
import java.util.HashSet;
33+
import java.util.List;
2834
import java.util.Map;
2935
import java.util.Set;
3036

37+
import static java.util.Map.Entry.comparingByKey;
38+
3139
public final class TransformIndex {
3240
private static final Logger logger = LogManager.getLogger(TransformIndex.class);
3341

3442
public static final String DOC_TYPE = "_doc";
43+
/**
44+
* The list of object types used in the mappings.
45+
* We include {@code null} as an alternative for "object", which is the default.
46+
*/
47+
private static final Set<String> OBJECT_TYPES =
48+
new HashSet<>(Arrays.asList(null, ObjectMapper.CONTENT_TYPE, ObjectMapper.NESTED_CONTENT_TYPE));
3549
private static final String PROPERTIES = "properties";
50+
private static final String FIELDS = "fields";
3651
private static final String META = "_meta";
3752

3853
private TransformIndex() {}
@@ -137,10 +152,28 @@ private static Settings createSettings() {
137152
* }
138153
* @param mappings A Map of the form {"fieldName": "fieldType"}
139154
*/
140-
private static Map<String, Object> createMappingsFromStringMap(Map<String, String> mappings) {
155+
static Map<String, Object> createMappingsFromStringMap(Map<String, String> mappings) {
156+
List<Map.Entry<String, String>> sortedMappingsEntries = new ArrayList<>(mappings.entrySet());
157+
// We sort the entry list to make sure that for each (parent, parent.child) pair, parent entry will be processed before child entry.
158+
sortedMappingsEntries.sort(comparingByKey());
141159
Map<String, Object> fieldMappings = new HashMap<>();
142-
mappings.forEach((k, v) -> fieldMappings.put(k, Collections.singletonMap("type", v)));
143-
160+
for (Map.Entry<String, String> entry : sortedMappingsEntries) {
161+
String[] parts = Strings.tokenizeToStringArray(entry.getKey(), ".");
162+
String type = entry.getValue();
163+
Map<String, Object> current = fieldMappings;
164+
current = diveInto(current, parts[0]);
165+
for (int j = 1; j < parts.length; ++j) {
166+
// Here we decide whether a dot ('.') means inner object or a multi-field.
167+
current = diveInto(current, OBJECT_TYPES.contains(current.get("type")) ? PROPERTIES : FIELDS);
168+
current = diveInto(current, parts[j]);
169+
}
170+
current.put("type", type);
171+
}
144172
return fieldMappings;
145173
}
174+
175+
@SuppressWarnings("unchecked")
176+
private static Map<String, Object> diveInto(Map<String, Object> map, String key) {
177+
return (Map<String, Object>) map.computeIfAbsent(key, k -> new HashMap<>());
178+
}
146179
}

‎x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformIndexTests.java‎

Lines changed: 101 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,12 @@
2323
import java.util.HashMap;
2424
import java.util.Map;
2525

26+
import static java.util.Collections.emptyMap;
27+
import static java.util.Collections.singletonMap;
2628
import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
29+
import static org.hamcrest.Matchers.anEmptyMap;
2730
import static org.hamcrest.Matchers.equalTo;
31+
import static org.hamcrest.Matchers.is;
2832
import static org.mockito.Matchers.any;
2933
import static org.mockito.Matchers.eq;
3034
import static org.mockito.Mockito.doAnswer;
@@ -52,7 +56,7 @@ public void testCreateDestinationIndex() throws IOException {
5256
TransformIndex.createDestinationIndex(
5357
client,
5458
TransformConfigTests.randomTransformConfig(TRANSFORM_ID),
55-
TransformIndex.createTransformDestIndexSettings(new HashMap<>(), TRANSFORM_ID, clock),
59+
TransformIndex.createTransformDestIndexSettings(new HashMap<String, String>(), TRANSFORM_ID, clock),
5660
ActionListener.wrap(value -> assertTrue(value), e -> fail(e.getMessage()))
5761
);
5862

@@ -68,4 +72,100 @@ public void testCreateDestinationIndex() throws IOException {
6872
assertThat(extractValue("_doc._meta.created_by", map), equalTo(CREATED_BY));
6973
}
7074
}
75+
76+
public void testCreateMappingsFromStringMap() {
77+
assertThat(TransformIndex.createMappingsFromStringMap(emptyMap()), is(anEmptyMap()));
78+
assertThat(
79+
TransformIndex.createMappingsFromStringMap(singletonMap("a", "long")),
80+
is(equalTo(singletonMap("a", singletonMap("type", "long"))))
81+
);
82+
assertThat(
83+
TransformIndex.createMappingsFromStringMap(new HashMap<String, String>() {{
84+
put("a", "long");
85+
put("b", "keyword");
86+
}}),
87+
is(equalTo(new HashMap<String, Object>() {{
88+
put("a", singletonMap("type", "long"));
89+
put("b", singletonMap("type", "keyword"));
90+
}}))
91+
);
92+
assertThat(
93+
TransformIndex.createMappingsFromStringMap(new HashMap<String, String>() {{
94+
put("a", "long");
95+
put("a.b", "keyword");
96+
}}),
97+
is(equalTo(new HashMap<String, Object>() {{
98+
put("a", new HashMap<String, Object>() {{
99+
put("type", "long");
100+
put("fields", singletonMap("b", singletonMap("type", "keyword")));
101+
}});
102+
}}))
103+
);
104+
assertThat(
105+
TransformIndex.createMappingsFromStringMap(new HashMap<String, String>() {{
106+
put("a", "long");
107+
put("a.b", "text");
108+
put("a.b.c", "keyword");
109+
}}),
110+
is(equalTo(new HashMap<String, Object>() {{
111+
put("a", new HashMap<String, Object>() {{
112+
put("type", "long");
113+
put("fields", new HashMap<String, Object>() {{
114+
put("b", new HashMap<String, Object>() {{
115+
put("type", "text");
116+
put("fields", new HashMap<String, Object>() {{
117+
put("c", singletonMap("type", "keyword"));
118+
}});
119+
}});
120+
}});
121+
}});
122+
}}))
123+
);
124+
assertThat(
125+
TransformIndex.createMappingsFromStringMap(new HashMap<String, String>() {{
126+
put("a", "object");
127+
put("a.b", "long");
128+
put("c", "nested");
129+
put("c.d", "boolean");
130+
put("f", "object");
131+
put("f.g", "object");
132+
put("f.g.h", "text");
133+
put("f.g.h.i", "text");
134+
}}),
135+
is(equalTo(new HashMap<String, Object>() {{
136+
put("a", new HashMap<String, Object>() {{
137+
put("type", "object");
138+
put("properties", new HashMap<String, Object>() {{
139+
put("b", new HashMap<String, Object>() {{
140+
put("type", "long");
141+
}});
142+
}});
143+
}});
144+
put("c", new HashMap<String, Object>() {{
145+
put("type", "nested");
146+
put("properties", new HashMap<String, Object>() {{
147+
put("d", new HashMap<String, Object>() {{
148+
put("type", "boolean");
149+
}});
150+
}});
151+
}});
152+
put("f", new HashMap<String, Object>() {{
153+
put("type", "object");
154+
put("properties", new HashMap<String, Object>() {{
155+
put("g", new HashMap<String, Object>() {{
156+
put("type", "object");
157+
put("properties", new HashMap<String, Object>() {{
158+
put("h", new HashMap<String, Object>() {{
159+
put("type", "text");
160+
put("fields", new HashMap<String, Object>() {{
161+
put("i", singletonMap("type", "text"));
162+
}});
163+
}});
164+
}});
165+
}});
166+
}});
167+
}});
168+
}}))
169+
);
170+
}
71171
}

0 commit comments

Comments
 (0)