Skip to content

Commit 2438b89

Browse files
pgomulkapgomulka
authored
Support joda style date patterns in 7.x (#52555)
If an index was created in version 6 and contain a date field with a joda-style pattern it should still be allowed to search and insert document into it. Those created in 6 but date pattern starts with 8, should be considered as java style.
1 parent 4301c35 commit 2438b89

File tree

10 files changed

+528
-7
lines changed

10 files changed

+528
-7
lines changed

‎qa/rolling-upgrade/build.gradle‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ for (Version bwcVersion : bwcVersions.wireCompatible) {
5959
doFirst {
6060
project.delete("${buildDir}/cluster/shared/repo/${baseName}")
6161
}
62+
systemProperty 'tests.upgrade_from_version', bwcVersion.toString()
6263
systemProperty 'tests.rest.suite', 'old_cluster'
6364
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
6465
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
@@ -71,7 +72,7 @@ for (Version bwcVersion : bwcVersions.wireCompatible) {
7172
testClusters."${baseName}".nextNodeToNextVersion()
7273
}
7374
systemProperty 'tests.rest.suite', 'mixed_cluster'
74-
systemProperty 'tests.upgrade_from_version', project.version.replace("-SNAPSHOT", "")
75+
systemProperty 'tests.upgrade_from_version', bwcVersion.toString()
7576
systemProperty 'tests.first_round', 'true'
7677
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
7778
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
@@ -84,7 +85,7 @@ for (Version bwcVersion : bwcVersions.wireCompatible) {
8485
testClusters."${baseName}".nextNodeToNextVersion()
8586
}
8687
systemProperty 'tests.rest.suite', 'mixed_cluster'
87-
systemProperty 'tests.upgrade_from_version', project.version.replace("-SNAPSHOT", "")
88+
systemProperty 'tests.upgrade_from_version', bwcVersion.toString()
8889
systemProperty 'tests.first_round', 'false'
8990
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
9091
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
@@ -97,6 +98,7 @@ for (Version bwcVersion : bwcVersions.wireCompatible) {
9798
}
9899
useCluster testClusters."${baseName}"
99100
systemProperty 'tests.rest.suite', 'upgraded_cluster'
101+
systemProperty 'tests.upgrade_from_version', bwcVersion.toString()
100102

101103
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
102104
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")

‎qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/JodaCompatibilityIT.java‎

Lines changed: 269 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,269 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.upgrades;
20+
21+
import org.apache.http.HttpStatus;
22+
import org.apache.http.util.EntityUtils;
23+
import org.elasticsearch.Version;
24+
import org.elasticsearch.client.Node;
25+
import org.elasticsearch.client.Request;
26+
import org.elasticsearch.client.RequestOptions;
27+
import org.elasticsearch.client.Response;
28+
import org.elasticsearch.client.WarningsHandler;
29+
import org.elasticsearch.common.Booleans;
30+
import org.elasticsearch.common.io.stream.StreamInput;
31+
import org.elasticsearch.search.DocValueFormat;
32+
import org.junit.BeforeClass;
33+
34+
import java.io.IOException;
35+
import java.nio.charset.StandardCharsets;
36+
import java.util.Collections;
37+
import java.util.List;
38+
import java.util.function.Consumer;
39+
40+
import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM;
41+
42+
/**
43+
* This is test is meant to verify that when upgrading from 6.x version to 7.7 or newer it is able to parse date fields with joda pattern.
44+
*
45+
* The test is indexing documents and searches with use of joda or java pattern.
46+
* In order to make sure that serialization logic is used a search call is executed 3 times (using all nodes).
47+
* It cannot be guaranteed that serialization logic will always be used as it might happen that
48+
* all shards are allocated on the same node and client is connecting to it.
49+
* Because of this warnings assertions have to be ignored.
50+
*
51+
* A special flag used when serializing {@link DocValueFormat.DateTime#writeTo DocValueFormat.DateTime::writeTo}
52+
* is used to indicate that an index was created in 6.x and has a joda pattern. The same flag is read when
53+
* {@link DocValueFormat.DateTime#DateTime(StreamInput)} deserializing.
54+
* When upgrading from 7.0-7.6 to 7.7 there is no way to tell if a pattern was created in 6.x as this flag cannot be added.
55+
* Hence a skip assume section in init()
56+
*
57+
* @see org.elasticsearch.search.DocValueFormat.DateTime
58+
*/
59+
public class JodaCompatibilityIT extends AbstractRollingTestCase {
60+
61+
@BeforeClass
62+
public static void init(){
63+
assumeTrue("upgrading from 7.0-7.6 will fail parsing joda formats",
64+
UPGRADE_FROM_VERSION.before(Version.V_7_0_0));
65+
}
66+
67+
public void testJodaBackedDocValueAndDateFields() throws Exception {
68+
switch (CLUSTER_TYPE) {
69+
case OLD:
70+
Request createTestIndex = indexWithDateField("joda_time", "YYYY-MM-dd'T'HH:mm:ssZZ");
71+
createTestIndex.setOptions(ignoreWarnings());
72+
73+
Response resp = client().performRequest(createTestIndex);
74+
assertEquals(HttpStatus.SC_OK, resp.getStatusLine().getStatusCode());
75+
76+
postNewDoc("joda_time", 1);
77+
78+
break;
79+
case MIXED:
80+
int minute = Booleans.parseBoolean(System.getProperty("tests.first_round")) ? 2 : 3;
81+
postNewDoc("joda_time", minute);
82+
83+
Request search = dateRangeSearch("joda_time");
84+
search.setOptions(ignoreWarnings());
85+
86+
performOnAllNodes(search, r -> assertEquals(HttpStatus.SC_OK, r.getStatusLine().getStatusCode()));
87+
break;
88+
case UPGRADED:
89+
postNewDoc("joda_time", 4);
90+
91+
search = searchWithAgg("joda_time");
92+
search.setOptions(ignoreWarnings());
93+
//making sure all nodes were used for search
94+
performOnAllNodes(search, r -> assertResponseHasAllDocuments(r));
95+
break;
96+
}
97+
}
98+
99+
public void testJavaBackedDocValueAndDateFields() throws Exception {
100+
switch (CLUSTER_TYPE) {
101+
case OLD:
102+
Request createTestIndex = indexWithDateField("java_time", "8yyyy-MM-dd'T'HH:mm:ssXXX");
103+
Response resp = client().performRequest(createTestIndex);
104+
assertEquals(HttpStatus.SC_OK, resp.getStatusLine().getStatusCode());
105+
106+
postNewDoc("java_time", 1);
107+
108+
break;
109+
case MIXED:
110+
int minute = Booleans.parseBoolean(System.getProperty("tests.first_round")) ? 2 : 3;
111+
postNewDoc("java_time", minute);
112+
113+
Request search = dateRangeSearch("java_time");
114+
Response searchResp = client().performRequest(search);
115+
assertEquals(HttpStatus.SC_OK, searchResp.getStatusLine().getStatusCode());
116+
break;
117+
case UPGRADED:
118+
postNewDoc("java_time", 4);
119+
120+
search = searchWithAgg("java_time");
121+
//making sure all nodes were used for search
122+
performOnAllNodes(search, r -> assertResponseHasAllDocuments(r));
123+
124+
break;
125+
}
126+
}
127+
128+
private RequestOptions ignoreWarnings() {
129+
RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder();
130+
options.setWarningsHandler(WarningsHandler.PERMISSIVE);
131+
return options.build();
132+
}
133+
134+
private void performOnAllNodes(Request search, Consumer<Response> consumer) throws IOException {
135+
List<Node> nodes = client().getNodes();
136+
for (Node node : nodes) {
137+
client().setNodes(Collections.singletonList(node));
138+
Response response = client().performRequest(search);
139+
consumer.accept(response);
140+
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
141+
}
142+
client().setNodes(nodes);
143+
}
144+
145+
private void assertResponseHasAllDocuments(Response searchResp) {
146+
assertEquals(HttpStatus.SC_OK, searchResp.getStatusLine().getStatusCode());
147+
try {
148+
assertEquals(removeWhiteSpace("{" +
149+
" \"_shards\": {" +
150+
" \"total\": 3," +
151+
" \"successful\": 3" +
152+
" },"+
153+
" \"hits\": {" +
154+
" \"total\": 4," +
155+
" \"hits\": [" +
156+
" {" +
157+
" \"_source\": {" +
158+
" \"datetime\": \"2020-01-01T00:00:01+01:00\"" +
159+
" }" +
160+
" }," +
161+
" {" +
162+
" \"_source\": {" +
163+
" \"datetime\": \"2020-01-01T00:00:02+01:00\"" +
164+
" }" +
165+
" }," +
166+
" {" +
167+
" \"_source\": {" +
168+
" \"datetime\": \"2020-01-01T00:00:03+01:00\"" +
169+
" }" +
170+
" }," +
171+
" {" +
172+
" \"_source\": {" +
173+
" \"datetime\": \"2020-01-01T00:00:04+01:00\"" +
174+
" }" +
175+
" }" +
176+
" ]" +
177+
" }" +
178+
"}"),
179+
EntityUtils.toString(searchResp.getEntity(), StandardCharsets.UTF_8));
180+
} catch (IOException e) {
181+
throw new AssertionError("Exception during response parising", e);
182+
}
183+
}
184+
185+
private String removeWhiteSpace(String input) {
186+
return input.replaceAll("[\\n\\r\\t\\ ]", "");
187+
}
188+
189+
private Request dateRangeSearch(String endpoint) {
190+
Request search = new Request("GET", endpoint+"/_search");
191+
search.addParameter(TOTAL_HITS_AS_INT_PARAM, "true");
192+
search.addParameter("filter_path", "hits.total,hits.hits._source.datetime,_shards.total,_shards.successful");
193+
search.setJsonEntity("" +
194+
"{\n" +
195+
" \"track_total_hits\": true,\n" +
196+
" \"sort\": \"datetime\",\n" +
197+
" \"query\": {\n" +
198+
" \"range\": {\n" +
199+
" \"datetime\": {\n" +
200+
" \"gte\": \"2020-01-01T00:00:00+01:00\",\n" +
201+
" \"lte\": \"2020-01-02T00:00:00+01:00\"\n" +
202+
" }\n" +
203+
" }\n" +
204+
" }\n" +
205+
"}\n"
206+
);
207+
return search;
208+
}
209+
210+
private Request searchWithAgg(String endpoint) throws IOException {
211+
Request search = new Request("GET", endpoint+"/_search");
212+
search.addParameter(TOTAL_HITS_AS_INT_PARAM, "true");
213+
search.addParameter("filter_path", "hits.total,hits.hits._source.datetime,_shards.total,_shards.successful");
214+
215+
search.setJsonEntity("{\n" +
216+
" \"track_total_hits\": true,\n" +
217+
" \"sort\": \"datetime\",\n" +
218+
" \"query\": {\n" +
219+
" \"range\": {\n" +
220+
" \"datetime\": {\n" +
221+
" \"gte\": \"2020-01-01T00:00:00+01:00\",\n" +
222+
" \"lte\": \"2020-01-02T00:00:00+01:00\"\n" +
223+
" }\n" +
224+
" }\n" +
225+
" },\n" +
226+
" \"aggs\" : {\n" +
227+
" \"docs_per_year\" : {\n" +
228+
" \"date_histogram\" : {\n" +
229+
" \"field\" : \"date\",\n" +
230+
" \"calendar_interval\" : \"year\"\n" +
231+
" }\n" +
232+
" }\n" +
233+
" }\n" +
234+
"}\n"
235+
);
236+
return search;
237+
}
238+
private Request indexWithDateField(String indexName, String format) {
239+
Request createTestIndex = new Request("PUT", indexName);
240+
createTestIndex.addParameter("include_type_name", "false");
241+
createTestIndex.setJsonEntity("{\n" +
242+
" \"settings\": {\n" +
243+
" \"index.number_of_shards\": 3\n" +
244+
" },\n" +
245+
" \"mappings\": {\n" +
246+
" \"properties\": {\n" +
247+
" \"datetime\": {\n" +
248+
" \"type\": \"date\",\n" +
249+
" \"format\": \"" + format + "\"\n" +
250+
" }\n" +
251+
" }\n" +
252+
" }\n" +
253+
"}"
254+
);
255+
return createTestIndex;
256+
}
257+
258+
private void postNewDoc(String endpoint, int minute) throws IOException {
259+
Request putDoc = new Request("POST", endpoint+"/_doc");
260+
putDoc.addParameter("refresh", "true");
261+
putDoc.addParameter("wait_for_active_shards", "all");
262+
putDoc.setJsonEntity("{\n" +
263+
" \"datetime\": \"2020-01-01T00:00:0" + minute + "+01:00\"\n" +
264+
"}"
265+
);
266+
Response resp = client().performRequest(putDoc);
267+
assertEquals(HttpStatus.SC_CREATED, resp.getStatusLine().getStatusCode());
268+
}
269+
}

‎qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/20_date_range.yml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
---
2+
"Insert more docs to joda index":
3+
- do:
4+
bulk:
5+
refresh: true
6+
body:
7+
- '{"index": {"_index": "joda_for_range"}}'
8+
- '{"time_frame": {"gte": "2019-01-01T00:00+01:00", "lte" : "2019-03-01T00:00+01:00"}}'
9+
10+
- do:
11+
search:
12+
rest_total_hits_as_int: true
13+
index: joda_for_range
14+
body:
15+
query:
16+
range:
17+
time_frame:
18+
gte: "2019-02-01T00:00+01:00"
19+
lte: "2019-02-01T00:00+01:00"
20+
21+
---
22+
"Insert more docs to java index":
23+
- do:
24+
bulk:
25+
refresh: true
26+
body:
27+
- '{"index": {"_index": "java_for_range"}}'
28+
- '{"time_frame": {"gte": "2019-01-01T00:00+01:00", "lte" : "2019-03-01T00:00+01:00"}}'
29+
30+
- do:
31+
search:
32+
rest_total_hits_as_int: true
33+
index: java_for_range
34+
body:
35+
query:
36+
range:
37+
time_frame:
38+
gte: "2019-02-01T00:00+01:00"
39+
lte: "2019-02-01T00:00+01:00"

0 commit comments

Comments
 (0)