Skip to content

Commit 2cde43b

Browse files
nik9000nik9000
authored
Allows nanosecond resolution in search_after (backport of #60328) (#60426)
Allows nanosecond resolution in search_after (#60328) This fixes `search_after` to properly parse string formatted dates that have nanosecond resolution. Closes #52424
1 parent b0d601f commit 2cde43b

File tree

6 files changed

+153
-8
lines changed

6 files changed

+153
-8
lines changed

‎rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml‎

Lines changed: 135 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
setup:
1+
"search with search_after parameter":
22
- do:
33
indices.create:
44
index: test
@@ -24,9 +24,6 @@ setup:
2424
indices.refresh:
2525
index: test
2626

27-
---
28-
"search with search_after parameter":
29-
3027
- do:
3128
search:
3229
rest_total_hits_as_int: true
@@ -97,3 +94,137 @@ setup:
9794

9895
- match: {hits.total: 3}
9996
- length: {hits.hits: 0 }
97+
98+
---
99+
"date":
100+
- skip:
101+
version: " - 7.9.99"
102+
reason: fixed in 7.10.0
103+
104+
- do:
105+
indices.create:
106+
index: test
107+
body:
108+
mappings:
109+
properties:
110+
timestamp:
111+
type: date
112+
format: yyyy-MM-dd HH:mm:ss.SSS
113+
- do:
114+
bulk:
115+
refresh: true
116+
index: test
117+
body: |
118+
{"index":{}}
119+
{"timestamp":"2019-10-21 00:30:04.828"}
120+
{"index":{}}
121+
{"timestamp":"2019-10-21 08:30:04.828"}
122+
123+
- do:
124+
search:
125+
index: test
126+
rest_total_hits_as_int: true
127+
body:
128+
size: 1
129+
sort: [{ timestamp: desc }]
130+
- match: {hits.total: 2 }
131+
- length: {hits.hits: 1 }
132+
- match: {hits.hits.0._index: test }
133+
- match: {hits.hits.0._source.timestamp: "2019-10-21 08:30:04.828" }
134+
- match: {hits.hits.0.sort: [1571646604828] }
135+
136+
# search_after with the sort
137+
- do:
138+
search:
139+
index: test
140+
rest_total_hits_as_int: true
141+
body:
142+
size: 1
143+
sort: [{ timestamp: desc }]
144+
search_after: [1571646604828]
145+
- match: {hits.total: 2 }
146+
- length: {hits.hits: 1 }
147+
- match: {hits.hits.0._index: test }
148+
- match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828" }
149+
- match: {hits.hits.0.sort: [1571617804828] }
150+
151+
# search_after with the formatted date
152+
- do:
153+
search:
154+
index: test
155+
rest_total_hits_as_int: true
156+
body:
157+
size: 1
158+
sort: [{ timestamp: desc }]
159+
search_after: ["2019-10-21 08:30:04.828"]
160+
- match: {hits.total: 2 }
161+
- length: {hits.hits: 1 }
162+
- match: {hits.hits.0._index: test }
163+
- match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828" }
164+
- match: {hits.hits.0.sort: [1571617804828] }
165+
166+
---
167+
"date_nanos":
168+
- skip:
169+
version: " - 7.9.99"
170+
reason: fixed in 7.10.0
171+
172+
- do:
173+
indices.create:
174+
index: test
175+
body:
176+
mappings:
177+
properties:
178+
timestamp:
179+
type: date_nanos
180+
format: yyyy-MM-dd HH:mm:ss.SSSSSS
181+
- do:
182+
bulk:
183+
refresh: true
184+
index: test
185+
body: |
186+
{"index":{}}
187+
{"timestamp":"2019-10-21 00:30:04.828740"}
188+
{"index":{}}
189+
{"timestamp":"2019-10-21 08:30:04.828733"}
190+
191+
- do:
192+
search:
193+
index: test
194+
body:
195+
size: 1
196+
sort: [{ timestamp: desc }]
197+
- match: {hits.total.value: 2 }
198+
- length: {hits.hits: 1 }
199+
- match: {hits.hits.0._index: test }
200+
- match: {hits.hits.0._source.timestamp: "2019-10-21 08:30:04.828733" }
201+
- match: {hits.hits.0.sort: [1571646604828733000] }
202+
203+
# search_after with the sort
204+
- do:
205+
search:
206+
index: test
207+
body:
208+
size: 1
209+
sort: [{ timestamp: desc }]
210+
search_after: [1571646604828733000]
211+
- match: {hits.total.value: 2 }
212+
- length: {hits.hits: 1 }
213+
- match: {hits.hits.0._index: test }
214+
- match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828740" }
215+
- match: {hits.hits.0.sort: [1571617804828740000] }
216+
217+
# search_after with the formatted date
218+
- do:
219+
search:
220+
index: test
221+
body:
222+
size: 1
223+
sort: [{ timestamp: desc }]
224+
search_after: ["2019-10-21 08:30:04.828733"]
225+
- match: {hits.total.value: 2 }
226+
- length: {hits.hits: 1 }
227+
- match: {hits.hits.0._index: test }
228+
- match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828740" }
229+
- match: {hits.hits.0.sort: [1571617804828740000] }
230+

‎server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ protected boolean sortRequiresCustomComparator() {
100100
@Override
101101
protected XFieldComparatorSource dateComparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
102102
if (numericType == NumericType.DATE_NANOSECONDS) {
103-
// converts date values to nanosecond resolution
103+
// converts date_nanos values to millisecond resolution
104104
return new LongValuesComparatorSource(this, missingValue,
105105
sortMode, nested, dvs -> convertNumeric(dvs, DateUtils::toMilliSeconds));
106106
}
@@ -110,7 +110,7 @@ protected XFieldComparatorSource dateComparatorSource(Object missingValue, Multi
110110
@Override
111111
protected XFieldComparatorSource dateNanosComparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
112112
if (numericType == NumericType.DATE) {
113-
// converts date_nanos values to millisecond resolution
113+
// converts date values to nanosecond resolution
114114
return new LongValuesComparatorSource(this, missingValue,
115115
sortMode, nested, dvs -> convertNumeric(dvs, DateUtils::toNanoSeconds));
116116
}

‎server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -451,6 +451,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
451451
}
452452
// the resolution here is always set to milliseconds, as aggregations use this formatter mainly and those are always in
453453
// milliseconds. The only special case here is docvalue fields, which are handled somewhere else
454+
// TODO maybe aggs should force millis because lots so of other places want nanos?
454455
return new DocValueFormat.DateTime(dateTimeFormatter, timeZone, Resolution.MILLISECONDS);
455456
}
456457
}

‎server/src/main/java/org/elasticsearch/search/DocValueFormat.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,11 @@ public long parseLong(String value, boolean roundUp, LongSupplier now) {
274274
public double parseDouble(String value, boolean roundUp, LongSupplier now) {
275275
return parseLong(value, roundUp, now);
276276
}
277+
278+
@Override
279+
public String toString() {
280+
return "DocValueFormat.DateTime(" + formatter + ", " + timeZone + ", " + resolution + ")";
281+
}
277282
}
278283

279284
DocValueFormat GEOHASH = new DocValueFormat() {

‎server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,7 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
404404
throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields");
405405
}
406406
final SortField field;
407+
boolean isNanosecond = false;
407408
if (numericType != null) {
408409
if (fieldData instanceof IndexNumericFieldData == false) {
409410
throw new QueryShardException(context,
@@ -414,8 +415,15 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
414415
field = numericFieldData.sortField(resolvedType, missing, localSortMode(), nested, reverse);
415416
} else {
416417
field = fieldData.sortField(missing, localSortMode(), nested, reverse);
418+
if (fieldData instanceof IndexNumericFieldData) {
419+
isNanosecond = ((IndexNumericFieldData) fieldData).getNumericType() == NumericType.DATE_NANOSECONDS;
420+
}
421+
}
422+
DocValueFormat format = fieldType.docValueFormat(null, null);
423+
if (isNanosecond) {
424+
format = DocValueFormat.withNanosecondResolution(format);
417425
}
418-
return new SortFieldAndFormat(field, fieldType.docValueFormat(null, null));
426+
return new SortFieldAndFormat(field, format);
419427
}
420428

421429
public boolean canRewriteToMatchNone() {

‎server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
8080
/**
8181
* {@link #provideMappedFieldType(String)} will return a
8282
*/
83-
private static String MAPPED_STRING_FIELDNAME = "_stringField";
83+
private static final String MAPPED_STRING_FIELDNAME = "_stringField";
8484

8585
@Override
8686
protected FieldSortBuilder createTestItem() {

0 commit comments

Comments
 (0)