Skip to content

Fix sorted query when date_nanos is used as the numeric_type #64183

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 27, 2020
Merged

Fix sorted query when date_nanos is used as the numeric_type #64183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
35b0fff
Fix sorted query when date_nanos is used as the numeric_type
jimczi Oct 26, 2020
a6fb8c9
Merge branch 'master' into bugs/date_nanos_numeric_type
jimczi Oct 26, 2020
b20f22b
Don't try to build date and date_nanos numeric_type on random fields
jimczi Oct 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 60 additions & 17 deletions server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1762,7 +1762,6 @@ public void testCastNumericType() throws Exception {
}
}

@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/63719")
public void testCastDate() throws Exception {
assertAcked(prepareCreate("index_date")
.setMapping("field", "type=date"));
Expand All @@ -1780,7 +1779,7 @@ public void testCastDate() throws Exception {
{
SearchResponse response = client().prepareSearch()
.setQuery(matchAllQuery())
.setSize(builders.size())
.setSize(2)
.addSort(SortBuilders.fieldSort("field").setNumericType("date"))
.get();
SearchHits hits = response.getHits();
Expand All @@ -1791,12 +1790,36 @@ public void testCastDate() throws Exception {
}
assertEquals(1712879236854L, hits.getAt(0).getSortValues()[0]);
assertEquals(1712879237000L, hits.getAt(1).getSortValues()[0]);

response = client().prepareSearch()
.setMaxConcurrentShardRequests(1)
.setQuery(matchAllQuery())
.setSize(1)
.addSort(SortBuilders.fieldSort("field").setNumericType("date"))
.get();
hits = response.getHits();

assertEquals(1, hits.getHits().length);
assertThat(hits.getAt(0).getSortValues()[0].getClass(), equalTo(Long.class));
assertEquals(1712879236854L, hits.getAt(0).getSortValues()[0]);

response = client().prepareSearch()
.setMaxConcurrentShardRequests(1)
.setQuery(matchAllQuery())
.setSize(1)
.addSort(SortBuilders.fieldSort("field").order(SortOrder.DESC).setNumericType("date"))
.get();
hits = response.getHits();

assertEquals(1, hits.getHits().length);
assertThat(hits.getAt(0).getSortValues()[0].getClass(), equalTo(Long.class));
assertEquals(1712879237000L, hits.getAt(0).getSortValues()[0]);
}

{
SearchResponse response = client().prepareSearch()
.setQuery(matchAllQuery())
.setSize(builders.size())
.setSize(2)
.addSort(SortBuilders.fieldSort("field").setNumericType("date_nanos"))
.get();
SearchHits hits = response.getHits();
Expand All @@ -1806,38 +1829,58 @@ public void testCastDate() throws Exception {
}
assertEquals(1712879236854775807L, hits.getAt(0).getSortValues()[0]);
assertEquals(1712879237000000000L, hits.getAt(1).getSortValues()[0]);

response = client().prepareSearch()
.setMaxConcurrentShardRequests(1)
.setQuery(matchAllQuery())
.setSize(1)
.addSort(SortBuilders.fieldSort("field").setNumericType("date_nanos"))
.get();
hits = response.getHits();
assertEquals(1, hits.getHits().length);
assertThat(hits.getAt(0).getSortValues()[0].getClass(), equalTo(Long.class));
assertEquals(1712879236854775807L, hits.getAt(0).getSortValues()[0]);

response = client().prepareSearch()
.setMaxConcurrentShardRequests(1)
.setQuery(matchAllQuery())
.setSize(1)
.addSort(SortBuilders.fieldSort("field").order(SortOrder.DESC).setNumericType("date_nanos"))
.get();
hits = response.getHits();
assertEquals(1, hits.getHits().length);
assertThat(hits.getAt(0).getSortValues()[0].getClass(), equalTo(Long.class));
assertEquals(1712879237000000000L, hits.getAt(0).getSortValues()[0]);
}

{
builders.clear();
builders.add(client().prepareIndex("index_date")
.setSource("field", "1905-04-11T23:47:17"));
indexRandom(true, true, builders);
SearchPhaseExecutionException exc = expectThrows(SearchPhaseExecutionException.class,
() -> client().prepareSearch()
SearchResponse response = client().prepareSearch()
.setQuery(matchAllQuery())
.setSize(builders.size())
.setAllowPartialSearchResults(false)
.setSize(1)
.addSort(SortBuilders.fieldSort("field").setNumericType("date_nanos"))
.get()
);
assertThat(exc.toString(), containsString("are before the epoch in 1970"));
.get();
assertNotNull(response.getShardFailures());
assertThat(response.getShardFailures().length, equalTo(1));
assertThat(response.getShardFailures()[0].toString(), containsString("are before the epoch in 1970"));
}

{
builders.clear();
builders.add(client().prepareIndex("index_date")
.setSource("field", "2346-04-11T23:47:17"));
indexRandom(true, true, builders);
SearchPhaseExecutionException exc = expectThrows(SearchPhaseExecutionException.class,
() -> client().prepareSearch()
SearchResponse response = client().prepareSearch()
.setQuery(QueryBuilders.rangeQuery("field").gt("1970-01-01"))
.setSize(builders.size())
.setAllowPartialSearchResults(false)
.setSize(10)
.addSort(SortBuilders.fieldSort("field").setNumericType("date_nanos"))
.get()
);
assertThat(exc.toString(), containsString("are after 2262"));
.get();
assertNotNull(response.getShardFailures());
assertThat(response.getShardFailures().length, equalTo(1));
assertThat(response.getShardFailures()[0].toString(), containsString("are after 2262"));
}
}

Expand Down
6 changes: 1 addition & 5 deletions server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,7 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
IndexNumericFieldData numericFieldData = (IndexNumericFieldData) fieldData;
NumericType resolvedType = resolveNumericType(numericType);
field = numericFieldData.sortField(resolvedType, missing, localSortMode(), nested, reverse);
isNanosecond = resolvedType == NumericType.DATE_NANOSECONDS;
} else {
field = fieldData.sortField(missing, localSortMode(), nested, reverse);
if (fieldData instanceof IndexNumericFieldData) {
Expand Down Expand Up @@ -387,11 +388,6 @@ public boolean isBottomSortShardDisjoint(QueryShardContext context, SearchSortVa
DocValueFormat docValueFormat = bottomSortValues.getSortValueFormats()[0];
final DateMathParser dateMathParser;
if (docValueFormat instanceof DocValueFormat.DateTime) {
if (fieldType instanceof DateFieldType && ((DateFieldType) fieldType).resolution() == NANOSECONDS) {
// we parse the formatted value with the resolution of the local field because
// the provided format can use a different one (date vs date_nanos).
docValueFormat = DocValueFormat.withNanosecondResolution(docValueFormat);
}
dateMathParser = ((DocValueFormat.DateTime) docValueFormat).getDateMathParser();
} else {
dateMathParser = null;
Expand Down
4 changes: 2 additions & 2 deletions server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ public FieldSortBuilder randomFieldSortBuilder() {
builder.setNestedSort(createRandomNestedSort(3));
}
if (randomBoolean()) {
builder.setNumericType(randomFrom(random(), "long", "double", "date", "date_nanos"));
builder.setNumericType(randomFrom(random(), "long", "double"));
}
return builder;
}
Expand Down Expand Up @@ -147,7 +147,7 @@ protected FieldSortBuilder mutate(FieldSortBuilder original) throws IOException
break;
case 5:
mutated.setNumericType(randomValueOtherThan(original.getNumericType(),
() -> randomFrom("long", "double", "date", "date_nanos")));
() -> randomFrom("long", "double")));
break;
default:
throw new IllegalStateException("Unsupported mutation.");
Expand Down