Skip to content

Commit 19cfc25

Browse files
jimczijimczi
authored
Adds the ability to specify a format on composite date_histogram source (#28310)
This commit adds the ability to specify a date format on the `date_histogram` composite source. If the format is defined, the key for the source is returned as a formatted date. Closes #27923
1 parent d31e964 commit 19cfc25

File tree

15 files changed

+401
-76
lines changed

15 files changed

+401
-76
lines changed

‎docs/reference/aggregations/bucket/composite-aggregation.asciidoc‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,41 @@ Note that fractional time values are not supported, but you can address this by
225225
time unit (e.g., `1.5h` could instead be specified as `90m`).
226226

227227
[float]
228-
===== Time Zone
228+
====== Format
229+
230+
Internally, a date is represented as a 64 bit number representing a timestamp in milliseconds-since-the-epoch.
231+
These timestamps are returned as the bucket keys. It is possible to return a formatted date string instead using
232+
the format specified with the format parameter:
233+
234+
[source,js]
235+
--------------------------------------------------
236+
GET /_search
237+
{
238+
"aggs" : {
239+
"my_buckets": {
240+
"composite" : {
241+
"sources" : [
242+
{
243+
"date": {
244+
"date_histogram" : {
245+
"field": "timestamp",
246+
"interval": "1d",
247+
"format": "yyyy-MM-dd" <1>
248+
}
249+
}
250+
}
251+
]
252+
}
253+
}
254+
}
255+
}
256+
--------------------------------------------------
257+
// CONSOLE
258+
259+
<1> Supports expressive date <<date-format-pattern,format pattern>>
260+
261+
[float]
262+
====== Time Zone
229263

230264
Date-times are stored in Elasticsearch in UTC. By default, all bucketing and
231265
rounding is also done in UTC. The `time_zone` parameter can be used to indicate

‎rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml‎

Lines changed: 79 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ setup:
77
mappings:
88
doc:
99
properties:
10+
date:
11+
type: date
1012
keyword:
1113
type: keyword
1214
long:
@@ -40,6 +42,20 @@ setup:
4042
id: 4
4143
body: { "keyword": "bar", "long": [1000, 0] }
4244

45+
- do:
46+
index:
47+
index: test
48+
type: doc
49+
id: 5
50+
body: { "date": "2017-10-20T03:08:45" }
51+
52+
- do:
53+
index:
54+
index: test
55+
type: doc
56+
id: 6
57+
body: { "date": "2017-10-21T07:00:00" }
58+
4359
- do:
4460
indices.refresh:
4561
index: [test]
@@ -66,7 +82,7 @@ setup:
6682
}
6783
]
6884

69-
- match: {hits.total: 4}
85+
- match: {hits.total: 6}
7086
- length: { aggregations.test.buckets: 2 }
7187
- match: { aggregations.test.buckets.0.key.kw: "bar" }
7288
- match: { aggregations.test.buckets.0.doc_count: 3 }
@@ -104,7 +120,7 @@ setup:
104120
}
105121
]
106122

107-
- match: {hits.total: 4}
123+
- match: {hits.total: 6}
108124
- length: { aggregations.test.buckets: 5 }
109125
- match: { aggregations.test.buckets.0.key.long: 0}
110126
- match: { aggregations.test.buckets.0.key.kw: "bar" }
@@ -154,7 +170,7 @@ setup:
154170
]
155171
after: { "long": 20, "kw": "foo" }
156172

157-
- match: {hits.total: 4}
173+
- match: {hits.total: 6}
158174
- length: { aggregations.test.buckets: 2 }
159175
- match: { aggregations.test.buckets.0.key.long: 100 }
160176
- match: { aggregations.test.buckets.0.key.kw: "bar" }
@@ -188,7 +204,7 @@ setup:
188204
]
189205
after: { "kw": "delta" }
190206

191-
- match: {hits.total: 4}
207+
- match: {hits.total: 6}
192208
- length: { aggregations.test.buckets: 1 }
193209
- match: { aggregations.test.buckets.0.key.kw: "foo" }
194210
- match: { aggregations.test.buckets.0.doc_count: 2 }
@@ -220,3 +236,62 @@ setup:
220236
}
221237
}
222238
]
239+
240+
---
241+
"Composite aggregation with format":
242+
- skip:
243+
version: " - 6.99.99"
244+
reason: this uses a new option (format) added in 7.0.0
245+
246+
- do:
247+
search:
248+
index: test
249+
body:
250+
aggregations:
251+
test:
252+
composite:
253+
sources: [
254+
{
255+
"date": {
256+
"date_histogram": {
257+
"field": "date",
258+
"interval": "1d",
259+
"format": "yyyy-MM-dd"
260+
}
261+
}
262+
}
263+
]
264+
265+
- match: {hits.total: 6}
266+
- length: { aggregations.test.buckets: 2 }
267+
- match: { aggregations.test.buckets.0.key.date: "2017-10-20" }
268+
- match: { aggregations.test.buckets.0.doc_count: 1 }
269+
- match: { aggregations.test.buckets.1.key.date: "2017-10-21" }
270+
- match: { aggregations.test.buckets.1.doc_count: 1 }
271+
272+
- do:
273+
search:
274+
index: test
275+
body:
276+
aggregations:
277+
test:
278+
composite:
279+
after: {
280+
date: "2017-10-20"
281+
}
282+
sources: [
283+
{
284+
"date": {
285+
"date_histogram": {
286+
"field": "date",
287+
"interval": "1d",
288+
"format": "yyyy-MM-dd"
289+
}
290+
}
291+
}
292+
]
293+
294+
- match: {hits.total: 6}
295+
- length: { aggregations.test.buckets: 1 }
296+
- match: { aggregations.test.buckets.0.key.date: "2017-10-21" }
297+
- match: { aggregations.test.buckets.0.doc_count: 1 }

‎server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -147,17 +147,15 @@ protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<
147147
Sort sort = indexSortConfig.buildIndexSort(shardContext::fieldMapper, shardContext::getForField);
148148
System.arraycopy(sort.getSort(), 0, sortFields, 0, sortFields.length);
149149
}
150-
List<String> sourceNames = new ArrayList<>();
151150
for (int i = 0; i < configs.length; i++) {
152151
configs[i] = sources.get(i).build(context, i, configs.length, sortFields[i]);
153-
sourceNames.add(sources.get(i).name());
154152
if (configs[i].valuesSource().needsScores()) {
155153
throw new IllegalArgumentException("[sources] cannot access _score");
156154
}
157155
}
158156
final CompositeKey afterKey;
159157
if (after != null) {
160-
if (after.size() != sources.size()) {
158+
if (after.size() != configs.length) {
161159
throw new IllegalArgumentException("[after] has " + after.size() +
162160
" value(s) but [sources] has " + sources.size());
163161
}
@@ -179,7 +177,7 @@ protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<
179177
} else {
180178
afterKey = null;
181179
}
182-
return new CompositeAggregationFactory(name, context, parent, subfactoriesBuilder, metaData, size, configs, sourceNames, afterKey);
180+
return new CompositeAggregationFactory(name, context, parent, subfactoriesBuilder, metaData, size, configs, afterKey);
183181
}
184182

185183

‎server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationFactory.java‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,24 +32,21 @@
3232
class CompositeAggregationFactory extends AggregatorFactory<CompositeAggregationFactory> {
3333
private final int size;
3434
private final CompositeValuesSourceConfig[] sources;
35-
private final List<String> sourceNames;
3635
private final CompositeKey afterKey;
3736

3837
CompositeAggregationFactory(String name, SearchContext context, AggregatorFactory<?> parent,
3938
AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData,
40-
int size, CompositeValuesSourceConfig[] sources,
41-
List<String> sourceNames, CompositeKey afterKey) throws IOException {
39+
int size, CompositeValuesSourceConfig[] sources, CompositeKey afterKey) throws IOException {
4240
super(name, context, parent, subFactoriesBuilder, metaData);
4341
this.size = size;
4442
this.sources = sources;
45-
this.sourceNames = sourceNames;
4643
this.afterKey = afterKey;
4744
}
4845

4946
@Override
5047
protected Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBucket,
5148
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
5249
return new CompositeAggregator(name, factories, context, parent, pipelineAggregators, metaData,
53-
size, sources, sourceNames, afterKey);
50+
size, sources, afterKey);
5451
}
5552
}

‎server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.lucene.search.Scorer;
2828
import org.apache.lucene.search.Weight;
2929
import org.apache.lucene.util.RoaringDocIdSet;
30+
import org.elasticsearch.search.DocValueFormat;
3031
import org.elasticsearch.search.aggregations.Aggregator;
3132
import org.elasticsearch.search.aggregations.AggregatorFactories;
3233
import org.elasticsearch.search.aggregations.InternalAggregation;
@@ -43,11 +44,13 @@
4344
import java.util.List;
4445
import java.util.Map;
4546
import java.util.TreeMap;
47+
import java.util.stream.Collectors;
4648

4749
final class CompositeAggregator extends BucketsAggregator {
4850
private final int size;
4951
private final CompositeValuesSourceConfig[] sources;
5052
private final List<String> sourceNames;
53+
private final List<DocValueFormat> formats;
5154
private final boolean canEarlyTerminate;
5255

5356
private final TreeMap<Integer, Integer> keys;
@@ -59,12 +62,12 @@ final class CompositeAggregator extends BucketsAggregator {
5962

6063
CompositeAggregator(String name, AggregatorFactories factories, SearchContext context, Aggregator parent,
6164
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData,
62-
int size, CompositeValuesSourceConfig[] sources, List<String> sourceNames,
63-
CompositeKey rawAfterKey) throws IOException {
65+
int size, CompositeValuesSourceConfig[] sources, CompositeKey rawAfterKey) throws IOException {
6466
super(name, factories, context, parent, pipelineAggregators, metaData);
6567
this.size = size;
6668
this.sources = sources;
67-
this.sourceNames = sourceNames;
69+
this.sourceNames = Arrays.stream(sources).map(CompositeValuesSourceConfig::name).collect(Collectors.toList());
70+
this.formats = Arrays.stream(sources).map(CompositeValuesSourceConfig::format).collect(Collectors.toList());
6871
// we use slot 0 to fill the current document (size+1).
6972
this.array = new CompositeValuesComparator(context.searcher().getIndexReader(), sources, size+1);
7073
if (rawAfterKey != null) {
@@ -131,15 +134,17 @@ public InternalAggregation buildAggregation(long zeroBucket) throws IOException
131134
CompositeKey key = array.toCompositeKey(slot);
132135
InternalAggregations aggs = bucketAggregations(slot);
133136
int docCount = bucketDocCount(slot);
134-
buckets[pos++] = new InternalComposite.InternalBucket(sourceNames, key, reverseMuls, docCount, aggs);
137+
buckets[pos++] = new InternalComposite.InternalBucket(sourceNames, formats, key, reverseMuls, docCount, aggs);
135138
}
136-
return new InternalComposite(name, size, sourceNames, Arrays.asList(buckets), reverseMuls, pipelineAggregators(), metaData());
139+
return new InternalComposite(name, size, sourceNames, formats, Arrays.asList(buckets), reverseMuls,
140+
pipelineAggregators(), metaData());
137141
}
138142

139143
@Override
140144
public InternalAggregation buildEmptyAggregation() {
141145
final int[] reverseMuls = getReverseMuls();
142-
return new InternalComposite(name, size, sourceNames, Collections.emptyList(), reverseMuls, pipelineAggregators(), metaData());
146+
return new InternalComposite(name, size, sourceNames, formats, Collections.emptyList(), reverseMuls,
147+
pipelineAggregators(), metaData());
143148
}
144149

145150
@Override

‎server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesComparator.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ final class CompositeValuesComparator {
5656
if (vs.isFloatingPoint()) {
5757
arrays[i] = CompositeValuesSource.wrapDouble(vs, size, reverseMul);
5858
} else {
59-
arrays[i] = CompositeValuesSource.wrapLong(vs, size, reverseMul);
59+
arrays[i] = CompositeValuesSource.wrapLong(vs, sources[i].format(), size, reverseMul);
6060
}
6161
}
6262
}

‎server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSource.java‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,10 @@
2323
import org.apache.lucene.index.SortedSetDocValues;
2424
import org.apache.lucene.search.LeafCollector;
2525
import org.apache.lucene.util.BytesRef;
26+
import org.elasticsearch.common.joda.FormatDateTimeFormatter;
2627
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
2728
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
29+
import org.elasticsearch.search.DocValueFormat;
2830
import org.elasticsearch.search.aggregations.support.ValuesSource;
2931
import org.elasticsearch.search.sort.SortOrder;
3032

@@ -96,8 +98,9 @@ interface Collector {
9698
/**
9799
* Creates a {@link CompositeValuesSource} that generates long values.
98100
*/
99-
static CompositeValuesSource<ValuesSource.Numeric, Long> wrapLong(ValuesSource.Numeric vs, int size, int reverseMul) {
100-
return new LongValuesSource(vs, size, reverseMul);
101+
static CompositeValuesSource<ValuesSource.Numeric, Long> wrapLong(ValuesSource.Numeric vs, DocValueFormat format,
102+
int size, int reverseMul) {
103+
return new LongValuesSource(vs, format, size, reverseMul);
101104
}
102105

103106
/**
@@ -273,9 +276,12 @@ Collector getLeafCollector(LeafReaderContext context, Collector next) throws IOE
273276
*/
274277
private static class LongValuesSource extends CompositeValuesSource<ValuesSource.Numeric, Long> {
275278
private final long[] values;
279+
// handles "format" for date histogram source
280+
private final DocValueFormat format;
276281

277-
LongValuesSource(ValuesSource.Numeric vs, int size, int reverseMul) {
282+
LongValuesSource(ValuesSource.Numeric vs, DocValueFormat format, int size, int reverseMul) {
278283
super(vs, size, reverseMul);
284+
this.format = format;
279285
this.values = new long[size];
280286
}
281287

@@ -304,7 +310,11 @@ void setTop(Comparable<?> value) {
304310
if (value instanceof Number) {
305311
topValue = ((Number) value).longValue();
306312
} else {
307-
topValue = Long.parseLong(value.toString());
313+
// for date histogram source with "format", the after value is formatted
314+
// as a string so we need to retrieve the original value in milliseconds.
315+
topValue = format.parseLong(value.toString(), false, () -> {
316+
throw new IllegalArgumentException("now() is not supported in [after] key");
317+
});
308318
}
309319
}
310320

0 commit comments

Comments
 (0)