From 4abb8e7b66299ebb32efb195e34260741768a26a Mon Sep 17 00:00:00 2001 From: David Roberts Date: Fri, 5 Oct 2018 16:58:32 +0100 Subject: [PATCH 1/2] [ML] Add an ingest pipeline definition to structure finder The ingest pipeline that is produced is very simple. It contains a grok processor if the format is semi-structured text, a date processor if the format contains a timestamp, and a remove processor if required to remove the interim timestamp field parsed out of semi-structured text. Eventually the UI should offer the option to customize the pipeline with additional processors to perform other data preparation steps before ingesting data to an index. --- .../ml/apis/find-file-structure.asciidoc | 68 ++++++++++++++++++ .../ml/filestructurefinder/FileStructure.java | 32 ++++++++- .../FileStructureTests.java | 9 +++ .../DelimitedFileStructureFinder.java | 6 +- .../FileStructureUtils.java | 53 ++++++++++++++ .../JsonFileStructureFinder.java | 6 +- .../TextLogFileStructureFinder.java | 6 +- .../XmlFileStructureFinder.java | 6 +- .../FileStructureUtilsTests.java | 69 +++++++++++++++++++ 9 files changed, 248 insertions(+), 7 deletions(-) diff --git a/docs/reference/ml/apis/find-file-structure.asciidoc b/docs/reference/ml/apis/find-file-structure.asciidoc index e72555d272356..5bd32750685a9 100644 --- a/docs/reference/ml/apis/find-file-structure.asciidoc +++ b/docs/reference/ml/apis/find-file-structure.asciidoc @@ -613,6 +613,20 @@ If the request does not encounter errors, you receive the following result: "type" : "double" } }, + "ingest_pipeline" : { + "description" : "Ingest pipeline created by file structure finder", + "processors" : [ + { + "date" : { + "field" : "tpep_pickup_datetime", + "timezone" : "{{ beat.timezone }}", + "formats" : [ + "YYYY-MM-dd HH:mm:ss" + ] + } + } + ] + }, "field_stats" : { "DOLocationID" : { "count" : 19998, @@ -1366,6 +1380,33 @@ this: "type" : "text" } }, + "ingest_pipeline" : { + "description" : "Ingest pipeline created by file structure finder", + "processors" : [ + { + "grok" : { + "field" : "message", + "patterns" : [ + "\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{LOGLEVEL:loglevel}.*" + ] + } + }, + { + "date" : { + "field" : "timestamp", + "timezone" : "{{ beat.timezone }}", + "formats" : [ + "ISO8601" + ] + } + }, + { + "remove" : { + "field" : "timestamp" + } + } + ] + }, "field_stats" : { "loglevel" : { "count" : 53, @@ -1499,6 +1540,33 @@ this: "type" : "keyword" } }, + "ingest_pipeline" : { + "description" : "Ingest pipeline created by file structure finder", + "processors" : [ + { + "grok" : { + "field" : "message", + "patterns" : [ + "\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{LOGLEVEL:loglevel} *\\]\\[%{JAVACLASS:class} *\\] \\[%{HOSTNAME:node}\\] %{JAVALOGMESSAGE:message}" + ] + } + }, + { + "date" : { + "field" : "timestamp", + "timezone" : "{{ beat.timezone }}", + "formats" : [ + "ISO8601" + ] + } + }, + { + "remove" : { + "field" : "timestamp" + } + } + ] + }, "field_stats" : { <2> "class" : { "count" : 53, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/filestructurefinder/FileStructure.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/filestructurefinder/FileStructure.java index 1ac9f081ebe6d..f381d5296a48b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/filestructurefinder/FileStructure.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/filestructurefinder/FileStructure.java @@ -17,6 +17,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -103,6 +104,7 @@ public String toString() { public static final ParseField JAVA_TIMESTAMP_FORMATS = new ParseField("java_timestamp_formats"); public static final ParseField NEED_CLIENT_TIMEZONE = new ParseField("need_client_timezone"); public static final ParseField MAPPINGS = new ParseField("mappings"); + public static final ParseField INGEST_PIPELINE = new ParseField("ingest_pipeline"); public static final ParseField FIELD_STATS = new ParseField("field_stats"); public static final ParseField EXPLANATION = new ParseField("explanation"); @@ -128,6 +130,7 @@ public String toString() { PARSER.declareStringArray(Builder::setJavaTimestampFormats, JAVA_TIMESTAMP_FORMATS); PARSER.declareBoolean(Builder::setNeedClientTimezone, NEED_CLIENT_TIMEZONE); PARSER.declareObject(Builder::setMappings, (p, c) -> new TreeMap<>(p.map()), MAPPINGS); + PARSER.declareObject(Builder::setIngestPipeline, (p, c) -> p.mapOrdered(), INGEST_PIPELINE); PARSER.declareObject(Builder::setFieldStats, (p, c) -> { Map fieldStats = new TreeMap<>(); while (p.nextToken() == XContentParser.Token.FIELD_NAME) { @@ -157,6 +160,7 @@ public String toString() { private final String timestampField; private final boolean needClientTimezone; private final SortedMap mappings; + private final Map ingestPipeline; private final SortedMap fieldStats; private final List explanation; @@ -164,8 +168,8 @@ public FileStructure(int numLinesAnalyzed, int numMessagesAnalyzed, String sampl Format format, String multilineStartPattern, String excludeLinesPattern, List columnNames, Boolean hasHeaderRow, Character delimiter, Character quote, Boolean shouldTrimFields, String grokPattern, String timestampField, List jodaTimestampFormats, List javaTimestampFormats, - boolean needClientTimezone, Map mappings, Map fieldStats, - List explanation) { + boolean needClientTimezone, Map mappings, Map ingestPipeline, + Map fieldStats, List explanation) { this.numLinesAnalyzed = numLinesAnalyzed; this.numMessagesAnalyzed = numMessagesAnalyzed; @@ -188,6 +192,7 @@ public FileStructure(int numLinesAnalyzed, int numMessagesAnalyzed, String sampl (javaTimestampFormats == null) ? null : Collections.unmodifiableList(new ArrayList<>(javaTimestampFormats)); this.needClientTimezone = needClientTimezone; this.mappings = Collections.unmodifiableSortedMap(new TreeMap<>(mappings)); + this.ingestPipeline = (ingestPipeline == null) ? null : Collections.unmodifiableMap(new LinkedHashMap<>(ingestPipeline)); this.fieldStats = Collections.unmodifiableSortedMap(new TreeMap<>(fieldStats)); this.explanation = Collections.unmodifiableList(new ArrayList<>(explanation)); } @@ -212,6 +217,7 @@ public FileStructure(StreamInput in) throws IOException { timestampField = in.readOptionalString(); needClientTimezone = in.readBoolean(); mappings = Collections.unmodifiableSortedMap(new TreeMap<>(in.readMap())); + ingestPipeline = in.readBoolean() ? Collections.unmodifiableMap(in.readMap()) : null; fieldStats = Collections.unmodifiableSortedMap(new TreeMap<>(in.readMap(StreamInput::readString, FieldStats::new))); explanation = Collections.unmodifiableList(in.readList(StreamInput::readString)); } @@ -262,6 +268,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(timestampField); out.writeBoolean(needClientTimezone); out.writeMap(mappings); + if (ingestPipeline == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeMap(ingestPipeline); + } out.writeMap(fieldStats, StreamOutput::writeString, (out1, value) -> value.writeTo(out1)); out.writeCollection(explanation, StreamOutput::writeString); } @@ -342,6 +354,10 @@ public SortedMap getMappings() { return mappings; } + public Map getIngestPipeline() { + return ingestPipeline; + } + public SortedMap getFieldStats() { return fieldStats; } @@ -397,6 +413,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.field(NEED_CLIENT_TIMEZONE.getPreferredName(), needClientTimezone); builder.field(MAPPINGS.getPreferredName(), mappings); + if (ingestPipeline != null) { + builder.field(INGEST_PIPELINE.getPreferredName(), ingestPipeline); + } if (fieldStats.isEmpty() == false) { builder.startObject(FIELD_STATS.getPreferredName()); for (Map.Entry entry : fieldStats.entrySet()) { @@ -476,6 +495,7 @@ public static class Builder { private List javaTimestampFormats; private boolean needClientTimezone; private Map mappings; + private Map ingestPipeline; private Map fieldStats = Collections.emptyMap(); private List explanation; @@ -582,6 +602,11 @@ public Builder setMappings(Map mappings) { return this; } + public Builder setIngestPipeline(Map ingestPipeline) { + this.ingestPipeline = ingestPipeline; + return this; + } + public Builder setFieldStats(Map fieldStats) { this.fieldStats = Objects.requireNonNull(fieldStats); return this; @@ -708,7 +733,8 @@ public FileStructure build() { return new FileStructure(numLinesAnalyzed, numMessagesAnalyzed, sampleStart, charset, hasByteOrderMarker, format, multilineStartPattern, excludeLinesPattern, columnNames, hasHeaderRow, delimiter, quote, shouldTrimFields, grokPattern, - timestampField, jodaTimestampFormats, javaTimestampFormats, needClientTimezone, mappings, fieldStats, explanation); + timestampField, jodaTimestampFormats, javaTimestampFormats, needClientTimezone, mappings, ingestPipeline, fieldStats, + explanation); } } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/filestructurefinder/FileStructureTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/filestructurefinder/FileStructureTests.java index d008b31f9a6f7..d1493f2fe4dae 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/filestructurefinder/FileStructureTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/filestructurefinder/FileStructureTests.java @@ -14,6 +14,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; +import java.util.LinkedHashMap; import java.util.Locale; import java.util.Map; import java.util.TreeMap; @@ -74,6 +75,14 @@ public static FileStructure createTestFileStructure() { } builder.setMappings(mappings); + if (randomBoolean()) { + Map ingestPipeline = new LinkedHashMap<>(); + for (String field : generateRandomStringArray(5, 20, false, false)) { + ingestPipeline.put(field, Collections.singletonMap(randomAlphaOfLength(5), randomAlphaOfLength(10))); + } + builder.setMappings(ingestPipeline); + } + if (randomBoolean()) { Map fieldStats = new TreeMap<>(); for (String field : generateRandomStringArray(5, 20, false, false)) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinder.java index 8cdbd030eb5dd..de938d0a9518c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinder.java @@ -142,10 +142,14 @@ static DelimitedFileStructureFinder makeDelimitedFileStructureFinder(List KEYWORD_MAX_LEN || length - str.replaceAll("\\s", "").length() > KEYWORD_MAX_SPACES; } + + /** + * Create an ingest pipeline definition appropriate for the file structure. + * @param grokPattern The Grok pattern used for parsing semi-structured text formats. null for + * fully structured formats. + * @param timestampField The input field containing the timestamp to be parsed into @timestamp. + * null if there is no timestamp. + * @param timestampFormats Timestamp formats to be used for parsing {@code timestampField}. + * May be null if {@code timestampField} is also null. + * @param needClientTimezone Is the timezone of the client supplying data to ingest required to uniquely parse the timestamp? + * @return The ingest pipeline definition, or null if none is required. + */ + public static Map makeIngestPipelineDefinition(String grokPattern, String timestampField, List timestampFormats, + boolean needClientTimezone) { + + if (grokPattern == null && timestampField == null) { + return null; + } + + Map pipeline = new LinkedHashMap<>(); + pipeline.put(Pipeline.DESCRIPTION_KEY, "Ingest pipeline created by file structure finder"); + + List> processors = new ArrayList<>(); + + if (grokPattern != null) { + Map grokProcessorSettings = new LinkedHashMap<>(); + grokProcessorSettings.put("field", "message"); + grokProcessorSettings.put("patterns", Collections.singletonList(grokPattern)); + processors.add(Collections.singletonMap("grok", grokProcessorSettings)); + } + + if (timestampField != null) { + Map dateProcessorSettings = new LinkedHashMap<>(); + dateProcessorSettings.put("field", timestampField); + if (needClientTimezone) { + dateProcessorSettings.put("timezone", "{{ " + BEAT_TIMEZONE_FIELD + " }}"); + } + dateProcessorSettings.put("formats", timestampFormats); + processors.add(Collections.singletonMap("date", dateProcessorSettings)); + } + + // This removes the interim timestamp field used for semi-structured text formats + if (grokPattern != null && timestampField != null) { + processors.add(Collections.singletonMap("remove", Collections.singletonMap("field", timestampField))); + } + + pipeline.put(Pipeline.PROCESSORS_KEY, processors); + return pipeline; + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/JsonFileStructureFinder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/JsonFileStructureFinder.java index 7263474505fe1..8d58ef4e5ca8c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/JsonFileStructureFinder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/JsonFileStructureFinder.java @@ -56,10 +56,14 @@ static JsonFileStructureFinder makeJsonFileStructureFinder(List explanat Tuple timeField = FileStructureUtils.guessTimestampField(explanation, sampleRecords, overrides, timeoutChecker); if (timeField != null) { + boolean needClientTimeZone = timeField.v2().hasTimezoneDependentParsing(); + structureBuilder.setTimestampField(timeField.v1()) .setJodaTimestampFormats(timeField.v2().jodaTimestampFormats) .setJavaTimestampFormats(timeField.v2().javaTimestampFormats) - .setNeedClientTimezone(timeField.v2().hasTimezoneDependentParsing()); + .setNeedClientTimezone(needClientTimeZone) + .setIngestPipeline(FileStructureUtils.makeIngestPipelineDefinition(null, timeField.v1(), + timeField.v2().jodaTimestampFormats, needClientTimeZone)); } Tuple, SortedMap> mappingsAndFieldStats = diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinder.java index 2d3072dda39e5..7578ca8f7fbfb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinder.java @@ -113,12 +113,16 @@ static TextLogFileStructureFinder makeTextLogFileStructureFinder(List ex } } + boolean needClientTimeZone = bestTimestamp.v1().hasTimezoneDependentParsing(); + FileStructure structure = structureBuilder .setTimestampField(interimTimestampField) .setJodaTimestampFormats(bestTimestamp.v1().jodaTimestampFormats) .setJavaTimestampFormats(bestTimestamp.v1().javaTimestampFormats) - .setNeedClientTimezone(bestTimestamp.v1().hasTimezoneDependentParsing()) + .setNeedClientTimezone(needClientTimeZone) .setGrokPattern(grokPattern) + .setIngestPipeline(FileStructureUtils.makeIngestPipelineDefinition(grokPattern, interimTimestampField, + bestTimestamp.v1().jodaTimestampFormats, needClientTimeZone)) .setMappings(mappings) .setFieldStats(fieldStats) .setExplanation(explanation) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinder.java index 1022d6d0ec0d7..4fe0c847c762d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinder.java @@ -95,10 +95,14 @@ static XmlFileStructureFinder makeXmlFileStructureFinder(List explanatio Tuple timeField = FileStructureUtils.guessTimestampField(explanation, sampleRecords, overrides, timeoutChecker); if (timeField != null) { + boolean needClientTimeZone = timeField.v2().hasTimezoneDependentParsing(); + structureBuilder.setTimestampField(timeField.v1()) .setJodaTimestampFormats(timeField.v2().jodaTimestampFormats) .setJavaTimestampFormats(timeField.v2().javaTimestampFormats) - .setNeedClientTimezone(timeField.v2().hasTimezoneDependentParsing()); + .setNeedClientTimezone(needClientTimeZone) + .setIngestPipeline(FileStructureUtils.makeIngestPipelineDefinition(null, topLevelTag + "." + timeField.v1(), + timeField.v2().jodaTimestampFormats, needClientTimeZone)); } Tuple, SortedMap> mappingsAndFieldStats = diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtilsTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtilsTests.java index c0e175f27b2c8..389a65da749a5 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtilsTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtilsTests.java @@ -345,6 +345,75 @@ public void testGuessMappingsAndCalculateFieldStats() { assertNull(fieldStats.get("nothing")); } + public void testMakeIngestPipelineDefinitionGivenStructuredWithoutTimestamp() { + + assertNull(FileStructureUtils.makeIngestPipelineDefinition(null, null, null, false)); + } + + @SuppressWarnings("unchecked") + public void testMakeIngestPipelineDefinitionGivenStructuredWithTimestamp() { + + String timestampField = randomAlphaOfLength(10); + List timestampFormats = randomFrom(TimestampFormatFinder.ORDERED_CANDIDATE_FORMATS).jodaTimestampFormats; + boolean needClientTimezone = randomBoolean(); + + Map pipeline = + FileStructureUtils.makeIngestPipelineDefinition(null, timestampField, timestampFormats, needClientTimezone); + assertNotNull(pipeline); + + assertEquals("Ingest pipeline created by file structure finder", pipeline.remove("description")); + + List> processors = (List>) pipeline.remove("processors"); + assertNotNull(processors); + assertEquals(1, processors.size()); + + Map dateProcessor = (Map) processors.get(0).get("date"); + assertNotNull(dateProcessor); + assertEquals(timestampField, dateProcessor.get("field")); + assertEquals(needClientTimezone, dateProcessor.containsKey("timezone")); + assertEquals(timestampFormats, dateProcessor.get("formats")); + + // After removing the two expected fields there should be nothing left in the pipeline + assertEquals(Collections.emptyMap(), pipeline); + } + + @SuppressWarnings("unchecked") + public void testMakeIngestPipelineDefinitionGivenSemiStructured() { + + String grokPattern = randomAlphaOfLength(100); + String timestampField = randomAlphaOfLength(10); + List timestampFormats = randomFrom(TimestampFormatFinder.ORDERED_CANDIDATE_FORMATS).jodaTimestampFormats; + boolean needClientTimezone = randomBoolean(); + + Map pipeline = + FileStructureUtils.makeIngestPipelineDefinition(grokPattern, timestampField, timestampFormats, needClientTimezone); + assertNotNull(pipeline); + + assertEquals("Ingest pipeline created by file structure finder", pipeline.remove("description")); + + List> processors = (List>) pipeline.remove("processors"); + assertNotNull(processors); + assertEquals(3, processors.size()); + + Map grokProcessor = (Map) processors.get(0).get("grok"); + assertNotNull(grokProcessor); + assertEquals("message", grokProcessor.get("field")); + assertEquals(Collections.singletonList(grokPattern), grokProcessor.get("patterns")); + + Map dateProcessor = (Map) processors.get(1).get("date"); + assertNotNull(dateProcessor); + assertEquals(timestampField, dateProcessor.get("field")); + assertEquals(needClientTimezone, dateProcessor.containsKey("timezone")); + assertEquals(timestampFormats, dateProcessor.get("formats")); + + Map removeProcessor = (Map) processors.get(2).get("remove"); + assertNotNull(removeProcessor); + assertEquals(timestampField, dateProcessor.get("field")); + + // After removing the two expected fields there should be nothing left in the pipeline + assertEquals(Collections.emptyMap(), pipeline); + } + private Map guessMapping(List explanation, String fieldName, List fieldValues) { Tuple, FieldStats> mappingAndFieldStats = FileStructureUtils.guessMappingAndCalculateFieldStats(explanation, fieldName, fieldValues, NOOP_TIMEOUT_CHECKER); From e900ca8676444e64c12a0129b675401e0450dbab Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 10 Oct 2018 09:04:19 +0100 Subject: [PATCH 2/2] Add ingest_pipeline checks to REST tests --- .../resources/rest-api-spec/test/ml/find_file_structure.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/find_file_structure.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/find_file_structure.yml index 6a0414fe9dd61..549305579ed64 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/find_file_structure.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/find_file_structure.yml @@ -36,6 +36,9 @@ - match: { mappings.sourcetype.type: keyword } - match: { mappings.time.type: date } - match: { mappings.time.format: epoch_second } + - match: { ingest_pipeline.description: "Ingest pipeline created by file structure finder" } + - match: { ingest_pipeline.processors.0.date.field: time } + - match: { ingest_pipeline.processors.0.date.formats.0: UNIX } - match: { field_stats.airline.count: 3 } - match: { field_stats.airline.cardinality: 2 } - match: { field_stats.responsetime.count: 3 } @@ -93,6 +96,9 @@ - match: { mappings.sourcetype.type: keyword } - match: { mappings.time.type: date } - match: { mappings.time.format: epoch_second } + - match: { ingest_pipeline.description: "Ingest pipeline created by file structure finder" } + - match: { ingest_pipeline.processors.0.date.field: time } + - match: { ingest_pipeline.processors.0.date.formats.0: UNIX } - match: { field_stats.airline.count: 3 } - match: { field_stats.airline.cardinality: 2 } - match: { field_stats.responsetime.count: 3 }