Skip to content

Commit b612cab

Browse files
spinscalespinscale
committed
Dates: More strict parsing of ISO dates
If you are using the default date or the named identifiers of dates, the current implementation was allowed to read a year with only one digit. In order to make this more strict, this fixes a year to be at least 4 digits. Same applies for month, day, hour, minute, seconds. Also the new default is `strictDateOptionalTime` for indices created with Elasticsearch 2.0 or newer. In addition a couple of not exposed date formats have been exposed, as they have been mentioned in the documentation. Closes #6158
1 parent 35ddc74 commit b612cab

File tree

16 files changed

+2709
-68
lines changed

16 files changed

+2709
-68
lines changed

‎core/src/main/java/org/elasticsearch/common/joda/Joda.java‎

Lines changed: 110 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,8 @@ public static FormatDateTimeFormatter forPattern(String input, Locale locale) {
118118
formatter = ISODateTimeFormat.ordinalDateTimeNoMillis();
119119
} else if ("time".equals(input)) {
120120
formatter = ISODateTimeFormat.time();
121+
} else if ("timeNoMillis".equals(input) || "time_no_millis".equals(input)) {
122+
formatter = ISODateTimeFormat.timeNoMillis();
121123
} else if ("tTime".equals(input) || "t_time".equals(input)) {
122124
formatter = ISODateTimeFormat.tTime();
123125
} else if ("tTimeNoMillis".equals(input) || "t_time_no_millis".equals(input)) {
@@ -126,10 +128,14 @@ public static FormatDateTimeFormatter forPattern(String input, Locale locale) {
126128
formatter = ISODateTimeFormat.weekDate();
127129
} else if ("weekDateTime".equals(input) || "week_date_time".equals(input)) {
128130
formatter = ISODateTimeFormat.weekDateTime();
131+
} else if ("weekDateTimeNoMillis".equals(input) || "week_date_time_no_millis".equals(input)) {
132+
formatter = ISODateTimeFormat.weekDateTimeNoMillis();
129133
} else if ("weekyear".equals(input) || "week_year".equals(input)) {
130134
formatter = ISODateTimeFormat.weekyear();
131-
} else if ("weekyearWeek".equals(input)) {
135+
} else if ("weekyearWeek".equals(input) || "weekyear_week".equals(input)) {
132136
formatter = ISODateTimeFormat.weekyearWeek();
137+
} else if ("weekyearWeekDay".equals(input) || "weekyear_week_day".equals(input)) {
138+
formatter = ISODateTimeFormat.weekyearWeekDay();
133139
} else if ("year".equals(input)) {
134140
formatter = ISODateTimeFormat.year();
135141
} else if ("yearMonth".equals(input) || "year_month".equals(input)) {
@@ -140,6 +146,77 @@ public static FormatDateTimeFormatter forPattern(String input, Locale locale) {
140146
formatter = new DateTimeFormatterBuilder().append(new EpochTimePrinter(false), new EpochTimeParser(false)).toFormatter();
141147
} else if ("epoch_millis".equals(input)) {
142148
formatter = new DateTimeFormatterBuilder().append(new EpochTimePrinter(true), new EpochTimeParser(true)).toFormatter();
149+
// strict date formats here, must be at least 4 digits for year and two for months and two for day
150+
} else if ("strictBasicWeekDate".equals(input) || "strict_basic_week_date".equals(input)) {
151+
formatter = StrictISODateTimeFormat.basicWeekDate();
152+
} else if ("strictBasicWeekDateTime".equals(input) || "strict_basic_week_date_time".equals(input)) {
153+
formatter = StrictISODateTimeFormat.basicWeekDateTime();
154+
} else if ("strictBasicWeekDateTimeNoMillis".equals(input) || "strict_basic_week_date_time_no_millis".equals(input)) {
155+
formatter = StrictISODateTimeFormat.basicWeekDateTimeNoMillis();
156+
} else if ("strictDate".equals(input) || "strict_date".equals(input)) {
157+
formatter = StrictISODateTimeFormat.date();
158+
} else if ("strictDateHour".equals(input) || "strict_date_hour".equals(input)) {
159+
formatter = StrictISODateTimeFormat.dateHour();
160+
} else if ("strictDateHourMinute".equals(input) || "strict_date_hour_minute".equals(input)) {
161+
formatter = StrictISODateTimeFormat.dateHourMinute();
162+
} else if ("strictDateHourMinuteSecond".equals(input) || "strict_date_hour_minute_second".equals(input)) {
163+
formatter = StrictISODateTimeFormat.dateHourMinuteSecond();
164+
} else if ("strictDateHourMinuteSecondFraction".equals(input) || "strict_date_hour_minute_second_fraction".equals(input)) {
165+
formatter = StrictISODateTimeFormat.dateHourMinuteSecondFraction();
166+
} else if ("strictDateHourMinuteSecondMillis".equals(input) || "strict_date_hour_minute_second_millis".equals(input)) {
167+
formatter = StrictISODateTimeFormat.dateHourMinuteSecondMillis();
168+
} else if ("strictDateOptionalTime".equals(input) || "strict_date_optional_time".equals(input)) {
169+
// in this case, we have a separate parser and printer since the dataOptionalTimeParser can't print
170+
// this sucks we should use the root local by default and not be dependent on the node
171+
return new FormatDateTimeFormatter(input,
172+
StrictISODateTimeFormat.dateOptionalTimeParser().withZone(DateTimeZone.UTC),
173+
StrictISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC), locale);
174+
} else if ("strictDateTime".equals(input) || "strict_date_time".equals(input)) {
175+
formatter = StrictISODateTimeFormat.dateTime();
176+
} else if ("strictDateTimeNoMillis".equals(input) || "strict_date_time_no_millis".equals(input)) {
177+
formatter = StrictISODateTimeFormat.dateTimeNoMillis();
178+
} else if ("strictHour".equals(input) || "strict_hour".equals(input)) {
179+
formatter = StrictISODateTimeFormat.hour();
180+
} else if ("strictHourMinute".equals(input) || "strict_hour_minute".equals(input)) {
181+
formatter = StrictISODateTimeFormat.hourMinute();
182+
} else if ("strictHourMinuteSecond".equals(input) || "strict_hour_minute_second".equals(input)) {
183+
formatter = StrictISODateTimeFormat.hourMinuteSecond();
184+
} else if ("strictHourMinuteSecondFraction".equals(input) || "strict_hour_minute_second_fraction".equals(input)) {
185+
formatter = StrictISODateTimeFormat.hourMinuteSecondFraction();
186+
} else if ("strictHourMinuteSecondMillis".equals(input) || "strict_hour_minute_second_millis".equals(input)) {
187+
formatter = StrictISODateTimeFormat.hourMinuteSecondMillis();
188+
} else if ("strictOrdinalDate".equals(input) || "strict_ordinal_date".equals(input)) {
189+
formatter = StrictISODateTimeFormat.ordinalDate();
190+
} else if ("strictOrdinalDateTime".equals(input) || "strict_ordinal_date_time".equals(input)) {
191+
formatter = StrictISODateTimeFormat.ordinalDateTime();
192+
} else if ("strictOrdinalDateTimeNoMillis".equals(input) || "strict_ordinal_date_time_no_millis".equals(input)) {
193+
formatter = StrictISODateTimeFormat.ordinalDateTimeNoMillis();
194+
} else if ("strictTime".equals(input) || "strict_time".equals(input)) {
195+
formatter = StrictISODateTimeFormat.time();
196+
} else if ("strictTimeNoMillis".equals(input) || "strict_time_no_millis".equals(input)) {
197+
formatter = StrictISODateTimeFormat.timeNoMillis();
198+
} else if ("strictTTime".equals(input) || "strict_t_time".equals(input)) {
199+
formatter = StrictISODateTimeFormat.tTime();
200+
} else if ("strictTTimeNoMillis".equals(input) || "strict_t_time_no_millis".equals(input)) {
201+
formatter = StrictISODateTimeFormat.tTimeNoMillis();
202+
} else if ("strictWeekDate".equals(input) || "strict_week_date".equals(input)) {
203+
formatter = StrictISODateTimeFormat.weekDate();
204+
} else if ("strictWeekDateTime".equals(input) || "strict_week_date_time".equals(input)) {
205+
formatter = StrictISODateTimeFormat.weekDateTime();
206+
} else if ("strictWeekDateTimeNoMillis".equals(input) || "strict_week_date_time_no_millis".equals(input)) {
207+
formatter = StrictISODateTimeFormat.weekDateTimeNoMillis();
208+
} else if ("strictWeekyear".equals(input) || "strict_weekyear".equals(input)) {
209+
formatter = StrictISODateTimeFormat.weekyear();
210+
} else if ("strictWeekyearWeek".equals(input) || "strict_weekyear_week".equals(input)) {
211+
formatter = StrictISODateTimeFormat.weekyearWeek();
212+
} else if ("strictWeekyearWeekDay".equals(input) || "strict_weekyear_week_day".equals(input)) {
213+
formatter = StrictISODateTimeFormat.weekyearWeekDay();
214+
} else if ("strictYear".equals(input) || "strict_year".equals(input)) {
215+
formatter = StrictISODateTimeFormat.year();
216+
} else if ("strictYearMonth".equals(input) || "strict_year_month".equals(input)) {
217+
formatter = StrictISODateTimeFormat.yearMonth();
218+
} else if ("strictYearMonthDay".equals(input) || "strict_year_month_day".equals(input)) {
219+
formatter = StrictISODateTimeFormat.yearMonthDay();
143220
} else if (Strings.hasLength(input) && input.contains("||")) {
144221
String[] formats = Strings.delimitedListToStringArray(input, "||");
145222
DateTimeParser[] parsers = new DateTimeParser[formats.length];
@@ -171,6 +248,38 @@ public static FormatDateTimeFormatter forPattern(String input, Locale locale) {
171248
return new FormatDateTimeFormatter(input, formatter.withZone(DateTimeZone.UTC), locale);
172249
}
173250

251+
public static FormatDateTimeFormatter getStrictStandardDateFormatter() {
252+
// 2014/10/10
253+
DateTimeFormatter shortFormatter = new DateTimeFormatterBuilder()
254+
.appendFixedDecimal(DateTimeFieldType.year(), 4)
255+
.appendLiteral('/')
256+
.appendFixedDecimal(DateTimeFieldType.monthOfYear(), 2)
257+
.appendLiteral('/')
258+
.appendFixedDecimal(DateTimeFieldType.dayOfMonth(), 2)
259+
.toFormatter()
260+
.withZoneUTC();
261+
262+
// 2014/10/10 12:12:12
263+
DateTimeFormatter longFormatter = new DateTimeFormatterBuilder()
264+
.appendFixedDecimal(DateTimeFieldType.year(), 4)
265+
.appendLiteral('/')
266+
.appendFixedDecimal(DateTimeFieldType.monthOfYear(), 2)
267+
.appendLiteral('/')
268+
.appendFixedDecimal(DateTimeFieldType.dayOfMonth(), 2)
269+
.appendLiteral(' ')
270+
.appendFixedSignedDecimal(DateTimeFieldType.hourOfDay(), 2)
271+
.appendLiteral(':')
272+
.appendFixedSignedDecimal(DateTimeFieldType.minuteOfHour(), 2)
273+
.appendLiteral(':')
274+
.appendFixedSignedDecimal(DateTimeFieldType.secondOfMinute(), 2)
275+
.toFormatter()
276+
.withZoneUTC();
277+
278+
DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder().append(longFormatter.withZone(DateTimeZone.UTC).getPrinter(), new DateTimeParser[] {longFormatter.getParser(), shortFormatter.getParser()});
279+
280+
return new FormatDateTimeFormatter("yyyy/MM/dd HH:mm:ss||yyyy/MM/dd", builder.toFormatter().withZone(DateTimeZone.UTC), Locale.ROOT);
281+
}
282+
174283

175284
public static final DurationFieldType Quarters = new DurationFieldType("quarters") {
176285
private static final long serialVersionUID = -8167713675442491871L;

‎core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,8 @@ public class DateFieldMapper extends NumberFieldMapper {
6969
public static final String CONTENT_TYPE = "date";
7070

7171
public static class Defaults extends NumberFieldMapper.Defaults {
72-
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("dateOptionalTime||epoch_millis", Locale.ROOT);
72+
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("strictDateOptionalTime||epoch_millis", Locale.ROOT);
73+
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER_BEFORE_2_0 = Joda.forPattern("dateOptionalTime", Locale.ROOT);
7374
public static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
7475
public static final DateFieldType FIELD_TYPE = new DateFieldType();
7576

@@ -123,15 +124,13 @@ public DateFieldMapper build(BuilderContext context) {
123124
}
124125

125126
protected void setupFieldType(BuilderContext context) {
126-
FormatDateTimeFormatter dateTimeFormatter = fieldType().dateTimeFormatter;
127-
// TODO MOVE ME OUTSIDE OF THIS SPACE?
128-
if (Version.indexCreated(context.indexSettings()).before(Version.V_2_0_0)) {
129-
boolean includesEpochFormatter = dateTimeFormatter.format().contains("epoch_");
130-
if (!includesEpochFormatter) {
131-
String format = fieldType().timeUnit().equals(TimeUnit.SECONDS) ? "epoch_second" : "epoch_millis";
132-
fieldType().setDateTimeFormatter(Joda.forPattern(format + "||" + dateTimeFormatter.format()));
133-
}
127+
if (Version.indexCreated(context.indexSettings()).before(Version.V_2_0_0) &&
128+
!fieldType().dateTimeFormatter().format().contains("epoch_")) {
129+
String format = fieldType().timeUnit().equals(TimeUnit.SECONDS) ? "epoch_second" : "epoch_millis";
130+
fieldType().setDateTimeFormatter(Joda.forPattern(format + "||" + fieldType().dateTimeFormatter().format()));
134131
}
132+
133+
FormatDateTimeFormatter dateTimeFormatter = fieldType().dateTimeFormatter;
135134
if (!locale.equals(dateTimeFormatter.locale())) {
136135
fieldType().setDateTimeFormatter(new FormatDateTimeFormatter(dateTimeFormatter.format(), dateTimeFormatter.parser(), dateTimeFormatter.printer(), locale));
137136
}
@@ -159,6 +158,7 @@ public static class TypeParser implements Mapper.TypeParser {
159158
public Mapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
160159
DateFieldMapper.Builder builder = dateField(name);
161160
parseNumberField(builder, name, node, parserContext);
161+
boolean configuredFormat = false;
162162
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
163163
Map.Entry<String, Object> entry = iterator.next();
164164
String propName = Strings.toUnderscoreCase(entry.getKey());
@@ -171,6 +171,7 @@ public static class TypeParser implements Mapper.TypeParser {
171171
iterator.remove();
172172
} else if (propName.equals("format")) {
173173
builder.dateTimeFormatter(parseDateTimeFormatter(propNode));
174+
configuredFormat = true;
174175
iterator.remove();
175176
} else if (propName.equals("numeric_resolution")) {
176177
builder.timeUnit(TimeUnit.valueOf(propNode.toString().toUpperCase(Locale.ROOT)));
@@ -180,6 +181,13 @@ public static class TypeParser implements Mapper.TypeParser {
180181
iterator.remove();
181182
}
182183
}
184+
if (!configuredFormat) {
185+
if (parserContext.indexVersionCreated().onOrAfter(Version.V_2_0_0)) {
186+
builder.dateTimeFormatter(Defaults.DATE_TIME_FORMATTER);
187+
} else {
188+
builder.dateTimeFormatter(Defaults.DATE_TIME_FORMATTER_BEFORE_2_0);
189+
}
190+
}
183191
return builder;
184192
}
185193
}

‎core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,12 @@
2424
import org.apache.lucene.index.IndexOptions;
2525
import org.elasticsearch.Version;
2626
import org.elasticsearch.action.TimestampParsingException;
27-
import org.elasticsearch.common.Explicit;
2827
import org.elasticsearch.common.Nullable;
2928
import org.elasticsearch.common.Strings;
3029
import org.elasticsearch.common.joda.FormatDateTimeFormatter;
3130
import org.elasticsearch.common.joda.Joda;
3231
import org.elasticsearch.common.settings.Settings;
3332
import org.elasticsearch.common.xcontent.XContentBuilder;
34-
import org.elasticsearch.index.analysis.NamedAnalyzer;
3533
import org.elasticsearch.index.analysis.NumericDateAnalyzer;
3634
import org.elasticsearch.index.fielddata.FieldDataType;
3735
import org.elasticsearch.index.mapper.MappedFieldType;
@@ -41,10 +39,8 @@
4139
import org.elasticsearch.index.mapper.MergeResult;
4240
import org.elasticsearch.index.mapper.ParseContext;
4341
import org.elasticsearch.index.mapper.MetadataFieldMapper;
44-
import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
4542
import org.elasticsearch.index.mapper.core.DateFieldMapper;
4643
import org.elasticsearch.index.mapper.core.LongFieldMapper;
47-
import org.elasticsearch.index.mapper.core.NumberFieldMapper;
4844

4945
import java.io.IOException;
5046
import java.util.Iterator;
@@ -59,15 +55,16 @@ public class TimestampFieldMapper extends MetadataFieldMapper {
5955

6056
public static final String NAME = "_timestamp";
6157
public static final String CONTENT_TYPE = "_timestamp";
62-
public static final String DEFAULT_DATE_TIME_FORMAT = "epoch_millis||dateOptionalTime";
58+
public static final String DEFAULT_DATE_TIME_FORMAT = "epoch_millis||strictDateOptionalTime";
6359

6460
public static class Defaults extends DateFieldMapper.Defaults {
6561
public static final String NAME = "_timestamp";
6662

6763
// TODO: this should be removed
68-
public static final MappedFieldType PRE_20_FIELD_TYPE;
69-
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern(DEFAULT_DATE_TIME_FORMAT);
64+
public static final TimestampFieldType PRE_20_FIELD_TYPE;
7065
public static final TimestampFieldType FIELD_TYPE = new TimestampFieldType();
66+
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern(DEFAULT_DATE_TIME_FORMAT);
67+
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER_BEFORE_2_0 = Joda.forPattern("epoch_millis||dateOptionalTime");
7168

7269
static {
7370
FIELD_TYPE.setStored(true);
@@ -82,6 +79,9 @@ public static class Defaults extends DateFieldMapper.Defaults {
8279
PRE_20_FIELD_TYPE = FIELD_TYPE.clone();
8380
PRE_20_FIELD_TYPE.setStored(false);
8481
PRE_20_FIELD_TYPE.setHasDocValues(false);
82+
PRE_20_FIELD_TYPE.setDateTimeFormatter(DATE_TIME_FORMATTER_BEFORE_2_0);
83+
PRE_20_FIELD_TYPE.setIndexAnalyzer(NumericDateAnalyzer.buildNamedAnalyzer(DATE_TIME_FORMATTER_BEFORE_2_0, Defaults.PRECISION_STEP_64_BIT));
84+
PRE_20_FIELD_TYPE.setSearchAnalyzer(NumericDateAnalyzer.buildNamedAnalyzer(DATE_TIME_FORMATTER_BEFORE_2_0, Integer.MAX_VALUE));
8585
PRE_20_FIELD_TYPE.freeze();
8686
}
8787

@@ -146,8 +146,23 @@ public TimestampFieldMapper build(BuilderContext context) {
146146
if (explicitStore == false && context.indexCreatedVersion().before(Version.V_2_0_0)) {
147147
fieldType.setStored(false);
148148
}
149+
150+
if (fieldType().dateTimeFormatter().equals(Defaults.DATE_TIME_FORMATTER)) {
151+
fieldType().setDateTimeFormatter(getDateTimeFormatter(context.indexSettings()));
152+
}
153+
149154
setupFieldType(context);
150-
return new TimestampFieldMapper(fieldType, defaultFieldType, enabledState, path, defaultTimestamp, ignoreMissing, context.indexSettings());
155+
return new TimestampFieldMapper(fieldType, defaultFieldType, enabledState, path, defaultTimestamp,
156+
ignoreMissing, context.indexSettings());
157+
}
158+
}
159+
160+
private static FormatDateTimeFormatter getDateTimeFormatter(Settings indexSettings) {
161+
Version indexCreated = Version.indexCreated(indexSettings);
162+
if (indexCreated.onOrAfter(Version.V_2_0_0)) {
163+
return Defaults.DATE_TIME_FORMATTER;
164+
} else {
165+
return Defaults.DATE_TIME_FORMATTER_BEFORE_2_0;
151166
}
152167
}
153168

@@ -341,7 +356,9 @@ && fieldType().dateTimeFormatter().format().equals(Defaults.DATE_TIME_FORMATTER.
341356
if (indexCreatedBefore2x && (includeDefaults || path != Defaults.PATH)) {
342357
builder.field("path", path);
343358
}
344-
if (includeDefaults || !fieldType().dateTimeFormatter().format().equals(Defaults.DATE_TIME_FORMATTER.format())) {
359+
// different format handling depending on index version
360+
String defaultDateFormat = indexCreatedBefore2x ? Defaults.DATE_TIME_FORMATTER_BEFORE_2_0.format() : Defaults.DATE_TIME_FORMATTER.format();
361+
if (includeDefaults || !fieldType().dateTimeFormatter().format().equals(defaultDateFormat)) {
345362
builder.field("format", fieldType().dateTimeFormatter().format());
346363
}
347364
if (includeDefaults || !Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) {

‎core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ public static class Defaults {
4949
public static final FormatDateTimeFormatter[] DYNAMIC_DATE_TIME_FORMATTERS =
5050
new FormatDateTimeFormatter[]{
5151
DateFieldMapper.Defaults.DATE_TIME_FORMATTER,
52-
Joda.forPattern("yyyy/MM/dd HH:mm:ss||yyyy/MM/dd")
52+
Joda.getStrictStandardDateFormatter()
5353
};
5454
public static final boolean DATE_DETECTION = true;
5555
public static final boolean NUMERIC_DETECTION = false;

0 commit comments

Comments
 (0)