[ML] Remove "8" prefixes from file structure finder timestamp formats (#38016)

In 7.x Java timestamp formats are the default timestamp format and
there is no need to prefix them with "8".  (The "8" prefix was used
in 6.7 to distinguish Java timestamp formats from Joda timestamp
formats.)

This change removes the "8" prefixes from timestamp formats in the
output of the ML file structure finder.

Author: David Roberts

docs/reference/ml/apis/find-file-structure.asciidoc

@@ -606,11 +606,11 @@ If the request does not encounter errors, you receive the following result:
     },
     "tpep_dropoff_datetime" : {
       "type" : "date",
-      "format" : "8yyyy-MM-dd HH:mm:ss"
+      "format" : "yyyy-MM-dd HH:mm:ss"
     },
     "tpep_pickup_datetime" : {
       "type" : "date",
-      "format" : "8yyyy-MM-dd HH:mm:ss"
+      "format" : "yyyy-MM-dd HH:mm:ss"
     },
     "trip_distance" : {
       "type" : "double"
@@ -624,7 +624,7 @@ If the request does not encounter errors, you receive the following result:
           "field" : "tpep_pickup_datetime",
           "timezone" : "{{ beat.timezone }}",
           "formats" : [
-            "8yyyy-MM-dd HH:mm:ss"
+            "yyyy-MM-dd HH:mm:ss"
           ]
         }
       }
@@ -1398,7 +1398,7 @@ this:
           "field" : "timestamp",
           "timezone" : "{{ beat.timezone }}",
           "formats" : [
-            "8yyyy-MM-dd'T'HH:mm:ss,SSS"
+            "yyyy-MM-dd'T'HH:mm:ss,SSS"
           ]
         }
       },
@@ -1558,7 +1558,7 @@ this:
           "field" : "timestamp",
           "timezone" : "{{ beat.timezone }}",
           "formats" : [
-            "8yyyy-MM-dd'T'HH:mm:ss,SSS"
+            "yyyy-MM-dd'T'HH:mm:ss,SSS"
           ]
         }
       },

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtils.java

@@ -353,7 +353,7 @@ public static Map<String, Object> makeIngestPipelineDefinition(String grokPatter
             if (needClientTimezone) {
                 dateProcessorSettings.put("timezone", "{{ " + BEAT_TIMEZONE_FIELD + " }}");
             }
-            dateProcessorSettings.put("formats", jodaBwcJavaTimestampFormatsForIngestPipeline(timestampFormats));
+            dateProcessorSettings.put("formats", timestampFormats);
             processors.add(Collections.singletonMap("date", dateProcessorSettings));
         }
 
@@ -365,19 +365,4 @@ public static Map<String, Object> makeIngestPipelineDefinition(String grokPatter
         pipeline.put(Pipeline.PROCESSORS_KEY, processors);
         return pipeline;
     }
-
-    // TODO: remove this method when Java time formats are the default
-    static List<String> jodaBwcJavaTimestampFormatsForIngestPipeline(List<String> javaTimestampFormats) {
-        return javaTimestampFormats.stream().map(format -> {
-            switch (format) {
-                case "ISO8601":
-                case "UNIX_MS":
-                case "UNIX":
-                case "TAI64N":
-                    return format;
-                default:
-                    return "8" + format;
-            }
-        }).collect(Collectors.toList());
-    }
 }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinder.java

@@ -472,8 +472,7 @@ public Map<String, String> getEsDateMappingTypeWithFormat() {
                     case "UNIX":
                         return Stream.of("epoch_second");
                     default:
-                        // TODO: remove the "8" prefix when Java time formats are the default
-                        return Stream.of("8" + format);
+                        return Stream.of(format);
                 }
             }).collect(Collectors.joining("||"));
             if (formats.isEmpty() == false) {

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtilsTests.java

@@ -331,8 +331,7 @@ public void testGuessMappingsAndCalculateFieldStats() {
         assertEquals(Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "keyword"), mappings.get("foo"));
         Map<String, String> expectedTimeMapping = new HashMap<>();
         expectedTimeMapping.put(FileStructureUtils.MAPPING_TYPE_SETTING, "date");
-        // TODO: remove the "8" prefix when Java time formats are the default
-        expectedTimeMapping.put(FileStructureUtils.MAPPING_FORMAT_SETTING, "8" + "yyyy-MM-dd HH:mm:ss,SSS");
+        expectedTimeMapping.put(FileStructureUtils.MAPPING_FORMAT_SETTING, "yyyy-MM-dd HH:mm:ss,SSS");
         assertEquals(expectedTimeMapping, mappings.get("time"));
         assertEquals(Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "long"), mappings.get("bar"));
         assertNull(mappings.get("nothing"));
@@ -372,8 +371,7 @@ public void testMakeIngestPipelineDefinitionGivenStructuredWithTimestamp() {
         assertNotNull(dateProcessor);
         assertEquals(timestampField, dateProcessor.get("field"));
         assertEquals(needClientTimezone, dateProcessor.containsKey("timezone"));
-        // TODO: remove the call to jodaBwcJavaTimestampFormatsForIngestPipeline() when Java time formats are the default
-        assertEquals(FileStructureUtils.jodaBwcJavaTimestampFormatsForIngestPipeline(timestampFormats), dateProcessor.get("formats"));
+        assertEquals(timestampFormats, dateProcessor.get("formats"));
 
         // After removing the two expected fields there should be nothing left in the pipeline
         assertEquals(Collections.emptyMap(), pipeline);
@@ -406,8 +404,7 @@ public void testMakeIngestPipelineDefinitionGivenSemiStructured() {
         assertNotNull(dateProcessor);
         assertEquals(timestampField, dateProcessor.get("field"));
         assertEquals(needClientTimezone, dateProcessor.containsKey("timezone"));
-        // TODO: remove the call to jodaBwcJavaTimestampFormatsForIngestPipeline() when Java time formats are the default
-        assertEquals(FileStructureUtils.jodaBwcJavaTimestampFormatsForIngestPipeline(timestampFormats), dateProcessor.get("formats"));
+        assertEquals(timestampFormats, dateProcessor.get("formats"));
 
         Map<String, Object> removeProcessor = (Map<String, Object>) processors.get(2).get("remove");
         assertNotNull(removeProcessor);