Assume rows with duplicate values are not headers

Author: David Roberts

x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/SeparatedValuesLogStructureFinder.java

@@ -18,6 +18,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.DoubleSummaryStatistics;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
@@ -189,12 +190,17 @@ static Tuple<Boolean, String[]> findHeaderFromSample(List<String> explanation, L
         assert rows.isEmpty() == false;
 
         List<String> firstRow = rows.get(0);
-        boolean isHeaderInFile = true;
 
-        if (rows.size() < 3) {
-            explanation.add("Too little data to accurately assess whether header is in sample - guessing it is");
+        boolean isHeaderInFile = true;
+        if (rowContainsDuplicateNonEmptyValues(firstRow)) {
+            isHeaderInFile = false;
+            explanation.add("First row contains duplicate values, so assuming it's not a header");
         } else {
-            isHeaderInFile = isFirstRowUnusual(explanation, rows);
+            if (rows.size() < 3) {
+                explanation.add("Too little data to accurately assess whether header is in sample - guessing it is");
+            } else {
+                isHeaderInFile = isFirstRowUnusual(explanation, rows);
+            }
         }
 
         if (isHeaderInFile) {
@@ -205,6 +211,19 @@ static Tuple<Boolean, String[]> findHeaderFromSample(List<String> explanation, L
         }
     }
 
+    static boolean rowContainsDuplicateNonEmptyValues(List<String> row) {
+
+        HashSet<String> values = new HashSet<>();
+
+        for (String value : row) {
+            if (value != null && value.isEmpty() == false && values.add(value) == false) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     private static boolean isFirstRowUnusual(List<String> explanation, List<List<String>> rows) {
 
         assert rows.size() >= 3;

x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/SeparatedValuesLogStructureFinderTests.java

@@ -279,4 +279,15 @@ public void testLineHasUnescapedQuote() {
         assertTrue(SeparatedValuesLogStructureFinder.lineHasUnescapedQuote("between\"words\tb\tc", CsvPreference.TAB_PREFERENCE));
         assertTrue(SeparatedValuesLogStructureFinder.lineHasUnescapedQuote("x and \"y\"\tb\tc", CsvPreference.TAB_PREFERENCE));
     }
+
+    public void testRowContainsDuplicateNonEmptyValues() {
+
+        assertFalse(SeparatedValuesLogStructureFinder.rowContainsDuplicateNonEmptyValues(Collections.singletonList("a")));
+        assertFalse(SeparatedValuesLogStructureFinder.rowContainsDuplicateNonEmptyValues(Collections.singletonList("")));
+        assertFalse(SeparatedValuesLogStructureFinder.rowContainsDuplicateNonEmptyValues(Arrays.asList("a", "b", "c")));
+        assertTrue(SeparatedValuesLogStructureFinder.rowContainsDuplicateNonEmptyValues(Arrays.asList("a", "b", "a")));
+        assertTrue(SeparatedValuesLogStructureFinder.rowContainsDuplicateNonEmptyValues(Arrays.asList("a", "b", "b")));
+        assertFalse(SeparatedValuesLogStructureFinder.rowContainsDuplicateNonEmptyValues(Arrays.asList("a", "", "")));
+        assertFalse(SeparatedValuesLogStructureFinder.rowContainsDuplicateNonEmptyValues(Arrays.asList("", "a", "")));
+    }
 }