Backport #14607 to branch_10x (Index open performs version check on each segment, ignores indexCreatedVersionMajor)

lucene/CHANGES.txt

@@ -122,6 +122,8 @@ Other
   Applications using SecurityManager now need to grant SerializablePermission("serialFilter")
   to the analysis-smartcn module. (Uwe Schindler, Isaac David)
 
+* GITHUB#15431: Index open performs version check on each segment, ignores indexCreatedVersionMajor (Rahul Goswami)
+
 Build
 ---------------------
 * Upgrade forbiddenapis to version 3.10.  (Uwe Schindler)

lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.java

@@ -45,6 +45,7 @@
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.tests.util.TestUtil;
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.Version;
 
 public class TestAncientIndicesCompatibility extends LuceneTestCase {
   static final Set<String> UNSUPPORTED_INDEXES;
@@ -198,7 +199,7 @@ public void testUnsupportedOldIndexes() throws Exception {
       checker.setInfoStream(new PrintStream(bos, false, UTF_8));
       checker.setLevel(CheckIndex.Level.MIN_LEVEL_FOR_INTEGRITY_CHECKS);
       CheckIndex.Status indexStatus = checker.checkIndex();
-      if (version.startsWith("8.")) {
+      if (getVersion(version).onOrAfter(Version.fromBits(8, 6, 0))) {
         assertTrue(indexStatus.clean);
       } else {
         assertFalse(indexStatus.clean);
@@ -216,6 +217,18 @@ public void testUnsupportedOldIndexes() throws Exception {
     }
   }
 
+  private Version getVersion(String version) {
+    if (version.startsWith("5x")) {
+      // couple of indices in unsupported_indices.txt start with "5x'
+      return Version.fromBits(5, 0, 0);
+    }
+    String[] versionBitsStr = version.split("[.\\-]");
+    return Version.fromBits(
+        Integer.parseInt(versionBitsStr[0]),
+        Integer.parseInt(versionBitsStr[1]),
+        Integer.parseInt(versionBitsStr[2]));
+  }
+
   // #12895: test on a carefully crafted 9.8.0 index (from a small contiguous subset
   // of wikibigall unique terms) that shows the read-time exception of
   // IntersectTermsEnum (used by WildcardQuery)

lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.java

@@ -861,7 +861,10 @@ public void testFailOpenOldIndex() throws IOException {
             () -> StandardDirectoryReader.open(commit, Version.LATEST.major, null));
     assertTrue(
         ex.getMessage()
-            .contains("only supports reading from version " + Version.LATEST.major + " upwards."));
+            .contains(
+                "This Lucene version only supports indexes with major version "
+                    + Version.LATEST.major
+                    + " or later"));
     // now open with allowed min version
     StandardDirectoryReader.open(commit, Version.MIN_SUPPORTED_MAJOR, null).close();
   }

lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java

@@ -328,7 +328,7 @@ public static final SegmentInfos readCommit(
         throw new IndexFormatTooOldException(
             input, magic, CodecUtil.CODEC_MAGIC, CodecUtil.CODEC_MAGIC);
       }
-      format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_74, VERSION_CURRENT);
+      format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_86, VERSION_CURRENT);
       byte[] id = new byte[StringHelper.ID_LENGTH];
       input.readBytes(id, 0, id.length);
       CodecUtil.checkIndexHeaderSuffix(input, Long.toString(generation, Character.MAX_RADIX));
@@ -346,25 +346,12 @@ public static final SegmentInfos readCommit(
             input);
       }
 
-      if (indexCreatedVersion < minSupportedMajorVersion) {
-        throw new IndexFormatTooOldException(
-            input,
-            "This index was initially created with Lucene "
-                + indexCreatedVersion
-                + ".x while the current version is "
-                + Version.LATEST
-                + " and Lucene only supports reading"
-                + (minSupportedMajorVersion == Version.MIN_SUPPORTED_MAJOR
-                    ? " the current and previous major versions"
-                    : " from version " + minSupportedMajorVersion + " upwards"));
-      }
-
       SegmentInfos infos = new SegmentInfos(indexCreatedVersion);
       infos.id = id;
       infos.generation = generation;
       infos.lastGeneration = generation;
       infos.luceneVersion = luceneVersion;
-      parseSegmentInfos(directory, input, infos, format);
+      parseSegmentInfos(directory, input, infos, format, minSupportedMajorVersion);
       return infos;
 
     } catch (Throwable t) {
@@ -380,7 +367,12 @@ public static final SegmentInfos readCommit(
   }
 
   private static void parseSegmentInfos(
-      Directory directory, DataInput input, SegmentInfos infos, int format) throws IOException {
+      Directory directory,
+      DataInput input,
+      SegmentInfos infos,
+      int format,
+      int minSupportedMajorVersion)
+      throws IOException {
     infos.version = CodecUtil.readBELong(input);
     // System.out.println("READ sis version=" + infos.version);
     infos.counter = input.readVLong();
@@ -397,6 +389,7 @@ private static void parseSegmentInfos(
     }
 
     long totalDocs = 0;
+
     for (int seg = 0; seg < numSegments; seg++) {
       String segName = input.readString();
       byte[] segmentID = new byte[StringHelper.ID_LENGTH];
@@ -490,6 +483,30 @@ private static void parseSegmentInfos(
                 + infos.indexCreatedVersionMajor,
             input);
       }
+
+      int createdOrSegmentMinVersion =
+          info.getMinVersion() == null
+              ? infos.indexCreatedVersionMajor
+              : info.getMinVersion().major;
+
+      // version >=7 are expected to record minVersion
+      if (info.getMinVersion() == null || info.getMinVersion().major < minSupportedMajorVersion) {
+        throw new IndexFormatTooOldException(
+            input,
+            "Index has segments derived from Lucene version "
+                + createdOrSegmentMinVersion
+                + ".x and is not supported by Lucene "
+                + Version.LATEST
+                + ". This Lucene version only supports indexes with major version "
+                + minSupportedMajorVersion
+                + " or later (found: "
+                + createdOrSegmentMinVersion
+                + ", minimum supported: "
+                + minSupportedMajorVersion
+                + "). To resolve this issue re-index your data using Lucene "
+                + minSupportedMajorVersion
+                + ".x or later.");
+      }
     }
 
     infos.userData = input.readMapOfStrings();