Remove legacy MetaDataStateFormat (#31603)

Removes the legacy (pre-1.5) legacy MetaDataStateFormat.

Author: ywelsch

server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java

@@ -29,19 +29,17 @@
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.OutputStreamIndexOutput;
 import org.apache.lucene.store.SimpleFSDirectory;
-import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.store.IndexOutputOutputStream;
 import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.core.internal.io.IOUtils;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -54,7 +52,6 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -70,9 +67,8 @@ public abstract class MetaDataStateFormat<T> {
     public static final String STATE_FILE_EXTENSION = ".st";
 
     private static final String STATE_FILE_CODEC = "state";
-    private static final int MIN_COMPATIBLE_STATE_FILE_VERSION = 0;
+    private static final int MIN_COMPATIBLE_STATE_FILE_VERSION = 1;
     private static final int STATE_FILE_VERSION = 1;
-    private static final int STATE_FILE_VERSION_ES_2X_AND_BELOW = 0;
     private static final int BUFFER_SIZE = 4096;
     private final String prefix;
     private final Pattern stateFilePattern;
@@ -186,16 +182,11 @@ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) thro
             try (IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) {
                  // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
                 CodecUtil.checksumEntireFile(indexInput);
-                final int fileVersion = CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION,
-                    STATE_FILE_VERSION);
+                CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION, STATE_FILE_VERSION);
                 final XContentType xContentType = XContentType.values()[indexInput.readInt()];
                 if (xContentType != FORMAT) {
                     throw new IllegalStateException("expected state in " + file + " to be " + FORMAT + " format but was " + xContentType);
                 }
-                if (fileVersion == STATE_FILE_VERSION_ES_2X_AND_BELOW) {
-                    // format version 0, wrote a version that always came from the content state file and was never used
-                    indexInput.readLong(); // version currently unused
-                }
                 long filePointer = indexInput.getFilePointer();
                 long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
                 try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) {
@@ -263,10 +254,9 @@ long findMaxStateId(final String prefix, Path... locations) throws IOException {
      * @param dataLocations the data-locations to try.
      * @return the latest state or <code>null</code> if no state was found.
      */
-    public  T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegistry, Path... dataLocations) throws IOException {
+    public T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegistry, Path... dataLocations) throws IOException {
         List<PathAndStateId> files = new ArrayList<>();
         long maxStateId = -1;
-        boolean maxStateIdIsLegacy = true;
         if (dataLocations != null) { // select all eligible files first
             for (Path dataLocation : dataLocations) {
                 final Path stateDir = dataLocation.resolve(STATE_DIR_NAME);
@@ -280,9 +270,7 @@ public  T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegi
                         if (matcher.matches()) {
                             final long stateId = Long.parseLong(matcher.group(1));
                             maxStateId = Math.max(maxStateId, stateId);
-                            final boolean legacy = MetaDataStateFormat.STATE_FILE_EXTENSION.equals(matcher.group(2)) == false;
-                            maxStateIdIsLegacy &= legacy; // on purpose, see NOTE below
-                            PathAndStateId pav = new PathAndStateId(stateFile, stateId, legacy);
+                            PathAndStateId pav = new PathAndStateId(stateFile, stateId);
                             logger.trace("found state file: {}", pav);
                             files.add(pav);
                         }
@@ -292,39 +280,19 @@ public  T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegi
                 }
             }
         }
-        final List<Throwable> exceptions = new ArrayList<>();
-        T state = null;
         // NOTE: we might have multiple version of the latest state if there are multiple data dirs.. for this case
-        //       we iterate only over the ones with the max version. If we have at least one state file that uses the
-        //       new format (ie. legacy == false) then we know that the latest version state ought to use this new format.
-        //       In case the state file with the latest version does not use the new format while older state files do,
-        //       the list below will be empty and loading the state will fail
+        //       we iterate only over the ones with the max version.
+        long finalMaxStateId = maxStateId;
         Collection<PathAndStateId> pathAndStateIds = files
                 .stream()
-                .filter(new StateIdAndLegacyPredicate(maxStateId, maxStateIdIsLegacy))
+                .filter(pathAndStateId -> pathAndStateId.id == finalMaxStateId)
                 .collect(Collectors.toCollection(ArrayList::new));
 
+        final List<Throwable> exceptions = new ArrayList<>();
         for (PathAndStateId pathAndStateId : pathAndStateIds) {
             try {
-                final Path stateFile = pathAndStateId.file;
-                final long id = pathAndStateId.id;
-                if (pathAndStateId.legacy) { // read the legacy format -- plain XContent
-                    final byte[] data = Files.readAllBytes(stateFile);
-                    if (data.length == 0) {
-                        logger.debug("{}: no data for [{}], ignoring...", prefix, stateFile.toAbsolutePath());
-                        continue;
-                    }
-                    try (XContentParser parser = XContentHelper
-                            .createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, new BytesArray(data))) {
-                        state = fromXContent(parser);
-                    }
-                    if (state == null) {
-                        logger.debug("{}: no data for [{}], ignoring...", prefix, stateFile.toAbsolutePath());
-                    }
-                } else {
-                    state = read(namedXContentRegistry, stateFile);
-                    logger.trace("state id [{}] read from [{}]", id, stateFile.getFileName());
-                }
+                T state = read(namedXContentRegistry, pathAndStateId.file);
+                logger.trace("state id [{}] read from [{}]", pathAndStateId.id, pathAndStateId.file.getFileName());
                 return state;
             } catch (Exception e) {
                 exceptions.add(new IOException("failed to read " + pathAndStateId.toString(), e));
@@ -338,46 +306,24 @@ public  T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegi
             // We have some state files but none of them gave us a usable state
             throw new IllegalStateException("Could not find a state file to recover from among " + files);
         }
-        return state;
-    }
-
-    /**
-     * Filters out all {@link org.elasticsearch.gateway.MetaDataStateFormat.PathAndStateId} instances with a different id than
-     * the given one.
-     */
-    private static final class StateIdAndLegacyPredicate implements Predicate<PathAndStateId> {
-        private final long id;
-        private final boolean legacy;
-
-        StateIdAndLegacyPredicate(long id, boolean legacy) {
-            this.id = id;
-            this.legacy = legacy;
-        }
-
-        @Override
-        public boolean test(PathAndStateId input) {
-            return input.id == id && input.legacy == legacy;
-        }
+        return null;
     }
 
     /**
-     * Internal struct-like class that holds the parsed state id, the file
-     * and a flag if the file is a legacy state ie. pre 1.5
+     * Internal struct-like class that holds the parsed state id and the file
      */
     private static class PathAndStateId {
         final Path file;
         final long id;
-        final boolean legacy;
 
-        private PathAndStateId(Path file, long id, boolean legacy) {
+        private PathAndStateId(Path file, long id) {
             this.file = file;
             this.id = id;
-            this.legacy = legacy;
         }
 
         @Override
         public String toString() {
-            return "[id:" + id + ", legacy:" + legacy + ", file:" + file.toAbsolutePath() + "]";
+            return "[id:" + id + ", file:" + file.toAbsolutePath() + "]";
         }
     }
 

server/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java

@@ -81,16 +81,12 @@ protected Settings prepareBackwardsDataDir(Path backwardsIndex) throws IOExcepti
         return builder.build();
     }
 
-    public void testUpgradeStartClusterOn_0_20_6() throws Exception {
-        String indexName = "unsupported-0.20.6";
+    public void testUpgradeStartClusterOn_2_4_5() throws Exception {
+        String indexName = "unsupported-2.4.5";
 
         logger.info("Checking static index {}", indexName);
         Settings nodeSettings = prepareBackwardsDataDir(getBwcIndicesPath().resolve(indexName + ".zip"));
-        try {
-            internalCluster().startNode(nodeSettings);
-            fail();
-        } catch (Exception ex) {
-            assertThat(ex.getCause().getCause().getMessage(), containsString(" was created before v2.0.0.beta1 and wasn't upgraded"));
-        }
+        assertThat(expectThrows(Exception.class, () -> internalCluster().startNode(nodeSettings))
+            .getCause().getCause().getMessage(), containsString("Format version is not supported"));
     }
 }

server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java

@@ -39,7 +39,6 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
@@ -92,7 +91,7 @@ public MetaData fromXContent(XContentParser parser) throws IOException {
         Files.copy(resource, dst);
         MetaData read = format.read(xContentRegistry(), dst);
         assertThat(read, notNullValue());
-        assertThat(read.clusterUUID(), equalTo("3O1tDF1IRB6fSJ-GrTMUtg"));
+        assertThat(read.clusterUUID(), equalTo("y9XcwLJGTROoOEfixlRwfQ"));
         // indices are empty since they are serialized separately
     }
 
@@ -237,7 +236,6 @@ public static void corruptFile(Path file, Logger logger) throws IOException {
     public void testLoadState() throws IOException {
         final Path[] dirs = new Path[randomIntBetween(1, 5)];
         int numStates = randomIntBetween(1, 5);
-        int numLegacy = randomIntBetween(0, numStates);
         List<MetaData> meta = new ArrayList<>();
         for (int i = 0; i < numStates; i++) {
             meta.add(randomMeta());
@@ -247,20 +245,7 @@ public void testLoadState() throws IOException {
         for (int i = 0; i < dirs.length; i++) {
             dirs[i] = createTempDir();
             Files.createDirectories(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME));
-            for (int j = 0; j < numLegacy; j++) {
-                if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) {
-                    Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-"+j);
-                    Files.createFile(file); // randomly create 0-byte files -- there is extra logic to skip them
-                } else {
-                    try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(MetaDataStateFormat.FORMAT,
-                        Files.newOutputStream(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j)))) {
-                        xcontentBuilder.startObject();
-                        MetaData.Builder.toXContent(meta.get(j), xcontentBuilder, ToXContent.EMPTY_PARAMS);
-                        xcontentBuilder.endObject();
-                    }
-                }
-            }
-            for (int j = numLegacy; j < numStates; j++) {
+            for (int j = 0; j < numStates; j++) {
                 format.write(meta.get(j), dirs[i]);
                 if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) {  // corrupt a file that we do not necessarily need here....
                     Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j + ".st");
@@ -290,20 +275,18 @@ public void testLoadState() throws IOException {
         assertThat(loadedMetaData.indexGraveyard(), equalTo(latestMetaData.indexGraveyard()));
 
         // now corrupt all the latest ones and make sure we fail to load the state
-        if (numStates > numLegacy) {
-            for (int i = 0; i < dirs.length; i++) {
-                Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + (numStates-1) + ".st");
-                if (corruptedFiles.contains(file)) {
-                    continue;
-                }
-                MetaDataStateFormatTests.corruptFile(file, logger);
-            }
-            try {
-                format.loadLatestState(logger, xContentRegistry(), dirList.toArray(new Path[0]));
-                fail("latest version can not be read");
-            } catch (ElasticsearchException ex) {
-                assertThat(ExceptionsHelper.unwrap(ex, CorruptStateException.class), notNullValue());
+        for (int i = 0; i < dirs.length; i++) {
+            Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + (numStates-1) + ".st");
+            if (corruptedFiles.contains(file)) {
+                continue;
             }
+            MetaDataStateFormatTests.corruptFile(file, logger);
+        }
+        try {
+            format.loadLatestState(logger, xContentRegistry(), dirList.toArray(new Path[0]));
+            fail("latest version can not be read");
+        } catch (ElasticsearchException ex) {
+            assertThat(ExceptionsHelper.unwrap(ex, CorruptStateException.class), notNullValue());
         }
     }