From 41ebbce438adba421edf1b21aa728206343fb992 Mon Sep 17 00:00:00 2001 From: Iraklis Psaroudakis Date: Tue, 27 Sep 2022 13:45:29 +0300 Subject: [PATCH 1/3] Remove legacy translog header versions and checks Fixes #42699 --- .../index/translog/TranslogHeader.java | 18 +++------- .../index/translog/TranslogHeaderTests.java | 35 +++++++------------ 2 files changed, 17 insertions(+), 36 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java index 21e2311306b7f..e15a20a124ef8 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java @@ -31,9 +31,7 @@ final class TranslogHeader { public static final String TRANSLOG_CODEC = "translog"; - public static final int VERSION_CHECKSUMS = 1; // pre-2.0 - unsupported - public static final int VERSION_CHECKPOINTS = 2; // added checkpoints - public static final int VERSION_PRIMARY_TERM = 3; // added primary term + public static final int VERSION_PRIMARY_TERM = 3; // with: checksums, checkpoints and primary term public static final int CURRENT_VERSION = VERSION_PRIMARY_TERM; private final String translogUUID; @@ -96,17 +94,11 @@ private static int headerSizeInBytes(int version, int uuidLength) { static int readHeaderVersion(final Path path, final FileChannel channel, final StreamInput in) throws IOException { final int version; try { - version = CodecUtil.checkHeader(new InputStreamDataInput(in), TRANSLOG_CODEC, VERSION_CHECKSUMS, VERSION_PRIMARY_TERM); + version = CodecUtil.checkHeader(new InputStreamDataInput(in), TRANSLOG_CODEC, VERSION_PRIMARY_TERM, VERSION_PRIMARY_TERM); } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) { tryReportOldVersionError(path, channel); throw new TranslogCorruptedException(path.toString(), "translog header corrupted", e); } - if (version == VERSION_CHECKSUMS) { - throw new IllegalStateException("pre-2.0 translog found [" + path + "]"); - } - if (version == VERSION_CHECKPOINTS) { - throw new IllegalStateException("pre-6.3 translog found [" + path + "]"); - } return version; } @@ -168,10 +160,8 @@ private static void tryReportOldVersionError(final Path path, final FileChannel // 0x3f => Lucene's magic number, so we can assume it's version 1 or later // 0x00 => version 0 of the translog final byte b1 = Channels.readFromFileChannel(channel, 0, 1)[0]; - if (b1 == 0x3f) { // LUCENE_CODEC_HEADER_BYTE - throw new TranslogCorruptedException(path.toString(), "translog looks like version 1 or later, but has corrupted header"); - } else if (b1 == 0x00) { // UNVERSIONED_TRANSLOG_HEADER_BYTE - throw new IllegalStateException("pre-1.4 translog found [" + path + "]"); + if (b1 == 0x3f || b1 == 0x00) { // LUCENE_CODEC_HEADER_BYTE or UNVERSIONED_TRANSLOG_HEADER_BYTE + throw new TranslogCorruptedException(path.toString(), "translog has corrupted header"); } } diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java index c47ed7c2c39e7..89fa8c1e3712f 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java @@ -17,9 +17,7 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; @@ -51,37 +49,30 @@ public void testCurrentHeaderVersion() throws Exception { final TranslogCorruptedException corruption = expectThrows(TranslogCorruptedException.class, () -> { try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.READ)) { TranslogHeader.read(randomBoolean() ? outHeader.getTranslogUUID() : UUIDs.randomBase64UUID(), translogFile, channel); - } catch (IllegalStateException e) { - // corruption corrupted the version byte making this look like a v2, v1 or v0 translog - assertThat( - "version " + TranslogHeader.VERSION_CHECKPOINTS + "-or-earlier translog", - e.getMessage(), - anyOf( - containsString("pre-2.0 translog found"), - containsString("pre-1.4 translog found"), - containsString("pre-6.3 translog found") - ) - ); - throw new TranslogCorruptedException(translogFile.toString(), "adjusted translog version", e); } }); assertThat(corruption.getMessage(), not(containsString("this translog file belongs to a different translog"))); } public void testLegacyTranslogVersions() { - checkFailsToOpen("/org/elasticsearch/index/translog/translog-v0.binary", IllegalStateException.class, "pre-1.4 translog"); - checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1.binary", IllegalStateException.class, "pre-2.0 translog"); - checkFailsToOpen("/org/elasticsearch/index/translog/translog-v2.binary", IllegalStateException.class, "pre-6.3 translog"); - checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1-truncated.binary", IllegalStateException.class, "pre-2.0 translog"); + final String expectedMessage = "translog has corrupted header"; + checkFailsToOpen("/org/elasticsearch/index/translog/translog-v0.binary", TranslogCorruptedException.class, expectedMessage); + checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1.binary", TranslogCorruptedException.class, expectedMessage); + checkFailsToOpen("/org/elasticsearch/index/translog/translog-v2.binary", TranslogCorruptedException.class, expectedMessage); + checkFailsToOpen( + "/org/elasticsearch/index/translog/translog-v1-truncated.binary", + TranslogCorruptedException.class, + expectedMessage + ); checkFailsToOpen( "/org/elasticsearch/index/translog/translog-v1-corrupted-magic.binary", TranslogCorruptedException.class, - "translog looks like version 1 or later, but has corrupted header" + expectedMessage ); checkFailsToOpen( "/org/elasticsearch/index/translog/translog-v1-corrupted-body.binary", - IllegalStateException.class, - "pre-2.0 translog" + TranslogCorruptedException.class, + expectedMessage ); } @@ -101,7 +92,7 @@ public void testCorruptTranslogHeader() throws Exception { TranslogHeader.read(randomValueOtherThan(translogUUID, UUIDs::randomBase64UUID), translogFile, channel); } }); - assertThat(error, either(instanceOf(IllegalStateException.class)).or(instanceOf(TranslogCorruptedException.class))); + assertThat(error, instanceOf(TranslogCorruptedException.class)); } private void checkFailsToOpen(String file, Class expectedErrorType, String expectedMessage) { From 7b658d3d05c20bc9c6026713d1142ed33be544a3 Mon Sep 17 00:00:00 2001 From: Iraklis Psaroudakis Date: Tue, 27 Sep 2022 14:55:15 +0300 Subject: [PATCH 2/3] Remove tryReportOldVersionError --- .../index/translog/TranslogHeader.java | 16 ---------------- .../index/translog/TranslogHeaderTests.java | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java index e15a20a124ef8..cd94801a5a4df 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java @@ -96,7 +96,6 @@ static int readHeaderVersion(final Path path, final FileChannel channel, final S try { version = CodecUtil.checkHeader(new InputStreamDataInput(in), TRANSLOG_CODEC, VERSION_PRIMARY_TERM, VERSION_PRIMARY_TERM); } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) { - tryReportOldVersionError(path, channel); throw new TranslogCorruptedException(path.toString(), "translog header corrupted", e); } return version; @@ -150,21 +149,6 @@ static TranslogHeader read(final String translogUUID, final Path path, final Fil } } - private static void tryReportOldVersionError(final Path path, final FileChannel channel) throws IOException { - // Lucene's CodecUtil writes a magic number of 0x3FD76C17 with the header, in binary this looks like: - // binary: 0011 1111 1101 0111 0110 1100 0001 0111 - // hex : 3 f d 7 6 c 1 7 - // - // With version 0 of the translog, the first byte is the Operation.Type, which will always be between 0-4, - // so we know if we grab the first byte, it can be: - // 0x3f => Lucene's magic number, so we can assume it's version 1 or later - // 0x00 => version 0 of the translog - final byte b1 = Channels.readFromFileChannel(channel, 0, 1)[0]; - if (b1 == 0x3f || b1 == 0x00) { // LUCENE_CODEC_HEADER_BYTE or UNVERSIONED_TRANSLOG_HEADER_BYTE - throw new TranslogCorruptedException(path.toString(), "translog has corrupted header"); - } - } - /** * Writes this header with the latest format into the file channel */ diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java index 89fa8c1e3712f..ea639860d4e09 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java @@ -55,7 +55,7 @@ public void testCurrentHeaderVersion() throws Exception { } public void testLegacyTranslogVersions() { - final String expectedMessage = "translog has corrupted header"; + final String expectedMessage = "translog header corrupted"; checkFailsToOpen("/org/elasticsearch/index/translog/translog-v0.binary", TranslogCorruptedException.class, expectedMessage); checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1.binary", TranslogCorruptedException.class, expectedMessage); checkFailsToOpen("/org/elasticsearch/index/translog/translog-v2.binary", TranslogCorruptedException.class, expectedMessage); From 8be407bf9df0957fcc54fec0798ad4f8391b46f6 Mon Sep 17 00:00:00 2001 From: Iraklis Psaroudakis Date: Tue, 27 Sep 2022 15:09:08 +0300 Subject: [PATCH 3/3] Fix checks --- .../java/org/elasticsearch/index/translog/TranslogHeader.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java index cd94801a5a4df..a43edd4e256f1 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java @@ -15,7 +15,6 @@ import org.apache.lucene.store.InputStreamDataInput; import org.apache.lucene.store.OutputStreamDataOutput; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.io.Channels; import org.elasticsearch.common.io.stream.InputStreamStreamInput; import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; import org.elasticsearch.common.io.stream.StreamInput;