From 41405e9c09feb16618c49316539745246016ff80 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Fri, 23 Mar 2018 08:58:54 -0400 Subject: [PATCH 01/13] Add primary term to translog header This commit adds the current primary term to the header of the current translog file. Having a term in a translog file allows us to trim translog operations in that file given the max valid seq# for that term. This commit also updates tests to conform the primary term invariant which guarantees that all translog operations in a translog file have its terms at most the term stored in the translog header. --- .../elasticsearch/index/IndexSettings.java | 4 +- .../elasticsearch/index/engine/Engine.java | 15 +- .../index/engine/EngineConfig.java | 11 +- .../index/engine/InternalEngine.java | 2 +- .../elasticsearch/index/shard/IndexShard.java | 2 +- .../index/translog/BaseTranslogReader.java | 22 +- .../index/translog/Translog.java | 35 +-- .../index/translog/TranslogHeader.java | 164 +++++++++++++ .../index/translog/TranslogReader.java | 98 +------- .../index/translog/TranslogSnapshot.java | 8 +- .../index/translog/TranslogWriter.java | 46 +--- .../translog/TruncateTranslogCommand.java | 11 +- .../resync/ResyncReplicationRequestTests.java | 3 +- .../elasticsearch/index/IndexModuleTests.java | 2 +- .../index/engine/EngineDiskUtilsTests.java | 4 +- .../index/engine/InternalEngineTests.java | 68 +++--- .../index/shard/IndexShardIT.java | 2 +- .../index/shard/IndexShardTests.java | 22 +- .../shard/IndexingOperationListenerTests.java | 5 +- .../index/shard/RefreshListenersTests.java | 5 +- .../index/translog/TestTranslog.java | 51 +++-- .../translog/TranslogDeletionPolicyTests.java | 2 +- .../index/translog/TranslogHeaderTests.java | 103 +++++++++ .../index/translog/TranslogTests.java | 215 ++++++++++-------- .../index/translog/TranslogVersionTests.java | 96 -------- .../elasticsearch/indices/flush/FlushIT.java | 2 +- .../recovery/RecoverySourceHandlerTests.java | 2 +- .../translog-invalid-first-byte.binary | Bin 91 -> 0 bytes .../index/translog/translog-v0.binary | Bin 91 -> 0 bytes .../translog-v1-corrupted-body.binary | Bin 417 -> 0 bytes .../translog-v1-corrupted-magic.binary | Bin 417 -> 0 bytes .../translog/translog-v1-truncated.binary | Bin 383 -> 0 bytes .../index/translog/translog-v1.binary | Bin 417 -> 0 bytes .../index/engine/EngineTestCase.java | 25 +- 34 files changed, 582 insertions(+), 443 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java create mode 100644 server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java delete mode 100644 server/src/test/java/org/elasticsearch/index/translog/TranslogVersionTests.java delete mode 100644 server/src/test/resources/org/elasticsearch/index/translog/translog-invalid-first-byte.binary delete mode 100644 server/src/test/resources/org/elasticsearch/index/translog/translog-v0.binary delete mode 100644 server/src/test/resources/org/elasticsearch/index/translog/translog-v1-corrupted-body.binary delete mode 100644 server/src/test/resources/org/elasticsearch/index/translog/translog-v1-corrupted-magic.binary delete mode 100644 server/src/test/resources/org/elasticsearch/index/translog/translog-v1-truncated.binary delete mode 100644 server/src/test/resources/org/elasticsearch/index/translog/translog-v1.binary diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 6c6f05623c355..9fe046bbed8c6 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -185,7 +185,7 @@ public final class IndexSettings { public static final Setting INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING = Setting.byteSizeSetting("index.translog.flush_threshold_size", new ByteSizeValue(512, ByteSizeUnit.MB), /* - * An empty translog occupies 43 bytes on disk. If the flush threshold is below this, the flush thread + * An empty translog occupies 55 bytes on disk. If the flush threshold is below this, the flush thread * can get stuck in an infinite loop as the shouldPeriodicallyFlush can still be true after flushing. * However, small thresholds are useful for testing so we do not add a large lower bound here. */ @@ -220,7 +220,7 @@ public final class IndexSettings { "index.translog.generation_threshold_size", new ByteSizeValue(64, ByteSizeUnit.MB), /* - * An empty translog occupies 43 bytes on disk. If the generation threshold is + * An empty translog occupies 55 bytes on disk. If the generation threshold is * below this, the flush thread can get stuck in an infinite loop repeatedly * rolling the generation as every new generation will already exceed the * generation threshold. However, small thresholds are useful for testing so we diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 1ca4468539da1..351091b0ecfc4 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -1066,14 +1066,13 @@ public Index(Term uid, ParsedDocument doc, long seqNo, long primaryTerm, long ve this.autoGeneratedIdTimestamp = autoGeneratedIdTimestamp; } - public Index(Term uid, ParsedDocument doc) { - this(uid, doc, Versions.MATCH_ANY); + public Index(Term uid, long primaryTerm, ParsedDocument doc) { + this(uid, primaryTerm, doc, Versions.MATCH_ANY); } // TEST ONLY - Index(Term uid, ParsedDocument doc, long version) { - // use a primary term of 2 to allow tests to reduce it to a valid >0 term - this(uid, doc, SequenceNumbers.UNASSIGNED_SEQ_NO, 2, version, VersionType.INTERNAL, - Origin.PRIMARY, System.nanoTime(), -1, false); + Index(Term uid, long primaryTerm, ParsedDocument doc, long version) { + this(uid, doc, SequenceNumbers.UNASSIGNED_SEQ_NO, primaryTerm, version, VersionType.INTERNAL, + Origin.PRIMARY, System.nanoTime(), -1, false); } // TEST ONLY public ParsedDocument parsedDoc() { @@ -1147,8 +1146,8 @@ public Delete(String type, String id, Term uid, long seqNo, long primaryTerm, lo this.id = Objects.requireNonNull(id); } - public Delete(String type, String id, Term uid) { - this(type, id, uid, SequenceNumbers.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, Origin.PRIMARY, System.nanoTime()); + public Delete(String type, String id, Term uid, long primaryTerm) { + this(type, id, uid, SequenceNumbers.UNASSIGNED_SEQ_NO, primaryTerm, Versions.MATCH_ANY, VersionType.INTERNAL, Origin.PRIMARY, System.nanoTime()); } public Delete(Delete template, VersionType versionType) { diff --git a/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java index 352c3ba3e6280..b7c5a41691343 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java @@ -79,6 +79,7 @@ public final class EngineConfig { @Nullable private final CircuitBreakerService circuitBreakerService; private final LongSupplier globalCheckpointSupplier; + private final LongSupplier primaryTermSupplier; /** * Index setting to change the low level lucene codec used for writing new segments. @@ -125,7 +126,7 @@ public EngineConfig(ShardId shardId, String allocationId, ThreadPool threadPool, List externalRefreshListener, List internalRefreshListener, Sort indexSort, TranslogRecoveryRunner translogRecoveryRunner, CircuitBreakerService circuitBreakerService, - LongSupplier globalCheckpointSupplier) { + LongSupplier globalCheckpointSupplier, LongSupplier primaryTermSupplier) { this.shardId = shardId; this.allocationId = allocationId; this.indexSettings = indexSettings; @@ -152,6 +153,7 @@ public EngineConfig(ShardId shardId, String allocationId, ThreadPool threadPool, this.translogRecoveryRunner = translogRecoveryRunner; this.circuitBreakerService = circuitBreakerService; this.globalCheckpointSupplier = globalCheckpointSupplier; + this.primaryTermSupplier = primaryTermSupplier; } /** @@ -354,4 +356,11 @@ public Sort getIndexSort() { public CircuitBreakerService getCircuitBreakerService() { return this.circuitBreakerService; } + + /** + * Returns a supplier that supplies the latest primary term value of the associated shard. + */ + public LongSupplier getPrimaryTermSupplier() { + return primaryTermSupplier; + } } diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index cc5c4799479da..b39dcc02a86b3 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -438,7 +438,7 @@ private Translog openTranslog(EngineConfig engineConfig, TranslogDeletionPolicy final TranslogConfig translogConfig = engineConfig.getTranslogConfig(); final String translogUUID = loadTranslogUUIDFromLastCommit(); // We expect that this shard already exists, so it must already have an existing translog else something is badly wrong! - return new Translog(translogConfig, translogUUID, translogDeletionPolicy, globalCheckpointSupplier); + return new Translog(translogConfig, translogUUID, translogDeletionPolicy, globalCheckpointSupplier, engineConfig.getPrimaryTermSupplier()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 30f813e86e234..c098eec655c8f 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -2154,7 +2154,7 @@ private EngineConfig newEngineConfig() { IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.get(indexSettings.getSettings()), Collections.singletonList(refreshListeners), Collections.singletonList(new RefreshMetricUpdater(refreshMetric)), - indexSort, this::runTranslogRecovery, circuitBreakerService, replicationTracker); + indexSort, this::runTranslogRecovery, circuitBreakerService, replicationTracker, this::getPrimaryTerm); } /** diff --git a/server/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java b/server/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java index d86c4491b63e9..5e97c9598db04 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java +++ b/server/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java @@ -35,15 +35,15 @@ public abstract class BaseTranslogReader implements Comparable getPrimaryTerm() && getPrimaryTerm() != TranslogHeader.UNKNOWN_PRIMARY_TERM) { + throw new TranslogCorruptedException("Operation's term is newer than translog header term; " + + "operation term[" + op.primaryTerm() + "], translog header term [" + getPrimaryTerm() + "]"); + } + return op; } /** diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java index 0043472b72f7c..f7dc0ea88567c 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -109,7 +109,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC public static final String CHECKPOINT_FILE_NAME = "translog" + CHECKPOINT_SUFFIX; static final Pattern PARSE_STRICT_ID_PATTERN = Pattern.compile("^" + TRANSLOG_FILE_PREFIX + "(\\d+)(\\.tlog)$"); - public static final int DEFAULT_HEADER_SIZE_IN_BYTES = TranslogWriter.getHeaderLength(UUIDs.randomBase64UUID()); + public static final int DEFAULT_HEADER_SIZE_IN_BYTES = TranslogHeader.defaultSizeInBytes(UUIDs.randomBase64UUID()); // the list of translog readers is guaranteed to be in order of translog generation private final List readers = new ArrayList<>(); @@ -122,6 +122,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC private final AtomicBoolean closed = new AtomicBoolean(); private final TranslogConfig config; private final LongSupplier globalCheckpointSupplier; + private final LongSupplier primaryTermSupplier; private final String translogUUID; private final TranslogDeletionPolicy deletionPolicy; @@ -140,10 +141,11 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC */ public Translog( final TranslogConfig config, final String translogUUID, TranslogDeletionPolicy deletionPolicy, - final LongSupplier globalCheckpointSupplier) throws IOException { + final LongSupplier globalCheckpointSupplier, final LongSupplier primaryTermSupplier) throws IOException { super(config.getShardId(), config.getIndexSettings()); this.config = config; this.globalCheckpointSupplier = globalCheckpointSupplier; + this.primaryTermSupplier = primaryTermSupplier; this.deletionPolicy = deletionPolicy; this.translogUUID = translogUUID; bigArrays = config.getBigArrays(); @@ -165,7 +167,7 @@ public Translog( // // For this to happen we must have already copied the translog.ckp file into translog-gen.ckp so we first check if that file exists // if not we don't even try to clean it up and wait until we fail creating it - assert Files.exists(nextTranslogFile) == false || Files.size(nextTranslogFile) <= TranslogWriter.getHeaderLength(translogUUID) : "unexpected translog file: [" + nextTranslogFile + "]"; + assert Files.exists(nextTranslogFile) == false || Files.size(nextTranslogFile) <= TranslogHeader.defaultSizeInBytes(translogUUID) : "unexpected translog file: [" + nextTranslogFile + "]"; if (Files.exists(currentCheckpointFile) // current checkpoint is already copied && Files.deleteIfExists(nextTranslogFile)) { // delete it and log a warning logger.warn("deleted previously created, but not yet committed, next generation [{}]. This can happen due to a tragic exception when creating a new generation", nextTranslogFile.getFileName()); @@ -226,6 +228,9 @@ private ArrayList recoverFromFiles(Checkpoint checkpoint) throws minGenerationToRecoverFrom + " checkpoint: " + checkpoint.generation + " - translog ids must be consecutive"); } final TranslogReader reader = openReader(committedTranslogFile, Checkpoint.read(location.resolve(getCommitCheckpointFileName(i)))); + assert reader.getPrimaryTerm() <= primaryTermSupplier.getAsLong() : + "Primary terms go backwards; current term [" + primaryTermSupplier.getAsLong() + "]" + + "translog path [ " + committedTranslogFile + ", existing term [" + reader.getPrimaryTerm() + "]"; foundTranslogs.add(reader); logger.debug("recovered local translog from checkpoint {}", checkpoint); } @@ -459,7 +464,7 @@ TranslogWriter createWriter(long fileGeneration, long initialMinTranslogGen, lon getChannelFactory(), config.getBufferSize(), initialMinTranslogGen, initialGlobalCheckpoint, - globalCheckpointSupplier, this::getMinFileGeneration); + globalCheckpointSupplier, this::getMinFileGeneration, primaryTermSupplier.getAsLong()); } catch (final IOException e) { throw new TranslogException(shardId, "failed to create new translog file", e); } @@ -487,6 +492,10 @@ public Location add(final Operation operation) throws IOException { final ReleasablePagedBytesReference bytes = out.bytes(); try (ReleasableLock ignored = readLock.acquire()) { ensureOpen(); + if (operation.primaryTerm() > current.getPrimaryTerm()) { + throw new IllegalArgumentException("Operation term is newer than the current term;" + + "current term[" + current.getPrimaryTerm() + "], operation term[" + operation + "]"); + } return current.add(bytes, operation.seqNo()); } } catch (final AlreadyClosedException | IOException ex) { @@ -993,17 +1002,17 @@ public Index(Engine.Index index, Engine.IndexResult indexResult) { this.autoGeneratedIdTimestamp = index.getAutoGeneratedIdTimestamp(); } - public Index(String type, String id, long seqNo, byte[] source) { - this(type, id, seqNo, Versions.MATCH_ANY, VersionType.INTERNAL, source, null, null, -1); + public Index(String type, String id, long seqNo, long primaryTerm, byte[] source) { + this(type, id, seqNo, primaryTerm, Versions.MATCH_ANY, VersionType.INTERNAL, source, null, null, -1); } - public Index(String type, String id, long seqNo, long version, VersionType versionType, byte[] source, String routing, - String parent, long autoGeneratedIdTimestamp) { + public Index(String type, String id, long seqNo, long primaryTerm, long version, VersionType versionType, + byte[] source, String routing, String parent, long autoGeneratedIdTimestamp) { this.type = type; this.id = id; this.source = new BytesArray(source); this.seqNo = seqNo; - this.primaryTerm = 0; + this.primaryTerm = primaryTerm; this.version = version; this.versionType = versionType; this.routing = routing; @@ -1188,8 +1197,8 @@ public Delete(Engine.Delete delete, Engine.DeleteResult deleteResult) { } /** utility for testing */ - public Delete(String type, String id, long seqNo, Term uid) { - this(type, id, uid, seqNo, 0, Versions.MATCH_ANY, VersionType.INTERNAL); + public Delete(String type, String id, long seqNo, long primaryTerm, Term uid) { + this(type, id, uid, seqNo, primaryTerm, Versions.MATCH_ANY, VersionType.INTERNAL); } public Delete(String type, String id, Term uid, long seqNo, long primaryTerm, long version, VersionType versionType) { @@ -1393,7 +1402,7 @@ public enum Durability { } - private static void verifyChecksum(BufferedChecksumStreamInput in) throws IOException { + static void verifyChecksum(BufferedChecksumStreamInput in) throws IOException { // This absolutely must come first, or else reading the checksum becomes part of the checksum long expectedChecksum = in.getChecksum(); long readChecksum = in.readInt() & 0xFFFF_FFFFL; @@ -1723,7 +1732,7 @@ static String createEmptyTranslog(Path location, long initialGlobalCheckpoint, S final String translogUUID = UUIDs.randomBase64UUID(); TranslogWriter writer = TranslogWriter.create(shardId, translogUUID, 1, location.resolve(getFilename(1)), channelFactory, new ByteSizeValue(10), 1, initialGlobalCheckpoint, - () -> { throw new UnsupportedOperationException(); }, () -> { throw new UnsupportedOperationException(); } + () -> { throw new UnsupportedOperationException(); }, () -> { throw new UnsupportedOperationException(); }, 0L ); writer.close(); return translogUUID; diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java new file mode 100644 index 0000000000000..69a26684a97fb --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java @@ -0,0 +1,164 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.translog; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFormatTooNewException; +import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.store.InputStreamDataInput; +import org.apache.lucene.store.OutputStreamDataOutput; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.InputStreamStreamInput; +import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; + +import java.io.IOException; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.file.Path; + +/** + * Each translog file is started with a translog header then followed by translog operations. + */ +final class TranslogHeader { + public static final String TRANSLOG_CODEC = "translog"; + + // public static final int VERSION_CHECKSUMS = 1; unsupported version + public static final int VERSION_CHECKPOINTS = 2; // added checkpoints + public static final int VERSION_PRIMARY_TERM = 3; // added primary term + public static final int CURRENT_VERSION = VERSION_PRIMARY_TERM; + + public static final long UNKNOWN_PRIMARY_TERM = -1L; + + private final String translogUUID; + private final long primaryTerm; + private final int headerSizeInBytes; + + TranslogHeader(String translogUUID, long primaryTerm) { + this(translogUUID, primaryTerm, defaultSizeInBytes(translogUUID)); + assert primaryTerm >= 0 : "Primary term must be non-negative; term [" + primaryTerm + "]"; + } + + private TranslogHeader(String translogUUID, long primaryTerm, int headerSizeInBytes) { + this.translogUUID = translogUUID; + this.primaryTerm = primaryTerm; + this.headerSizeInBytes = headerSizeInBytes; + } + + public String getTranslogUUID() { + return translogUUID; + } + + /** + * Returns the primary term stored in this translog header. + * All operations in a translog file are expected to have their primary terms at most this term. + */ + public long getPrimaryTerm() { + return primaryTerm; + } + + /** + * Returns the header size in bytes. This value can be used as the offset of the first translog operation. + * See {@link BaseTranslogReader#getFirstOperationOffset()} + */ + public int sizeInBytes() { + return headerSizeInBytes; + } + + static int defaultSizeInBytes(String translogUUID) { + return headerSize(CURRENT_VERSION, new BytesRef(translogUUID).length); + } + + private static int headerSize(int version, int uuidLength) { + int size = CodecUtil.headerLength(TRANSLOG_CODEC); + size += Integer.BYTES + uuidLength; // uuid + if (version >= VERSION_PRIMARY_TERM) { + size += Long.BYTES; // primary term + size += Integer.BYTES; // checksum + } + return size; + } + + /** + * Read a translog header from the given path and file channel + */ + static TranslogHeader read(final String translogUUID, final Path path, final FileChannel channel) throws IOException { + // This input is intentionally not closed because closing it will close the FileChannel. + final BufferedChecksumStreamInput in = + new BufferedChecksumStreamInput(new InputStreamStreamInput(Channels.newInputStream(channel), channel.size())); + final int version; + try { + version = CodecUtil.checkHeader(new InputStreamDataInput(in), TRANSLOG_CODEC, VERSION_CHECKPOINTS, VERSION_PRIMARY_TERM); + } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) { + throw new TranslogCorruptedException("Translog header corrupted. path:" + path, e); + } + // Read the translogUUID + final int uuidLen = in.readInt(); + if (uuidLen > channel.size()) { + throw new TranslogCorruptedException("uuid length can't be larger than the translog"); + } + final BytesRef uuid = new BytesRef(uuidLen); + uuid.length = uuidLen; + in.read(uuid.bytes, uuid.offset, uuid.length); + final BytesRef expectedUUID = new BytesRef(translogUUID); + if (uuid.bytesEquals(expectedUUID) == false) { + throw new TranslogCorruptedException("expected shard UUID " + expectedUUID + " but got: " + uuid + + " this translog file belongs to a different translog. path:" + path); + } + // Read the primary term + final long primaryTerm; + if (version == VERSION_PRIMARY_TERM) { + primaryTerm = in.readLong(); + assert primaryTerm >= 0 : "Primary term must be non-negative [" + primaryTerm + "]; translog path [" + path + "]"; + } else { + assert version == VERSION_CHECKPOINTS : "Unknown header version [" + version + "]"; + primaryTerm = UNKNOWN_PRIMARY_TERM; + } + // Verify the checksum + if (version >= VERSION_PRIMARY_TERM) { + Translog.verifyChecksum(in); + } + final int headerSizeInBytes = headerSize(version, uuid.length); + assert channel.position() == headerSizeInBytes : + "Header is not fully read; header size [" + headerSizeInBytes + "], position [" + channel.position() + "]"; + return new TranslogHeader(translogUUID, primaryTerm, headerSizeInBytes); + } + + /** + * Writes this header with the latest format into the file channel + */ + void write(FileChannel channel) throws IOException { + // This output is intentionally not closed because closing it will close the FileChannel. + final BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput( + new OutputStreamStreamOutput(Channels.newOutputStream(channel))); + CodecUtil.writeHeader(new OutputStreamDataOutput(out), TRANSLOG_CODEC, CURRENT_VERSION); + // Write uuid + final BytesRef uuid = new BytesRef(translogUUID); + out.writeInt(uuid.length); + out.writeBytes(uuid.bytes, uuid.offset, uuid.length); + // Write primary term + out.writeLong(primaryTerm); + // Checksum header + out.writeInt((int) out.getChecksum()); + channel.force(true); + assert channel.position() == headerSizeInBytes : + "Header is not fully written; header size [" + headerSizeInBytes + "], channel position [" + channel.position() + "]"; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogReader.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogReader.java index b88037c32fd59..29e30bd25dd37 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TranslogReader.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogReader.java @@ -19,15 +19,8 @@ package org.elasticsearch.index.translog; -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexFormatTooNewException; -import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.store.AlreadyClosedException; -import org.apache.lucene.store.InputStreamDataInput; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.Channels; -import org.elasticsearch.common.io.stream.InputStreamStreamInput; import java.io.Closeable; import java.io.EOFException; @@ -41,10 +34,6 @@ * an immutable translog filereader */ public class TranslogReader extends BaseTranslogReader implements Closeable { - - private static final byte LUCENE_CODEC_HEADER_BYTE = 0x3f; - private static final byte UNVERSIONED_TRANSLOG_HEADER_BYTE = 0x00; - protected final long length; private final int totalOperations; private final Checkpoint checkpoint; @@ -53,13 +42,13 @@ public class TranslogReader extends BaseTranslogReader implements Closeable { /** * Create a translog writer against the specified translog file channel. * - * @param checkpoint the translog checkpoint - * @param channel the translog file channel to open a translog reader against - * @param path the path to the translog - * @param firstOperationOffset the offset to the first operation + * @param checkpoint the translog checkpoint + * @param channel the translog file channel to open a translog reader against + * @param path the path to the translog + * @param header the header of the translog file */ - TranslogReader(final Checkpoint checkpoint, final FileChannel channel, final Path path, final long firstOperationOffset) { - super(checkpoint.generation, channel, path, firstOperationOffset); + TranslogReader(final Checkpoint checkpoint, final FileChannel channel, final Path path, final TranslogHeader header) { + super(checkpoint.generation, channel, path, header); this.length = checkpoint.offset; this.totalOperations = checkpoint.numOps; this.checkpoint = checkpoint; @@ -77,75 +66,8 @@ public class TranslogReader extends BaseTranslogReader implements Closeable { */ public static TranslogReader open( final FileChannel channel, final Path path, final Checkpoint checkpoint, final String translogUUID) throws IOException { - - try { - InputStreamStreamInput headerStream = new InputStreamStreamInput(java.nio.channels.Channels.newInputStream(channel), - channel.size()); // don't close - // Lucene's CodecUtil writes a magic number of 0x3FD76C17 with the - // header, in binary this looks like: - // - // binary: 0011 1111 1101 0111 0110 1100 0001 0111 - // hex : 3 f d 7 6 c 1 7 - // - // With version 0 of the translog, the first byte is the - // Operation.Type, which will always be between 0-4, so we know if - // we grab the first byte, it can be: - // 0x3f => Lucene's magic number, so we can assume it's version 1 or later - // 0x00 => version 0 of the translog - // - // otherwise the first byte of the translog is corrupted and we - // should bail - byte b1 = headerStream.readByte(); - if (b1 == LUCENE_CODEC_HEADER_BYTE) { - // Read 3 more bytes, meaning a whole integer has been read - byte b2 = headerStream.readByte(); - byte b3 = headerStream.readByte(); - byte b4 = headerStream.readByte(); - // Convert the 4 bytes that were read into an integer - int header = ((b1 & 0xFF) << 24) + ((b2 & 0xFF) << 16) + ((b3 & 0xFF) << 8) + ((b4 & 0xFF) << 0); - // We confirm CodecUtil's CODEC_MAGIC number (0x3FD76C17) - // ourselves here, because it allows us to read the first - // byte separately - if (header != CodecUtil.CODEC_MAGIC) { - throw new TranslogCorruptedException("translog looks like version 1 or later, but has corrupted header. path:" + path); - } - // Confirm the rest of the header using CodecUtil, extracting - // the translog version - int version = CodecUtil.checkHeaderNoMagic(new InputStreamDataInput(headerStream), TranslogWriter.TRANSLOG_CODEC, 1, Integer.MAX_VALUE); - switch (version) { - case TranslogWriter.VERSION_CHECKSUMS: - throw new IllegalStateException("pre-2.0 translog found [" + path + "]"); - case TranslogWriter.VERSION_CHECKPOINTS: - assert path.getFileName().toString().endsWith(Translog.TRANSLOG_FILE_SUFFIX) : "new file ends with old suffix: " + path; - assert checkpoint.numOps >= 0 : "expected at least 0 operation but got: " + checkpoint.numOps; - assert checkpoint.offset <= channel.size() : "checkpoint is inconsistent with channel length: " + channel.size() + " " + checkpoint; - int len = headerStream.readInt(); - if (len > channel.size()) { - throw new TranslogCorruptedException("uuid length can't be larger than the translog"); - } - BytesRef ref = new BytesRef(len); - ref.length = len; - headerStream.read(ref.bytes, ref.offset, ref.length); - BytesRef uuidBytes = new BytesRef(translogUUID); - if (uuidBytes.bytesEquals(ref) == false) { - throw new TranslogCorruptedException("expected shard UUID " + uuidBytes + " but got: " + ref + - " this translog file belongs to a different translog. path:" + path); - } - final long firstOperationOffset; - firstOperationOffset = ref.length + CodecUtil.headerLength(TranslogWriter.TRANSLOG_CODEC) + Integer.BYTES; - return new TranslogReader(checkpoint, channel, path, firstOperationOffset); - - default: - throw new TranslogCorruptedException("No known translog stream version: " + version + " path:" + path); - } - } else if (b1 == UNVERSIONED_TRANSLOG_HEADER_BYTE) { - throw new IllegalStateException("pre-1.4 translog found [" + path + "]"); - } else { - throw new TranslogCorruptedException("Invalid first byte in translog file, got: " + Long.toHexString(b1) + ", expected 0x00 or 0x3f. path:" + path); - } - } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) { - throw new TranslogCorruptedException("Translog header corrupted. path:" + path, e); - } + final TranslogHeader header = TranslogHeader.read(translogUUID, path, channel); + return new TranslogReader(checkpoint, channel, path, header); } public long sizeInBytes() { @@ -168,8 +90,8 @@ protected void readBytes(ByteBuffer buffer, long position) throws IOException { if (position >= length) { throw new EOFException("read requested past EOF. pos [" + position + "] end: [" + length + "]"); } - if (position < firstOperationOffset) { - throw new IOException("read requested before position of first ops. pos [" + position + "] first op on: [" + firstOperationOffset + "]"); + if (position < getFirstOperationOffset()) { + throw new IOException("read requested before position of first ops. pos [" + position + "] first op on: [" + getFirstOperationOffset() + "]"); } Channels.readFromFileChannelWithEofException(channel, position, buffer); } diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java index 656772fa8169d..24675abaa86a8 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java @@ -39,14 +39,14 @@ final class TranslogSnapshot extends BaseTranslogReader { * Create a snapshot of translog file channel. */ TranslogSnapshot(final BaseTranslogReader reader, final long length) { - super(reader.generation, reader.channel, reader.path, reader.firstOperationOffset); + super(reader.generation, reader.channel, reader.path, reader.header); this.length = length; this.totalOperations = reader.totalOperations(); this.checkpoint = reader.getCheckpoint(); this.reusableBuffer = ByteBuffer.allocate(1024); - readOperations = 0; - position = firstOperationOffset; - reuse = null; + this.readOperations = 0; + this.position = reader.getFirstOperationOffset(); + this.reuse = null; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java index a1e7e18801445..adcfd88254b42 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java @@ -19,10 +19,8 @@ package org.elasticsearch.index.translog; -import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.OutputStreamDataOutput; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.Assertions; import org.elasticsearch.common.bytes.BytesArray; @@ -47,12 +45,6 @@ import java.util.function.LongSupplier; public class TranslogWriter extends BaseTranslogReader implements Closeable { - - public static final String TRANSLOG_CODEC = "translog"; - public static final int VERSION_CHECKSUMS = 1; - public static final int VERSION_CHECKPOINTS = 2; // since 2.0 we have checkpoints? - public static final int VERSION = VERSION_CHECKPOINTS; - private final ShardId shardId; private final ChannelFactory channelFactory; // the last checkpoint that was written when the translog was last synced @@ -85,10 +77,10 @@ private TranslogWriter( final FileChannel channel, final Path path, final ByteSizeValue bufferSize, - final LongSupplier globalCheckpointSupplier, LongSupplier minTranslogGenerationSupplier) throws IOException { - super(initialCheckpoint.generation, channel, path, channel.position()); + final LongSupplier globalCheckpointSupplier, LongSupplier minTranslogGenerationSupplier, TranslogHeader header) throws IOException { + super(initialCheckpoint.generation, channel, path, header); assert initialCheckpoint.offset == channel.position() : - "initial checkpoint offset [" + initialCheckpoint.offset + "] is different than current channel poistion [" + "initial checkpoint offset [" + initialCheckpoint.offset + "] is different than current channel position [" + channel.position() + "]"; this.shardId = shardId; this.channelFactory = channelFactory; @@ -104,34 +96,16 @@ private TranslogWriter( this.seenSequenceNumbers = Assertions.ENABLED ? new HashMap<>() : null; } - static int getHeaderLength(String translogUUID) { - return getHeaderLength(new BytesRef(translogUUID).length); - } - - static int getHeaderLength(int uuidLength) { - return CodecUtil.headerLength(TRANSLOG_CODEC) + uuidLength + Integer.BYTES; - } - - static void writeHeader(OutputStreamDataOutput out, BytesRef ref) throws IOException { - CodecUtil.writeHeader(out, TRANSLOG_CODEC, VERSION); - out.writeInt(ref.length); - out.writeBytes(ref.bytes, ref.offset, ref.length); - } - public static TranslogWriter create(ShardId shardId, String translogUUID, long fileGeneration, Path file, ChannelFactory channelFactory, ByteSizeValue bufferSize, final long initialMinTranslogGen, long initialGlobalCheckpoint, - final LongSupplier globalCheckpointSupplier, final LongSupplier minTranslogGenerationSupplier) + final LongSupplier globalCheckpointSupplier, final LongSupplier minTranslogGenerationSupplier, + final long primaryTerm) throws IOException { - final BytesRef ref = new BytesRef(translogUUID); - final int firstOperationOffset = getHeaderLength(ref.length); final FileChannel channel = channelFactory.open(file); try { - // This OutputStreamDataOutput is intentionally not closed because - // closing it will close the FileChannel - final OutputStreamDataOutput out = new OutputStreamDataOutput(java.nio.channels.Channels.newOutputStream(channel)); - writeHeader(out, ref); - channel.force(true); - final Checkpoint checkpoint = Checkpoint.emptyTranslogCheckpoint(firstOperationOffset, fileGeneration, + final TranslogHeader header = new TranslogHeader(translogUUID, primaryTerm); + header.write(channel); + final Checkpoint checkpoint = Checkpoint.emptyTranslogCheckpoint(header.sizeInBytes(), fileGeneration, initialGlobalCheckpoint, initialMinTranslogGen); writeCheckpoint(channelFactory, file.getParent(), checkpoint); final LongSupplier writerGlobalCheckpointSupplier; @@ -146,7 +120,7 @@ public static TranslogWriter create(ShardId shardId, String translogUUID, long f writerGlobalCheckpointSupplier = globalCheckpointSupplier; } return new TranslogWriter(channelFactory, shardId, checkpoint, channel, file, bufferSize, - writerGlobalCheckpointSupplier, minTranslogGenerationSupplier); + writerGlobalCheckpointSupplier, minTranslogGenerationSupplier, header); } catch (Exception exception) { // if we fail to bake the file-generation into the checkpoint we stick with the file and once we recover and that // file exists we remove it. We only apply this logic to the checkpoint.generation+1 any other file with a higher generation is an error condition @@ -299,7 +273,7 @@ public TranslogReader closeIntoReader() throws IOException { throw e; } if (closed.compareAndSet(false, true)) { - return new TranslogReader(getLastSyncedCheckpoint(), channel, path, getFirstOperationOffset()); + return new TranslogReader(getLastSyncedCheckpoint(), channel, path, header); } else { throw new AlreadyClosedException("translog [" + getGeneration() + "] is already closed (path [" + path + "]", tragedy); } diff --git a/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java b/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java index 164d8fee956dd..bf4dd762a65f2 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java @@ -33,7 +33,6 @@ import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.NativeFSLockFactory; import org.apache.lucene.store.OutputStreamDataOutput; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cli.EnvironmentAwareCommand; @@ -213,13 +212,11 @@ static void writeEmptyCheckpoint(Path filename, int translogLength, long translo * Write a translog containing the given translog UUID to the given location. Returns the number of bytes written. */ public static int writeEmptyTranslog(Path filename, String translogUUID) throws IOException { - final BytesRef translogRef = new BytesRef(translogUUID); - try (FileChannel fc = FileChannel.open(filename, StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE_NEW); - OutputStreamDataOutput out = new OutputStreamDataOutput(Channels.newOutputStream(fc))) { - TranslogWriter.writeHeader(out, translogRef); - fc.force(true); + try (FileChannel fc = FileChannel.open(filename, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)) { + TranslogHeader header = new TranslogHeader(translogUUID, 0L); + header.write(fc); + return header.sizeInBytes(); } - return TranslogWriter.getHeaderLength(translogRef.length); } /** Show a warning about deleting files, asking for a confirmation if {@code batchMode} is false */ diff --git a/server/src/test/java/org/elasticsearch/action/resync/ResyncReplicationRequestTests.java b/server/src/test/java/org/elasticsearch/action/resync/ResyncReplicationRequestTests.java index f1f9fec34de59..313c8cda1fecd 100644 --- a/server/src/test/java/org/elasticsearch/action/resync/ResyncReplicationRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/resync/ResyncReplicationRequestTests.java @@ -37,7 +37,8 @@ public class ResyncReplicationRequestTests extends ESTestCase { public void testSerialization() throws IOException { final byte[] bytes = "{}".getBytes(Charset.forName("UTF-8")); - final Translog.Index index = new Translog.Index("type", "id", 0, Versions.MATCH_ANY, VersionType.INTERNAL, bytes, null, null, -1); + final Translog.Index index = new Translog.Index("type", "id", 0, randomNonNegativeLong(), + Versions.MATCH_ANY, VersionType.INTERNAL, bytes, null, null, -1); final ShardId shardId = new ShardId(new Index("index", "uuid"), 0); final ResyncReplicationRequest before = new ResyncReplicationRequest(shardId, new Translog.Operation[]{index}); diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 706421c5ce73a..8617d7230232a 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -243,7 +243,7 @@ public Engine.Index preIndex(ShardId shardId, Engine.Index operation) { assertSame(listener, indexService.getIndexOperationListeners().get(1)); ParsedDocument doc = InternalEngineTests.createParsedDoc("1", null); - Engine.Index index = new Engine.Index(new Term("_uid", Uid.createUidAsBytes(doc.type(), doc.id())), doc); + Engine.Index index = new Engine.Index(new Term("_uid", Uid.createUidAsBytes(doc.type(), doc.id())), randomNonNegativeLong(), doc); ShardId shardId = new ShardId(new Index("foo", "bar"), 0); for (IndexingOperationListener l : indexService.getIndexOperationListeners()) { l.preIndex(shardId, index); diff --git a/server/src/test/java/org/elasticsearch/index/engine/EngineDiskUtilsTests.java b/server/src/test/java/org/elasticsearch/index/engine/EngineDiskUtilsTests.java index aca94708af9f8..902a2022b0f92 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/EngineDiskUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/EngineDiskUtilsTests.java @@ -96,7 +96,7 @@ threadPool, indexSettings, null, store, newMergePolicy(), config.getAnalyzer(), new CodecService(null, logger), config.getEventListener(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), config.getTranslogConfig(), TimeValue.timeValueMinutes(5), config.getExternalRefreshListener(), config.getInternalRefreshListener(), null, config.getTranslogRecoveryRunner(), - new NoneCircuitBreakerService(), () -> SequenceNumbers.NO_OPS_PERFORMED); + new NoneCircuitBreakerService(), () -> SequenceNumbers.NO_OPS_PERFORMED, config.getPrimaryTermSupplier()); engine = new InternalEngine(newConfig); engine.recoverFromTranslog(); assertVisibleCount(engine, numDocs, false); @@ -198,7 +198,7 @@ public void testHistoryUUIDCanBeForced() throws IOException { new CodecService(null, logger), config.getEventListener(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), config.getTranslogConfig(), TimeValue.timeValueMinutes(5), config.getExternalRefreshListener(), config.getInternalRefreshListener(), null, config.getTranslogRecoveryRunner(), - new NoneCircuitBreakerService(), () -> SequenceNumbers.NO_OPS_PERFORMED); + new NoneCircuitBreakerService(), () -> SequenceNumbers.NO_OPS_PERFORMED, () -> 0L); engine = new InternalEngine(newConfig); engine.recoverFromTranslog(); assertVisibleCount(engine, 0, false); diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index bc4ecbee4d6a8..8b71e98329939 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -230,7 +230,7 @@ public void testVersionMapAfterAutoIDDocument() throws IOException { assertEquals(2, searcher.reader().numDocs()); } assertFalse("safe access should NOT be required last indexing round was only append only", engine.isSafeAccessRequired()); - engine.delete(new Engine.Delete(operation.type(), operation.id(), operation.uid())); + engine.delete(new Engine.Delete(operation.type(), operation.id(), operation.uid(), primaryTerm.get())); assertTrue("safe access should be required", engine.isSafeAccessRequired()); engine.refresh("test"); assertTrue("safe access should be required", engine.isSafeAccessRequired()); @@ -312,7 +312,7 @@ public void testSegments() throws Exception { assertThat(segments.get(1).isCompound(), equalTo(true)); - engine.delete(new Engine.Delete("test", "1", newUid(doc))); + engine.delete(new Engine.Delete("test", "1", newUid(doc), primaryTerm.get())); engine.refresh("test"); segments = engine.segments(false); @@ -885,7 +885,7 @@ public void testSimpleOperations() throws Exception { searchResult.close(); // now delete - engine.delete(new Engine.Delete("test", "1", newUid(doc))); + engine.delete(new Engine.Delete("test", "1", newUid(doc), primaryTerm.get())); // its not deleted yet searchResult = engine.acquireSearcher("test"); @@ -912,7 +912,7 @@ public void testSimpleOperations() throws Exception { document = testDocumentWithTextField(); document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument("1", null, document, B_1, null); - engine.index(new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED)); + engine.index(new Engine.Index(newUid(doc), primaryTerm.get(), doc, Versions.MATCH_DELETED)); // its not there... searchResult = engine.acquireSearcher("test"); @@ -989,7 +989,7 @@ public void testSearchResultRelease() throws Exception { // don't release the search result yet... // delete, refresh and do a new search, it should not be there - engine.delete(new Engine.Delete("test", "1", newUid(doc))); + engine.delete(new Engine.Delete("test", "1", newUid(doc), primaryTerm.get())); engine.refresh("test"); Engine.Searcher updateSearchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(updateSearchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); @@ -1108,7 +1108,7 @@ public void testRenewSyncFlush() throws Exception { engine.index(doc4); assertEquals(engine.getLastWriteNanos(), doc4.startTime()); } else { - Engine.Delete delete = new Engine.Delete(doc1.type(), doc1.id(), doc1.uid()); + Engine.Delete delete = new Engine.Delete(doc1.type(), doc1.id(), doc1.uid(), primaryTerm.get()); engine.delete(delete); assertEquals(engine.getLastWriteNanos(), delete.startTime()); } @@ -1169,7 +1169,7 @@ public void testSyncedFlushVanishesOnReplay() throws IOException { public void testVersioningNewCreate() throws IOException { ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null); - Engine.Index create = new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED); + Engine.Index create = new Engine.Index(newUid(doc), primaryTerm.get(), doc, Versions.MATCH_DELETED); Engine.IndexResult indexResult = engine.index(create); assertThat(indexResult.getVersion(), equalTo(1L)); @@ -1181,7 +1181,7 @@ public void testVersioningNewCreate() throws IOException { public void testReplicatedVersioningWithFlush() throws IOException { ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null); - Engine.Index create = new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED); + Engine.Index create = new Engine.Index(newUid(doc), primaryTerm.get(), doc, Versions.MATCH_DELETED); Engine.IndexResult indexResult = engine.index(create); assertThat(indexResult.getVersion(), equalTo(1L)); assertTrue(indexResult.isCreated()); @@ -1200,7 +1200,7 @@ public void testReplicatedVersioningWithFlush() throws IOException { replicaEngine.flush(); } - Engine.Index update = new Engine.Index(newUid(doc), doc, 1); + Engine.Index update = new Engine.Index(newUid(doc), primaryTerm.get(), doc, 1); Engine.IndexResult updateResult = engine.index(update); assertThat(updateResult.getVersion(), equalTo(2L)); assertFalse(updateResult.isCreated()); @@ -1229,14 +1229,14 @@ public void testVersionedUpdate() throws IOException { final BiFunction searcherFactory = engine::acquireSearcher; ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null); - Engine.Index create = new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED); + Engine.Index create = new Engine.Index(newUid(doc), primaryTerm.get(), doc, Versions.MATCH_DELETED); Engine.IndexResult indexResult = engine.index(create); assertThat(indexResult.getVersion(), equalTo(1L)); try (Engine.GetResult get = engine.get(new Engine.Get(true, doc.type(), doc.id(), create.uid()), searcherFactory)) { assertEquals(1, get.version()); } - Engine.Index update_1 = new Engine.Index(newUid(doc), doc, 1); + Engine.Index update_1 = new Engine.Index(newUid(doc), primaryTerm.get(), doc, 1); Engine.IndexResult update_1_result = engine.index(update_1); assertThat(update_1_result.getVersion(), equalTo(2L)); @@ -1244,7 +1244,7 @@ public void testVersionedUpdate() throws IOException { assertEquals(2, get.version()); } - Engine.Index update_2 = new Engine.Index(newUid(doc), doc, 2); + Engine.Index update_2 = new Engine.Index(newUid(doc), primaryTerm.get(), doc, 2); Engine.IndexResult update_2_result = engine.index(update_2); assertThat(update_2_result.getVersion(), equalTo(3L)); @@ -1285,7 +1285,7 @@ public void testForceMerge() throws IOException { ParsedDocument doc = testParsedDocument(Integer.toString(0), null, testDocument(), B_1, null); Engine.Index index = indexForDoc(doc); - engine.delete(new Engine.Delete(index.type(), index.id(), index.uid())); + engine.delete(new Engine.Delete(index.type(), index.id(), index.uid(), primaryTerm.get())); engine.forceMerge(true, 10, true, false, false); //expunge deletes engine.refresh("test"); @@ -1297,7 +1297,7 @@ public void testForceMerge() throws IOException { doc = testParsedDocument(Integer.toString(1), null, testDocument(), B_1, null); index = indexForDoc(doc); - engine.delete(new Engine.Delete(index.type(), index.id(), index.uid())); + engine.delete(new Engine.Delete(index.type(), index.id(), index.uid(), primaryTerm.get())); engine.forceMerge(true, 10, false, false, false); //expunge deletes engine.refresh("test"); assertEquals(engine.segments(true).size(), 1); @@ -1884,7 +1884,7 @@ public void testBasicCreatedFlag() throws IOException { indexResult = engine.index(index); assertFalse(indexResult.isCreated()); - engine.delete(new Engine.Delete("doc", "1", newUid(doc))); + engine.delete(new Engine.Delete("doc", "1", newUid(doc), primaryTerm.get())); index = indexForDoc(doc); indexResult = engine.index(index); @@ -2358,7 +2358,8 @@ public void testMissingTranslog() throws IOException { // test that we can force start the engine , even if the translog is missing. engine.close(); // fake a new translog, causing the engine to point to a missing one. - Translog translog = createTranslog(); + final long primaryTerm = randomNonNegativeLong(); + Translog translog = createTranslog(() -> primaryTerm); long id = translog.currentFileGeneration(); translog.close(); IOUtils.rm(translog.location().resolve(Translog.getFilename(id))); @@ -2567,7 +2568,7 @@ public void testTranslogReplay() throws IOException { } parser = (TranslogHandler) engine.config().getTranslogRecoveryRunner(); assertEquals(flush ? 1 : 2, parser.appliedOperations()); - engine.delete(new Engine.Delete("test", Integer.toString(randomId), newUid(doc))); + engine.delete(new Engine.Delete("test", Integer.toString(randomId), newUid(doc), primaryTerm.get())); if (randomBoolean()) { engine.refresh("test"); } else { @@ -2596,8 +2597,8 @@ public void testRecoverFromForeignTranslog() throws IOException { final String badUUID = Translog.createEmptyTranslog(badTranslogLog, SequenceNumbers.NO_OPS_PERFORMED, shardId); Translog translog = new Translog( new TranslogConfig(shardId, badTranslogLog, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE), - badUUID, createTranslogDeletionPolicy(INDEX_SETTINGS), () -> SequenceNumbers.NO_OPS_PERFORMED); - translog.add(new Translog.Index("test", "SomeBogusId", 0, "{}".getBytes(Charset.forName("UTF-8")))); + badUUID, createTranslogDeletionPolicy(INDEX_SETTINGS), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get); + translog.add(new Translog.Index("test", "SomeBogusId", 0, primaryTerm.get(), "{}".getBytes(Charset.forName("UTF-8")))); assertEquals(generation.translogFileGeneration, translog.currentFileGeneration()); translog.close(); @@ -2611,7 +2612,7 @@ public void testRecoverFromForeignTranslog() throws IOException { new CodecService(null, logger), config.getEventListener(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), config.getExternalRefreshListener(), config.getInternalRefreshListener(), null, config.getTranslogRecoveryRunner(), - new NoneCircuitBreakerService(), () -> SequenceNumbers.UNASSIGNED_SEQ_NO); + new NoneCircuitBreakerService(), () -> SequenceNumbers.UNASSIGNED_SEQ_NO, primaryTerm::get); try { InternalEngine internalEngine = new InternalEngine(brokenConfig); fail("translog belongs to a different engine"); @@ -2794,11 +2795,11 @@ public void testHandleDocumentFailure() throws Exception { final Engine.DeleteResult deleteResult; if (randomBoolean()) { throwingIndexWriter.get().setThrowFailure(() -> new IOException("simulated")); - deleteResult = engine.delete(new Engine.Delete("test", "1", newUid(doc1))); + deleteResult = engine.delete(new Engine.Delete("test", "1", newUid(doc1), primaryTerm.get())); assertThat(deleteResult.getFailure(), instanceOf(IOException.class)); } else { throwingIndexWriter.get().setThrowFailure(() -> new IllegalArgumentException("simulated max token length")); - deleteResult = engine.delete(new Engine.Delete("test", "1", newUid(doc1))); + deleteResult = engine.delete(new Engine.Delete("test", "1", newUid(doc1), primaryTerm.get())); assertThat(deleteResult.getFailure(), instanceOf(IllegalArgumentException.class)); } @@ -2831,7 +2832,7 @@ public BytesRef binaryValue() { if (randomBoolean()) { engine.index(indexForDoc(doc1)); } else { - engine.delete(new Engine.Delete("test", "", newUid(doc1))); + engine.delete(new Engine.Delete("test", "", newUid(doc1), primaryTerm.get())); } fail("engine should be closed"); } catch (Exception e) { @@ -3377,7 +3378,7 @@ public void testSequenceIDs() throws Exception { seqID = getSequenceID(engine, newGet(false, doc)); logger.info("--> got seqID: {}", seqID); assertThat(seqID.v1(), equalTo(0L)); - assertThat(seqID.v2(), equalTo(2L)); + assertThat(seqID.v2(), equalTo(primaryTerm.get())); // Index the same document again document = testDocumentWithTextField(); @@ -3389,7 +3390,7 @@ public void testSequenceIDs() throws Exception { seqID = getSequenceID(engine, newGet(false, doc)); logger.info("--> got seqID: {}", seqID); assertThat(seqID.v1(), equalTo(1L)); - assertThat(seqID.v2(), equalTo(2L)); + assertThat(seqID.v2(), equalTo(primaryTerm.get())); // Index the same document for the third time, this time changing the primary term document = testDocumentWithTextField(); @@ -3602,13 +3603,12 @@ protected long doGenerateSeqNoForOperation(Operation operation) { } }; noOpEngine.recoverFromTranslog(); - final long primaryTerm = randomNonNegativeLong(); - final int gapsFilled = noOpEngine.fillSeqNoGaps(primaryTerm); + final int gapsFilled = noOpEngine.fillSeqNoGaps(primaryTerm.get()); final String reason = randomAlphaOfLength(16); noOpEngine.noOp( new Engine.NoOp( maxSeqNo + 1, - primaryTerm, + primaryTerm.get(), randomFrom(PRIMARY, REPLICA, PEER_RECOVERY, LOCAL_TRANSLOG_RECOVERY), System.nanoTime(), reason)); @@ -3626,7 +3626,7 @@ protected long doGenerateSeqNoForOperation(Operation operation) { assertThat(last, instanceOf(Translog.NoOp.class)); final Translog.NoOp noOp = (Translog.NoOp) last; assertThat(noOp.seqNo(), equalTo((long) (maxSeqNo + 1))); - assertThat(noOp.primaryTerm(), equalTo(primaryTerm)); + assertThat(noOp.primaryTerm(), equalTo(primaryTerm.get())); assertThat(noOp.reason(), equalTo(reason)); } finally { IOUtils.close(noOpEngine); @@ -3827,7 +3827,7 @@ public void testFillUpSequenceIdGapsOnRecovery() throws IOException { if (operation.opType() == Translog.Operation.Type.NO_OP) { assertEquals(2, operation.primaryTerm()); } else { - assertEquals(1, operation.primaryTerm()); + assertEquals(primaryTerm.get(), operation.primaryTerm()); } } @@ -4095,7 +4095,7 @@ public void testConcurrentAppendUpdateAndRefresh() throws InterruptedException, Engine.Index operation = appendOnlyPrimary(doc, false, 1); engine.index(operation); if (rarely()) { - engine.delete(new Engine.Delete(operation.type(), operation.id(), operation.uid())); + engine.delete(new Engine.Delete(operation.type(), operation.id(), operation.uid(), primaryTerm.get())); numDeletes.incrementAndGet(); } else { doc = testParsedDocument(docID, null, testDocumentWithTextField("updated"), @@ -4259,7 +4259,7 @@ public void testShouldPeriodicallyFlush() throws Exception { engine.index(indexForDoc(doc)); } assertThat("Not exceeded translog flush threshold yet", engine.shouldPeriodicallyFlush(), equalTo(false)); - long flushThreshold = RandomNumbers.randomLongBetween(random(), 100, + long flushThreshold = RandomNumbers.randomLongBetween(random(), 120, engine.getTranslog().stats().getUncommittedSizeInBytes()- extraTranslogSizeInNewEngine); final IndexSettings indexSettings = engine.config().getIndexSettings(); final IndexMetaData indexMetaData = IndexMetaData.builder(indexSettings.getIndexMetaData()) @@ -4301,7 +4301,7 @@ public void testShouldPeriodicallyFlush() throws Exception { } public void testStressShouldPeriodicallyFlush() throws Exception { - final long flushThreshold = randomLongBetween(100, 5000); + final long flushThreshold = randomLongBetween(120, 5000); final long generationThreshold = randomLongBetween(1000, 5000); final IndexSettings indexSettings = engine.config().getIndexSettings(); final IndexMetaData indexMetaData = IndexMetaData.builder(indexSettings.getIndexMetaData()) @@ -4342,7 +4342,7 @@ public void testStressUpdateSameDocWhileGettingIt() throws IOException, Interrup Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), 0, false); // first index an append only document and then delete it. such that we have it in the tombstones engine.index(doc); - engine.delete(new Engine.Delete(doc.type(), doc.id(), doc.uid())); + engine.delete(new Engine.Delete(doc.type(), doc.id(), doc.uid(), primaryTerm.get())); // now index more append only docs and refresh so we re-enabel the optimization for unsafe version map ParsedDocument document1 = testParsedDocument(Integer.toString(1), null, testDocumentWithTextField(), SOURCE, null); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index b14030d46e4ca..5275bfd8b7474 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -333,7 +333,7 @@ public void testMaybeFlush() throws Exception { assertFalse(shard.shouldPeriodicallyFlush()); client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder() .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), - new ByteSizeValue(160 /* size of the operation + two generations header&footer*/, ByteSizeUnit.BYTES)).build()).get(); + new ByteSizeValue(190 /* size of the operation + two generations header&footer*/, ByteSizeUnit.BYTES)).build()).get(); client().prepareIndex("test", "test", "0") .setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); assertFalse(shard.shouldPeriodicallyFlush()); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index f05fdc60c5cf7..eda26b4d00373 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -90,6 +90,7 @@ import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.store.StoreStats; +import org.elasticsearch.index.translog.TestTranslog; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogTests; import org.elasticsearch.indices.IndicesQueryCache; @@ -519,6 +520,7 @@ public void testPrimaryPromotionRollsGeneration() throws Exception { // promote the replica final ShardRouting replicaRouting = indexShard.routingEntry(); + final long newPrimaryTerm = indexShard.getPrimaryTerm() + between(1, 10000); final ShardRouting primaryRouting = newShardRouting( replicaRouting.shardId(), @@ -527,7 +529,7 @@ public void testPrimaryPromotionRollsGeneration() throws Exception { true, ShardRoutingState.STARTED, replicaRouting.allocationId()); - indexShard.updateShardState(primaryRouting, indexShard.getPrimaryTerm() + 1, (shard, listener) -> {}, + indexShard.updateShardState(primaryRouting, newPrimaryTerm, (shard, listener) -> {}, 0L, Collections.singleton(primaryRouting.allocationId().getId()), new IndexShardRoutingTable.Builder(primaryRouting.shardId()).addShard(primaryRouting).build(), Collections.emptySet()); @@ -553,6 +555,7 @@ public void onFailure(Exception e) { latch.await(); assertThat(indexShard.getTranslog().getGeneration().translogFileGeneration, equalTo(currentTranslogGeneration + 1)); + assertThat(TestTranslog.getCurrentTerm(indexShard.getTranslog()), equalTo(newPrimaryTerm)); closeShards(indexShard); } @@ -571,7 +574,10 @@ public void testOperationPermitsOnPrimaryShards() throws InterruptedException, E ShardRouting replicaRouting = indexShard.routingEntry(); ShardRouting primaryRouting = newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), null, true, ShardRoutingState.STARTED, replicaRouting.allocationId()); - indexShard.updateShardState(primaryRouting, indexShard.getPrimaryTerm() + 1, (shard, listener) -> {}, 0L, + final long newPrimaryTerm = indexShard.getPrimaryTerm() + between(1, 1000); + indexShard.updateShardState(primaryRouting, newPrimaryTerm, (shard, listener) -> { + assertThat(TestTranslog.getCurrentTerm(indexShard.getTranslog()), equalTo(newPrimaryTerm)); + }, 0L, Collections.singleton(indexShard.routingEntry().allocationId().getId()), new IndexShardRoutingTable.Builder(indexShard.shardId()).addShard(primaryRouting).build(), Collections.emptySet()); @@ -739,6 +745,7 @@ public void onFailure(Exception e) { @Override public void onResponse(Releasable releasable) { assertThat(indexShard.getPrimaryTerm(), equalTo(newPrimaryTerm)); + assertThat(TestTranslog.getCurrentTerm(indexShard.getTranslog()), equalTo(newPrimaryTerm)); assertThat(indexShard.getLocalCheckpoint(), equalTo(expectedLocalCheckpoint)); assertThat(indexShard.getGlobalCheckpoint(), equalTo(newGlobalCheckPoint)); onResponse.set(true); @@ -784,15 +791,18 @@ private void finish() { assertFalse(onResponse.get()); assertNull(onFailure.get()); assertThat(indexShard.getPrimaryTerm(), equalTo(primaryTerm)); + assertThat(TestTranslog.getCurrentTerm(indexShard.getTranslog()), equalTo(primaryTerm)); Releasables.close(operation1); // our operation should still be blocked assertFalse(onResponse.get()); assertNull(onFailure.get()); assertThat(indexShard.getPrimaryTerm(), equalTo(primaryTerm)); + assertThat(TestTranslog.getCurrentTerm(indexShard.getTranslog()), equalTo(primaryTerm)); Releasables.close(operation2); barrier.await(); // now lock acquisition should have succeeded assertThat(indexShard.getPrimaryTerm(), equalTo(newPrimaryTerm)); + assertThat(TestTranslog.getCurrentTerm(indexShard.getTranslog()), equalTo(newPrimaryTerm)); if (engineClosed) { assertFalse(onResponse.get()); assertThat(onFailure.get(), instanceOf(AlreadyClosedException.class)); @@ -1738,7 +1748,7 @@ public void testRecoverFromStoreRemoveStaleOperations() throws Exception { flushShard(shard); assertThat(getShardDocUIDs(shard), containsInAnyOrder("doc-0", "doc-1")); // Simulate resync (without rollback): Noop #1, index #2 - shard.primaryTerm++; + acquireReplicaOperationPermitBlockingly(shard, shard.primaryTerm + 1); shard.markSeqNoAsNoop(1, "test"); shard.applyIndexOperationOnReplica(2, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, SourceToParse.source(indexName, "doc", "doc-2", new BytesArray("{}"), XContentType.JSON), mapping); @@ -2051,18 +2061,18 @@ public void testRecoverFromTranslog() throws IOException { IndexMetaData metaData = IndexMetaData.builder("test") .putMapping("test", "{ \"properties\": { \"foo\": { \"type\": \"text\"}}}") .settings(settings) - .primaryTerm(0, 1).build(); + .primaryTerm(0, randomLongBetween(1, Long.MAX_VALUE)).build(); IndexShard primary = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, null); List operations = new ArrayList<>(); int numTotalEntries = randomIntBetween(0, 10); int numCorruptEntries = 0; for (int i = 0; i < numTotalEntries; i++) { if (randomBoolean()) { - operations.add(new Translog.Index("test", "1", 0, 1, VersionType.INTERNAL, + operations.add(new Translog.Index("test", "1", 0, primary.getPrimaryTerm(), 1, VersionType.INTERNAL, "{\"foo\" : \"bar\"}".getBytes(Charset.forName("UTF-8")), null, null, -1)); } else { // corrupt entry - operations.add(new Translog.Index("test", "2", 1, 1, VersionType.INTERNAL, + operations.add(new Translog.Index("test", "2", 1, primary.getPrimaryTerm(), 1, VersionType.INTERNAL, "{\"foo\" : \"bar}".getBytes(Charset.forName("UTF-8")), null, null, -1)); numCorruptEntries++; } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java index f3bf76c57a550..a847add0927a8 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java @@ -136,8 +136,9 @@ public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) { IndexingOperationListener.CompositeListener compositeListener = new IndexingOperationListener.CompositeListener(indexingOperationListeners, logger); ParsedDocument doc = InternalEngineTests.createParsedDoc("1", null); - Engine.Delete delete = new Engine.Delete("test", "1", new Term("_uid", Uid.createUidAsBytes(doc.type(), doc.id()))); - Engine.Index index = new Engine.Index(new Term("_uid", Uid.createUidAsBytes(doc.type(), doc.id())), doc); + Engine.Delete delete = new Engine.Delete("test", "1", + new Term("_uid", Uid.createUidAsBytes(doc.type(), doc.id())), randomNonNegativeLong()); + Engine.Index index = new Engine.Index(new Term("_uid", Uid.createUidAsBytes(doc.type(), doc.id())), randomNonNegativeLong(), doc); compositeListener.postDelete(randomShardId, delete, new Engine.DeleteResult(1, SequenceNumbers.UNASSIGNED_SEQ_NO, true)); assertEquals(0, preIndex.get()); assertEquals(0, postIndex.get()); diff --git a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java index 1f9c5ae6df359..467012c6ac090 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java @@ -122,11 +122,12 @@ public void onFailedEngine(String reason, @Nullable Exception e) { } }; EngineDiskUtils.createEmpty(store.directory(), translogConfig.getTranslogPath(), shardId); + final long primaryTerm = randomNonNegativeLong(); EngineConfig config = new EngineConfig(shardId, allocationId, threadPool, indexSettings, null, store, newMergePolicy(), iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), eventListener, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), Collections.singletonList(listeners), Collections.emptyList(), null, - (e, s) -> 0, new NoneCircuitBreakerService(), () -> SequenceNumbers.NO_OPS_PERFORMED); + (e, s) -> 0, new NoneCircuitBreakerService(), () -> SequenceNumbers.NO_OPS_PERFORMED, () -> primaryTerm); engine = new InternalEngine(config); engine.recoverFromTranslog(); listeners.setTranslog(engine.getTranslog()); @@ -360,7 +361,7 @@ private Engine.IndexResult index(String id, String testFieldValue) throws IOExce BytesReference source = new BytesArray(new byte[] { 1 }); ParsedDocument doc = new ParsedDocument(versionField, seqID, id, "test", null, Arrays.asList(document), source, XContentType.JSON, null); - Engine.Index index = new Engine.Index(new Term("_id", doc.id()), doc); + Engine.Index index = new Engine.Index(new Term("_id", doc.id()), engine.config().getPrimaryTermSupplier().getAsLong(), doc); return engine.index(index); } diff --git a/server/src/test/java/org/elasticsearch/index/translog/TestTranslog.java b/server/src/test/java/org/elasticsearch/index/translog/TestTranslog.java index 4077d033da9cd..7ab9fa6733011 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TestTranslog.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TestTranslog.java @@ -83,26 +83,7 @@ public static Set corruptTranslogFiles(Logger logger, Random random, Colle int corruptions = RandomNumbers.randomIntBetween(random, 5, 20); for (int i = 0; i < corruptions; i++) { Path fileToCorrupt = RandomPicks.randomFrom(random, candidates); - try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) { - // read - raf.position(RandomNumbers.randomLongBetween(random, 0, raf.size() - 1)); - long filePointer = raf.position(); - ByteBuffer bb = ByteBuffer.wrap(new byte[1]); - raf.read(bb); - bb.flip(); - - // corrupt - byte oldValue = bb.get(0); - byte newValue = (byte) (oldValue + 1); - bb.put(0, newValue); - - // rewrite - raf.position(filePointer); - raf.write(bb); - logger.info("--> corrupting file {} -- flipping at position {} from {} to {} file: {}", - fileToCorrupt, filePointer, Integer.toHexString(oldValue), - Integer.toHexString(newValue), fileToCorrupt); - } + corruptFile(logger, random, fileToCorrupt); corruptedFiles.add(fileToCorrupt); } } @@ -110,6 +91,29 @@ public static Set corruptTranslogFiles(Logger logger, Random random, Colle return corruptedFiles; } + static void corruptFile(Logger logger, Random random, Path fileToCorrupt) throws IOException { + try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + // read + raf.position(RandomNumbers.randomLongBetween(random, 0, raf.size() - 1)); + long filePointer = raf.position(); + ByteBuffer bb = ByteBuffer.wrap(new byte[1]); + raf.read(bb); + bb.flip(); + + // corrupt + byte oldValue = bb.get(0); + byte newValue = (byte) (oldValue + 1); + bb.put(0, newValue); + + // rewrite + raf.position(filePointer); + raf.write(bb); + logger.info("--> corrupting file {} -- flipping at position {} from {} to {} file: {}", + fileToCorrupt, filePointer, Integer.toHexString(oldValue), + Integer.toHexString(newValue), fileToCorrupt); + } + } + /** * Lists all existing commits in a given index path, then read the minimum translog generation that will be used in recoverFromTranslog. */ @@ -122,4 +126,11 @@ private static long minTranslogGenUsedInRecovery(Path translogPath) throws IOExc return Long.parseLong(recoveringCommit.getUserData().get(Translog.TRANSLOG_GENERATION_KEY)); } } + + /** + * Returns the primary term associated with the current translog writer of the given translog. + */ + public static long getCurrentTerm(Translog translog) { + return translog.getCurrent().getPrimaryTerm(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java index 2f6f4ee3178f2..9ae502fecb580 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java @@ -171,7 +171,7 @@ private Tuple, TranslogWriter> createReadersAndWriter(final } writer = TranslogWriter.create(new ShardId("index", "uuid", 0), translogUUID, gen, tempDir.resolve(Translog.getFilename(gen)), FileChannel::open, TranslogConfig.DEFAULT_BUFFER_SIZE, 1L, 1L, () -> 1L, - () -> 1L); + () -> 1L, randomNonNegativeLong()); writer = Mockito.spy(writer); Mockito.doReturn(now - (numberOfReaders - gen + 1) * 1000).when(writer).getLastModifiedTime(); diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java new file mode 100644 index 0000000000000..22c11e663cc3e --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java @@ -0,0 +1,103 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.translog; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.store.OutputStreamDataOutput; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThan; + +public class TranslogHeaderTests extends ESTestCase { + + public void testCurrentHeaderVersion() throws Exception { + final String translogUUID = UUIDs.randomBase64UUID(); + final TranslogHeader outHeader = new TranslogHeader(translogUUID, randomNonNegativeLong()); + final long generation = randomNonNegativeLong(); + final Path translogFile = createTempDir().resolve(Translog.getFilename(generation)); + try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + outHeader.write(channel); + assertThat(outHeader.sizeInBytes(), equalTo((int)channel.position())); + } + try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.READ)) { + final TranslogHeader inHeader = TranslogHeader.read(translogUUID, translogFile, channel); + assertThat(inHeader.getTranslogUUID(), equalTo(translogUUID)); + assertThat(inHeader.getPrimaryTerm(), equalTo(outHeader.getPrimaryTerm())); + assertThat(inHeader.sizeInBytes(), equalTo((int)channel.position())); + } + final TranslogCorruptedException mismatchUUID = expectThrows(TranslogCorruptedException.class, () -> { + try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.READ)) { + TranslogHeader.read(UUIDs.randomBase64UUID(), translogFile, channel); + } + }); + assertThat(mismatchUUID.getMessage(), containsString("this translog file belongs to a different translog")); + int corruptions = between(1, 10); + for (int i = 0; i < corruptions; i++) { + TestTranslog.corruptFile(logger, random(), translogFile); + } + expectThrows(TranslogCorruptedException.class, () -> { + try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.READ)) { + TranslogHeader.read(outHeader.getTranslogUUID(), translogFile, channel); + } + }); + } + + public void testHeaderWithoutPrimaryTerm() throws Exception { + final String translogUUID = UUIDs.randomBase64UUID(); + final long generation = randomNonNegativeLong(); + final Path translogFile = createTempDir().resolve(Translog.getFilename(generation)); + try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + writeLegacyTranslogHeader(channel, translogUUID); + assertThat((int)channel.position(), lessThan(TranslogHeader.defaultSizeInBytes(translogUUID))); + } + try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.READ)) { + final TranslogHeader inHeader = TranslogHeader.read(translogUUID, translogFile, channel); + assertThat(inHeader.getTranslogUUID(), equalTo(translogUUID)); + assertThat(inHeader.getPrimaryTerm(), equalTo(TranslogHeader.UNKNOWN_PRIMARY_TERM)); + assertThat(inHeader.sizeInBytes(), equalTo((int)channel.position())); + } + expectThrows(TranslogCorruptedException.class, () -> { + try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.READ)) { + TranslogHeader.read(UUIDs.randomBase64UUID(), translogFile, channel); + } + }); + } + + static void writeLegacyTranslogHeader(FileChannel channel, String translogUUID) throws IOException { + final OutputStreamStreamOutput out = new OutputStreamStreamOutput(Channels.newOutputStream(channel)); + CodecUtil.writeHeader(new OutputStreamDataOutput(out), TranslogHeader.TRANSLOG_CODEC, TranslogHeader.VERSION_CHECKPOINTS); + final BytesRef uuid = new BytesRef(translogUUID); + out.writeInt(uuid.length); + out.writeBytes(uuid.bytes, uuid.offset, uuid.length); + channel.force(true); + assertThat(channel.position(), equalTo(43L)); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 2317d8fb0d8bf..d8d88146d5108 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -106,8 +106,10 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongSupplier; import java.util.stream.Collectors; import java.util.stream.LongStream; +import java.util.stream.Stream; import static org.elasticsearch.common.util.BigArrays.NON_RECYCLING_INSTANCE; import static org.elasticsearch.index.translog.SnapshotMatchers.containsOperationsInAnyOrder; @@ -131,6 +133,8 @@ public class TranslogTests extends ESTestCase { protected Translog translog; private AtomicLong globalCheckpoint; protected Path translogDir; + // A default primary term is used by translog instances created in this test. + private final AtomicLong primaryTerm = new AtomicLong(); @Override protected void afterIfSuccessful() throws Exception { @@ -153,12 +157,12 @@ protected Translog createTranslog(TranslogConfig config) throws IOException { String translogUUID = Translog.createEmptyTranslog(config.getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId); return new Translog(config, translogUUID, createTranslogDeletionPolicy(config.getIndexSettings()), - () -> SequenceNumbers.NO_OPS_PERFORMED); + () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get); } protected Translog openTranslog(TranslogConfig config, String translogUUID) throws IOException { return new Translog(config, translogUUID, createTranslogDeletionPolicy(config.getIndexSettings()), - () -> SequenceNumbers.NO_OPS_PERFORMED); + () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get); } @@ -187,6 +191,7 @@ private void commit(Translog translog, long genToRetain, long genToCommit) throw @Before public void setUp() throws Exception { super.setUp(); + primaryTerm.set(randomLongBetween(1, Integer.MAX_VALUE)); // if a previous test failed we clean up things here translogDir = createTempDir(); translog = create(translogDir); @@ -208,7 +213,7 @@ private Translog create(Path path) throws IOException { final TranslogConfig translogConfig = getTranslogConfig(path); final TranslogDeletionPolicy deletionPolicy = createTranslogDeletionPolicy(translogConfig.getIndexSettings()); final String translogUUID = Translog.createEmptyTranslog(path, SequenceNumbers.NO_OPS_PERFORMED, shardId); - return new Translog(translogConfig, translogUUID, deletionPolicy, () -> globalCheckpoint.get()); + return new Translog(translogConfig, translogUUID, deletionPolicy, () -> globalCheckpoint.get(), primaryTerm::get); } private TranslogConfig getTranslogConfig(final Path path) { @@ -302,22 +307,22 @@ public void testSimpleOperations() throws IOException { assertThat(snapshot, SnapshotMatchers.size(0)); } - addToTranslogAndList(translog, ops, new Translog.Index("test", "1", 0, new byte[]{1})); + addToTranslogAndList(translog, ops, new Translog.Index("test", "1", 0, primaryTerm.get(), new byte[]{1})); try (Translog.Snapshot snapshot = translog.newSnapshot()) { assertThat(snapshot, SnapshotMatchers.equalsTo(ops)); assertThat(snapshot.totalOperations(), equalTo(ops.size())); } - addToTranslogAndList(translog, ops, new Translog.Delete("test", "2", 1, newUid("2"))); + addToTranslogAndList(translog, ops, new Translog.Delete("test", "2", 1, primaryTerm.get(), newUid("2"))); try (Translog.Snapshot snapshot = translog.newSnapshot()) { assertThat(snapshot, SnapshotMatchers.equalsTo(ops)); assertThat(snapshot.totalOperations(), equalTo(ops.size())); } final long seqNo = randomNonNegativeLong(); - final long primaryTerm = randomNonNegativeLong(); final String reason = randomAlphaOfLength(16); - addToTranslogAndList(translog, ops, new Translog.NoOp(seqNo, primaryTerm, reason)); + final long noopTerm = randomLongBetween(1, primaryTerm.get()); + addToTranslogAndList(translog, ops, new Translog.NoOp(seqNo, noopTerm, reason)); try (Translog.Snapshot snapshot = translog.newSnapshot()) { @@ -332,7 +337,7 @@ public void testSimpleOperations() throws IOException { Translog.NoOp noOp = (Translog.NoOp) snapshot.next(); assertNotNull(noOp); assertThat(noOp.seqNo(), equalTo(seqNo)); - assertThat(noOp.primaryTerm(), equalTo(primaryTerm)); + assertThat(noOp.primaryTerm(), equalTo(noopTerm)); assertThat(noOp.reason(), equalTo(reason)); assertNull(snapshot.next()); @@ -400,35 +405,35 @@ public void testStats() throws IOException { final TranslogStats stats = stats(); assertThat(stats.estimatedNumberOfOperations(), equalTo(0)); } - assertThat((int) firstOperationPosition, greaterThan(CodecUtil.headerLength(TranslogWriter.TRANSLOG_CODEC))); - translog.add(new Translog.Index("test", "1", 0, new byte[]{1})); + assertThat((int) firstOperationPosition, greaterThan(CodecUtil.headerLength(TranslogHeader.TRANSLOG_CODEC))); + translog.add(new Translog.Index("test", "1", 0, primaryTerm.get(), new byte[]{1})); { final TranslogStats stats = stats(); assertThat(stats.estimatedNumberOfOperations(), equalTo(1)); - assertThat(stats.getTranslogSizeInBytes(), equalTo(140L)); + assertThat(stats.getTranslogSizeInBytes(), equalTo(164L)); assertThat(stats.getUncommittedOperations(), equalTo(1)); - assertThat(stats.getUncommittedSizeInBytes(), equalTo(140L)); + assertThat(stats.getUncommittedSizeInBytes(), equalTo(164L)); assertThat(stats.getEarliestLastModifiedAge(), greaterThan(1L)); } - translog.add(new Translog.Delete("test", "2", 1, newUid("2"))); + translog.add(new Translog.Delete("test", "2", 1, primaryTerm.get(), newUid("2"))); { final TranslogStats stats = stats(); assertThat(stats.estimatedNumberOfOperations(), equalTo(2)); - assertThat(stats.getTranslogSizeInBytes(), equalTo(189L)); + assertThat(stats.getTranslogSizeInBytes(), equalTo(213L)); assertThat(stats.getUncommittedOperations(), equalTo(2)); - assertThat(stats.getUncommittedSizeInBytes(), equalTo(189L)); + assertThat(stats.getUncommittedSizeInBytes(), equalTo(213L)); assertThat(stats.getEarliestLastModifiedAge(), greaterThan(1L)); } - translog.add(new Translog.Delete("test", "3", 2, newUid("3"))); + translog.add(new Translog.Delete("test", "3", 2, primaryTerm.get(), newUid("3"))); { final TranslogStats stats = stats(); assertThat(stats.estimatedNumberOfOperations(), equalTo(3)); - assertThat(stats.getTranslogSizeInBytes(), equalTo(238L)); + assertThat(stats.getTranslogSizeInBytes(), equalTo(262L)); assertThat(stats.getUncommittedOperations(), equalTo(3)); - assertThat(stats.getUncommittedSizeInBytes(), equalTo(238L)); + assertThat(stats.getUncommittedSizeInBytes(), equalTo(262L)); assertThat(stats.getEarliestLastModifiedAge(), greaterThan(1L)); } @@ -436,13 +441,13 @@ public void testStats() throws IOException { { final TranslogStats stats = stats(); assertThat(stats.estimatedNumberOfOperations(), equalTo(4)); - assertThat(stats.getTranslogSizeInBytes(), equalTo(280L)); + assertThat(stats.getTranslogSizeInBytes(), equalTo(304L)); assertThat(stats.getUncommittedOperations(), equalTo(4)); - assertThat(stats.getUncommittedSizeInBytes(), equalTo(280L)); + assertThat(stats.getUncommittedSizeInBytes(), equalTo(304L)); assertThat(stats.getEarliestLastModifiedAge(), greaterThan(1L)); } - final long expectedSizeInBytes = 323L; + final long expectedSizeInBytes = 359L; translog.rollGeneration(); { final TranslogStats stats = stats(); @@ -493,7 +498,7 @@ public void testUncommittedOperations() throws Exception { int uncommittedOps = 0; int operationsInLastGen = 0; for (int i = 0; i < operations; i++) { - translog.add(new Translog.Index("test", Integer.toString(i), i, new byte[]{1})); + translog.add(new Translog.Index("test", Integer.toString(i), i, primaryTerm.get(), new byte[]{1})); uncommittedOps++; operationsInLastGen++; if (rarely()) { @@ -562,7 +567,7 @@ public void testSnapshot() throws IOException { assertThat(snapshot, SnapshotMatchers.size(0)); } - addToTranslogAndList(translog, ops, new Translog.Index("test", "1", 0, new byte[]{1})); + addToTranslogAndList(translog, ops, new Translog.Index("test", "1", 0, primaryTerm.get(), new byte[]{1})); try (Translog.Snapshot snapshot = translog.newSnapshot()) { assertThat(snapshot, SnapshotMatchers.equalsTo(ops)); @@ -587,16 +592,16 @@ public void testSnapshotWithNewTranslog() throws IOException { toClose.add(snapshot); assertThat(snapshot, SnapshotMatchers.size(0)); - addToTranslogAndList(translog, ops, new Translog.Index("test", "1", 0, new byte[]{1})); + addToTranslogAndList(translog, ops, new Translog.Index("test", "1", 0, primaryTerm.get(), new byte[]{1})); Translog.Snapshot snapshot1 = translog.newSnapshot(); toClose.add(snapshot1); - addToTranslogAndList(translog, ops, new Translog.Index("test", "2", 1, new byte[]{2})); + addToTranslogAndList(translog, ops, new Translog.Index("test", "2", 1, primaryTerm.get(), new byte[]{2})); assertThat(snapshot1, SnapshotMatchers.equalsTo(ops.get(0))); translog.rollGeneration(); - addToTranslogAndList(translog, ops, new Translog.Index("test", "3", 2, new byte[]{3})); + addToTranslogAndList(translog, ops, new Translog.Index("test", "3", 2, primaryTerm.get(), new byte[]{3})); Translog.Snapshot snapshot2 = translog.newSnapshot(); toClose.add(snapshot2); @@ -610,7 +615,7 @@ public void testSnapshotWithNewTranslog() throws IOException { public void testSnapshotOnClosedTranslog() throws IOException { assertTrue(Files.exists(translogDir.resolve(Translog.getFilename(1)))); - translog.add(new Translog.Index("test", "1", 0, new byte[]{1})); + translog.add(new Translog.Index("test", "1", 0, primaryTerm.get(), new byte[]{1})); translog.close(); try { Translog.Snapshot snapshot = translog.newSnapshot(); @@ -730,7 +735,7 @@ public void testTranslogChecksums() throws Exception { int translogOperations = randomIntBetween(10, 100); for (int op = 0; op < translogOperations; op++) { String ascii = randomAlphaOfLengthBetween(1, 50); - locations.add(translog.add(new Translog.Index("test", "" + op, op, ascii.getBytes("UTF-8")))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), ascii.getBytes("UTF-8")))); } translog.sync(); @@ -757,7 +762,7 @@ public void testTruncatedTranslogs() throws Exception { int translogOperations = randomIntBetween(10, 100); for (int op = 0; op < translogOperations; op++) { String ascii = randomAlphaOfLengthBetween(1, 50); - locations.add(translog.add(new Translog.Index("test", "" + op, op, ascii.getBytes("UTF-8")))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), ascii.getBytes("UTF-8")))); } translog.sync(); @@ -821,7 +826,7 @@ private Term newUid(String uid) { public void testVerifyTranslogIsNotDeleted() throws IOException { assertFileIsPresent(translog, 1); - translog.add(new Translog.Index("test", "1", 0, new byte[]{1})); + translog.add(new Translog.Index("test", "1", 0, primaryTerm.get(), new byte[]{1})); try (Translog.Snapshot snapshot = translog.newSnapshot()) { assertThat(snapshot, SnapshotMatchers.size(1)); assertFileIsPresent(translog, 1); @@ -873,10 +878,10 @@ public void doRun() throws BrokenBarrierException, InterruptedException, IOExcep switch (type) { case CREATE: case INDEX: - op = new Translog.Index("type", "" + id, id, new byte[]{(byte) id}); + op = new Translog.Index("type", "" + id, id, primaryTerm.get(), new byte[]{(byte) id}); break; case DELETE: - op = new Translog.Delete("test", Long.toString(id), id, newUid(Long.toString(id))); + op = new Translog.Delete("test", Long.toString(id), id, primaryTerm.get(), newUid(Long.toString(id))); break; case NO_OP: op = new Translog.NoOp(id, 1, Long.toString(id)); @@ -1035,13 +1040,13 @@ public void testSyncUpTo() throws IOException { for (int op = 0; op < translogOperations; op++) { int seqNo = ++count; final Translog.Location location = - translog.add(new Translog.Index("test", "" + op, seqNo, Integer.toString(seqNo).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "" + op, seqNo, primaryTerm.get(), Integer.toString(seqNo).getBytes(Charset.forName("UTF-8")))); if (randomBoolean()) { assertTrue("at least one operation pending", translog.syncNeeded()); assertTrue("this operation has not been synced", translog.ensureSynced(location)); assertFalse("the last call to ensureSycned synced all previous ops", translog.syncNeeded()); // we are the last location so everything should be synced seqNo = ++count; - translog.add(new Translog.Index("test", "" + op, seqNo, Integer.toString(seqNo).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "" + op, seqNo, primaryTerm.get(), Integer.toString(seqNo).getBytes(Charset.forName("UTF-8")))); assertTrue("one pending operation", translog.syncNeeded()); assertFalse("this op has been synced before", translog.ensureSynced(location)); // not syncing now assertTrue("we only synced a previous operation yet", translog.syncNeeded()); @@ -1070,7 +1075,7 @@ public void testSyncUpToStream() throws IOException { rollAndCommit(translog); // do this first so that there is at least one pending tlog entry } final Translog.Location location = - translog.add(new Translog.Index("test", "" + op, op, Integer.toString(++count).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), Integer.toString(++count).getBytes(Charset.forName("UTF-8")))); locations.add(location); } Collections.shuffle(locations, random()); @@ -1098,7 +1103,7 @@ public void testLocationComparison() throws IOException { int count = 0; for (int op = 0; op < translogOperations; op++) { locations.add( - translog.add(new Translog.Index("test", "" + op, op, Integer.toString(++count).getBytes(Charset.forName("UTF-8"))))); + translog.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), Integer.toString(++count).getBytes(Charset.forName("UTF-8"))))); if (rarely() && translogOperations > op + 1) { rollAndCommit(translog); } @@ -1135,7 +1140,7 @@ public void testBasicCheckpoint() throws IOException { int lastSynced = -1; long lastSyncedGlobalCheckpoint = globalCheckpoint.get(); for (int op = 0; op < translogOperations; op++) { - locations.add(translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); if (randomBoolean()) { globalCheckpoint.set(globalCheckpoint.get() + randomIntBetween(1, 16)); } @@ -1146,8 +1151,8 @@ public void testBasicCheckpoint() throws IOException { } } assertEquals(translogOperations, translog.totalOperations()); - translog.add(new Translog.Index( - "test", "" + translogOperations, translogOperations, Integer.toString(translogOperations).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "" + translogOperations, translogOperations, primaryTerm.get(), + Integer.toString(translogOperations).getBytes(Charset.forName("UTF-8")))); final Checkpoint checkpoint = Checkpoint.read(translog.location().resolve(Translog.CHECKPOINT_FILE_NAME)); try (TranslogReader reader = translog.openReader(translog.location().resolve(Translog.getFilename(translog.currentFileGeneration())), checkpoint)) { @@ -1271,7 +1276,7 @@ public void testBasicRecovery() throws IOException { int minUncommittedOp = -1; final boolean commitOften = randomBoolean(); for (int op = 0; op < translogOperations; op++) { - locations.add(translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); final boolean commit = commitOften ? frequently() : rarely(); if (commit && op < translogOperations - 1) { rollAndCommit(translog); @@ -1292,7 +1297,7 @@ public void testBasicRecovery() throws IOException { assertNull(snapshot.next()); } } else { - translog = new Translog(config, translogGeneration.translogUUID, translog.getDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED); + translog = new Translog(config, translogGeneration.translogUUID, translog.getDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get); assertEquals("lastCommitted must be 1 less than current", translogGeneration.translogFileGeneration + 1, translog.currentFileGeneration()); assertFalse(translog.syncNeeded()); try (Translog.Snapshot snapshot = translog.newSnapshotFromGen(translogGeneration.translogFileGeneration)) { @@ -1314,7 +1319,7 @@ public void testRecoveryUncommitted() throws IOException { Translog.TranslogGeneration translogGeneration = null; final boolean sync = randomBoolean(); for (int op = 0; op < translogOperations; op++) { - locations.add(translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); if (op == prepareOp) { translogGeneration = translog.getGeneration(); translog.rollGeneration(); @@ -1331,7 +1336,7 @@ public void testRecoveryUncommitted() throws IOException { TranslogConfig config = translog.getConfig(); final String translogUUID = translog.getTranslogUUID(); final TranslogDeletionPolicy deletionPolicy = translog.getDeletionPolicy(); - try (Translog translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED)) { + try (Translog translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) { assertNotNull(translogGeneration); assertEquals("lastCommitted must be 2 less than current - we never finished the commit", translogGeneration.translogFileGeneration + 2, translog.currentFileGeneration()); assertFalse(translog.syncNeeded()); @@ -1345,7 +1350,7 @@ public void testRecoveryUncommitted() throws IOException { } } if (randomBoolean()) { // recover twice - try (Translog translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED)) { + try (Translog translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) { assertNotNull(translogGeneration); assertEquals("lastCommitted must be 3 less than current - we never finished the commit and run recovery twice", translogGeneration.translogFileGeneration + 3, translog.currentFileGeneration()); @@ -1370,7 +1375,7 @@ public void testRecoveryUncommittedFileExists() throws IOException { Translog.TranslogGeneration translogGeneration = null; final boolean sync = randomBoolean(); for (int op = 0; op < translogOperations; op++) { - locations.add(translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); if (op == prepareOp) { translogGeneration = translog.getGeneration(); translog.rollGeneration(); @@ -1391,7 +1396,7 @@ public void testRecoveryUncommittedFileExists() throws IOException { final String translogUUID = translog.getTranslogUUID(); final TranslogDeletionPolicy deletionPolicy = translog.getDeletionPolicy(); - try (Translog translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED)) { + try (Translog translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) { assertNotNull(translogGeneration); assertEquals("lastCommitted must be 2 less than current - we never finished the commit", translogGeneration.translogFileGeneration + 2, translog.currentFileGeneration()); assertFalse(translog.syncNeeded()); @@ -1406,7 +1411,7 @@ public void testRecoveryUncommittedFileExists() throws IOException { } if (randomBoolean()) { // recover twice - try (Translog translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED)) { + try (Translog translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) { assertNotNull(translogGeneration); assertEquals("lastCommitted must be 3 less than current - we never finished the commit and run recovery twice", translogGeneration.translogFileGeneration + 3, translog.currentFileGeneration()); @@ -1430,7 +1435,7 @@ public void testRecoveryUncommittedCorruptedCheckpoint() throws IOException { Translog.TranslogGeneration translogGeneration = null; final boolean sync = randomBoolean(); for (int op = 0; op < translogOperations; op++) { - locations.add(translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); if (op == prepareOp) { translogGeneration = translog.getGeneration(); translog.rollGeneration(); @@ -1449,15 +1454,15 @@ public void testRecoveryUncommittedCorruptedCheckpoint() throws IOException { Checkpoint.write(FileChannel::open, config.getTranslogPath().resolve(Translog.getCommitCheckpointFileName(read.generation)), corrupted, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW); final String translogUUID = translog.getTranslogUUID(); final TranslogDeletionPolicy deletionPolicy = translog.getDeletionPolicy(); - try (Translog ignored = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED)) { + try (Translog ignored = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) { fail("corrupted"); } catch (IllegalStateException ex) { - assertEquals("Checkpoint file translog-3.ckp already exists but has corrupted content expected: Checkpoint{offset=3123, " + + assertEquals("Checkpoint file translog-3.ckp already exists but has corrupted content expected: Checkpoint{offset=3135, " + "numOps=55, generation=3, minSeqNo=45, maxSeqNo=99, globalCheckpoint=-1, minTranslogGeneration=1} but got: Checkpoint{offset=0, numOps=0, " + "generation=0, minSeqNo=-1, maxSeqNo=-1, globalCheckpoint=-1, minTranslogGeneration=0}", ex.getMessage()); } Checkpoint.write(FileChannel::open, config.getTranslogPath().resolve(Translog.getCommitCheckpointFileName(read.generation)), read, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); - try (Translog translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED)) { + try (Translog translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) { assertNotNull(translogGeneration); assertEquals("lastCommitted must be 2 less than current - we never finished the commit", translogGeneration.translogFileGeneration + 2, translog.currentFileGeneration()); assertFalse(translog.syncNeeded()); @@ -1477,7 +1482,7 @@ public void testSnapshotFromStreamInput() throws IOException { List ops = new ArrayList<>(); int translogOperations = randomIntBetween(10, 100); for (int op = 0; op < translogOperations; op++) { - Translog.Index test = new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))); + Translog.Index test = new Translog.Index("test", "" + op, op, primaryTerm.get(), Integer.toString(op).getBytes(Charset.forName("UTF-8"))); ops.add(test); } Translog.writeOperations(out, ops); @@ -1492,8 +1497,8 @@ public void testLocationHashCodeEquals() throws IOException { int translogOperations = randomIntBetween(10, 100); try (Translog translog2 = create(createTempDir())) { for (int op = 0; op < translogOperations; op++) { - locations.add(translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); - locations2.add(translog2.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + locations2.add(translog2.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); } int iters = randomIntBetween(10, 100); for (int i = 0; i < iters; i++) { @@ -1519,7 +1524,7 @@ public void testOpenForeignTranslog() throws IOException { int translogOperations = randomIntBetween(1, 10); int firstUncommitted = 0; for (int op = 0; op < translogOperations; op++) { - locations.add(translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); + locations.add(translog.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), Integer.toString(op).getBytes(Charset.forName("UTF-8"))))); if (randomBoolean()) { rollAndCommit(translog); firstUncommitted = op + 1; @@ -1534,12 +1539,12 @@ public void testOpenForeignTranslog() throws IOException { final String foreignTranslog = randomRealisticUnicodeOfCodepointLengthBetween(1, translogGeneration.translogUUID.length()); try { - new Translog(config, foreignTranslog, createTranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED); + new Translog(config, foreignTranslog, createTranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get); fail("translog doesn't belong to this UUID"); } catch (TranslogCorruptedException ex) { } - this.translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED); + this.translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get); try (Translog.Snapshot snapshot = this.translog.newSnapshotFromGen(translogGeneration.translogFileGeneration)) { for (int i = firstUncommitted; i < translogOperations; i++) { Translog.Operation next = snapshot.next(); @@ -1551,10 +1556,10 @@ public void testOpenForeignTranslog() throws IOException { } public void testFailOnClosedWrite() throws IOException { - translog.add(new Translog.Index("test", "1", 0, Integer.toString(1).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "1", 0, primaryTerm.get(), Integer.toString(1).getBytes(Charset.forName("UTF-8")))); translog.close(); try { - translog.add(new Translog.Index("test", "1", 0, Integer.toString(1).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "1", 0, primaryTerm.get(), Integer.toString(1).getBytes(Charset.forName("UTF-8")))); fail("closed"); } catch (AlreadyClosedException ex) { // all is well @@ -1592,7 +1597,7 @@ public void testCloseConcurrently() throws Throwable { } } - private static class TranslogThread extends Thread { + private class TranslogThread extends Thread { private final CountDownLatch downLatch; private final int opsPerThread; private final int threadId; @@ -1623,19 +1628,19 @@ public void run() { case CREATE: case INDEX: op = new Translog.Index("test", threadId + "_" + opCount, seqNoGenerator.getAndIncrement(), - randomUnicodeOfLengthBetween(1, 20 * 1024).getBytes("UTF-8")); + primaryTerm.get(), randomUnicodeOfLengthBetween(1, 20 * 1024).getBytes("UTF-8")); break; case DELETE: op = new Translog.Delete( "test", threadId + "_" + opCount, new Term("_uid", threadId + "_" + opCount), seqNoGenerator.getAndIncrement(), - 0, + primaryTerm.get(), 1 + randomInt(100000), randomFrom(VersionType.values())); break; case NO_OP: - op = new Translog.NoOp(seqNoGenerator.getAndIncrement(), randomNonNegativeLong(), randomAlphaOfLength(16)); + op = new Translog.NoOp(seqNoGenerator.getAndIncrement(), primaryTerm.get(), randomAlphaOfLength(16)); break; default: throw new AssertionError("unsupported operation type [" + type + "]"); @@ -1670,7 +1675,7 @@ public void testFailFlush() throws IOException { while (failed == false) { try { locations.add(translog.add( - new Translog.Index("test", "" + opsSynced, opsSynced, Integer.toString(opsSynced).getBytes(Charset.forName("UTF-8"))))); + new Translog.Index("test", "" + opsSynced, opsSynced, primaryTerm.get(), Integer.toString(opsSynced).getBytes(Charset.forName("UTF-8"))))); translog.sync(); opsSynced++; } catch (MockDirectoryWrapper.FakeIOException ex) { @@ -1691,7 +1696,7 @@ public void testFailFlush() throws IOException { if (randomBoolean()) { try { locations.add(translog.add( - new Translog.Index("test", "" + opsSynced, opsSynced, Integer.toString(opsSynced).getBytes(Charset.forName("UTF-8"))))); + new Translog.Index("test", "" + opsSynced, opsSynced, primaryTerm.get(), Integer.toString(opsSynced).getBytes(Charset.forName("UTF-8"))))); fail("we are already closed"); } catch (AlreadyClosedException ex) { assertNotNull(ex.getCause()); @@ -1725,7 +1730,7 @@ public void testFailFlush() throws IOException { translog.close(); // we are closed final String translogUUID = translog.getTranslogUUID(); final TranslogDeletionPolicy deletionPolicy = translog.getDeletionPolicy(); - try (Translog tlog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED)) { + try (Translog tlog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) { assertEquals("lastCommitted must be 1 less than current", translogGeneration.translogFileGeneration + 1, tlog.currentFileGeneration()); assertFalse(tlog.syncNeeded()); @@ -1748,7 +1753,7 @@ public void testTranslogOpsCountIsCorrect() throws IOException { LineFileDocs lineFileDocs = new LineFileDocs(random()); // writes pretty big docs so we cross buffer borders regularly for (int opsAdded = 0; opsAdded < numOps; opsAdded++) { locations.add(translog.add( - new Translog.Index("test", "" + opsAdded, opsAdded, lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8"))))); + new Translog.Index("test", "" + opsAdded, opsAdded, primaryTerm.get(), lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8"))))); try (Translog.Snapshot snapshot = this.translog.newSnapshot()) { assertEquals(opsAdded + 1, snapshot.totalOperations()); for (int i = 0; i < opsAdded; i++) { @@ -1767,11 +1772,11 @@ public void testTragicEventCanBeAnyException() throws IOException { TranslogConfig config = getTranslogConfig(tempDir); Translog translog = getFailableTranslog(fail, config, false, true, null, createTranslogDeletionPolicy()); LineFileDocs lineFileDocs = new LineFileDocs(random()); // writes pretty big docs so we cross buffer boarders regularly - translog.add(new Translog.Index("test", "1", 0, lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "1", 0, primaryTerm.get(), lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8")))); fail.failAlways(); try { Translog.Location location = translog.add( - new Translog.Index("test", "2", 1, lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8")))); + new Translog.Index("test", "2", 1, primaryTerm.get(), lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8")))); if (randomBoolean()) { translog.ensureSynced(location); } else { @@ -1861,7 +1866,7 @@ protected void afterAdd() throws IOException { } } try (Translog tlog = - new Translog(config, translogUUID, createTranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED); + new Translog(config, translogUUID, createTranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get); Translog.Snapshot snapshot = tlog.newSnapshot()) { if (writtenOperations.size() != snapshot.totalOperations()) { for (int i = 0; i < threadCount; i++) { @@ -1888,7 +1893,7 @@ protected void afterAdd() throws IOException { public void testRecoveryFromAFutureGenerationCleansUp() throws IOException { int translogOperations = randomIntBetween(10, 100); for (int op = 0; op < translogOperations / 2; op++) { - translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), Integer.toString(op).getBytes(Charset.forName("UTF-8")))); if (rarely()) { translog.rollGeneration(); } @@ -1896,7 +1901,7 @@ public void testRecoveryFromAFutureGenerationCleansUp() throws IOException { translog.rollGeneration(); long comittedGeneration = randomLongBetween(2, translog.currentFileGeneration()); for (int op = translogOperations / 2; op < translogOperations; op++) { - translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), Integer.toString(op).getBytes(Charset.forName("UTF-8")))); if (rarely()) { translog.rollGeneration(); } @@ -1907,7 +1912,7 @@ public void testRecoveryFromAFutureGenerationCleansUp() throws IOException { final TranslogDeletionPolicy deletionPolicy = new TranslogDeletionPolicy(-1, -1); deletionPolicy.setTranslogGenerationOfLastCommit(randomLongBetween(comittedGeneration, Long.MAX_VALUE)); deletionPolicy.setMinTranslogGenerationForRecovery(comittedGeneration); - translog = new Translog(config, translog.getTranslogUUID(), deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED); + translog = new Translog(config, translog.getTranslogUUID(), deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get); assertThat(translog.getMinFileGeneration(), equalTo(1L)); // no trimming done yet, just recovered for (long gen = 1; gen < translog.currentFileGeneration(); gen++) { @@ -1938,7 +1943,7 @@ public void testRecoveryFromFailureOnTrimming() throws IOException { translogUUID = translog.getTranslogUUID(); int translogOperations = randomIntBetween(10, 100); for (int op = 0; op < translogOperations / 2; op++) { - translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), Integer.toString(op).getBytes(Charset.forName("UTF-8")))); if (rarely()) { translog.rollGeneration(); } @@ -1946,7 +1951,7 @@ public void testRecoveryFromFailureOnTrimming() throws IOException { translog.rollGeneration(); comittedGeneration = randomLongBetween(2, translog.currentFileGeneration()); for (int op = translogOperations / 2; op < translogOperations; op++) { - translog.add(new Translog.Index("test", "" + op, op, Integer.toString(op).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "" + op, op, primaryTerm.get(), Integer.toString(op).getBytes(Charset.forName("UTF-8")))); if (rarely()) { translog.rollGeneration(); } @@ -1963,7 +1968,7 @@ public void testRecoveryFromFailureOnTrimming() throws IOException { final TranslogDeletionPolicy deletionPolicy = new TranslogDeletionPolicy(-1, -1); deletionPolicy.setTranslogGenerationOfLastCommit(randomLongBetween(comittedGeneration, Long.MAX_VALUE)); deletionPolicy.setMinTranslogGenerationForRecovery(comittedGeneration); - try (Translog translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED)) { + try (Translog translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) { // we don't know when things broke exactly assertThat(translog.getMinFileGeneration(), greaterThanOrEqualTo(1L)); assertThat(translog.getMinFileGeneration(), lessThanOrEqualTo(comittedGeneration)); @@ -2029,7 +2034,7 @@ private Translog getFailableTranslog(final FailSwitch fail, final TranslogConfig translogUUID = Translog.createEmptyTranslog( config.getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId, channelFactory); } - return new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED) { + return new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get) { @Override ChannelFactory getChannelFactory() { return channelFactory; @@ -2137,10 +2142,10 @@ public void testFailWhileCreateWriteWithRecoveredTLogs() throws IOException { Path tempDir = createTempDir(); TranslogConfig config = getTranslogConfig(tempDir); Translog translog = createTranslog(config); - translog.add(new Translog.Index("test", "boom", 0, "boom".getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "boom", 0, primaryTerm.get(), "boom".getBytes(Charset.forName("UTF-8")))); translog.close(); try { - new Translog(config, translog.getTranslogUUID(), createTranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED) { + new Translog(config, translog.getTranslogUUID(), createTranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get) { @Override protected TranslogWriter createWriter(long fileGeneration, long initialMinTranslogGen, long initialGlobalCheckpoint) throws IOException { @@ -2155,7 +2160,7 @@ protected TranslogWriter createWriter(long fileGeneration, long initialMinTransl } public void testRecoverWithUnbackedNextGen() throws IOException { - translog.add(new Translog.Index("test", "" + 0, 0, Integer.toString(1).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "" + 0, 0, primaryTerm.get(), Integer.toString(1).getBytes(Charset.forName("UTF-8")))); translog.close(); TranslogConfig config = translog.getConfig(); @@ -2171,7 +2176,7 @@ public void testRecoverWithUnbackedNextGen() throws IOException { assertNotNull("operation 1 must be non-null", op); assertEquals("payload mismatch for operation 1", 1, Integer.parseInt(op.getSource().source.utf8ToString())); - tlog.add(new Translog.Index("test", "" + 1, 1, Integer.toString(2).getBytes(Charset.forName("UTF-8")))); + tlog.add(new Translog.Index("test", "" + 1, 1, primaryTerm.get(), Integer.toString(2).getBytes(Charset.forName("UTF-8")))); } try (Translog tlog = openTranslog(config, translog.getTranslogUUID()); @@ -2189,7 +2194,7 @@ public void testRecoverWithUnbackedNextGen() throws IOException { } public void testRecoverWithUnbackedNextGenInIllegalState() throws IOException { - translog.add(new Translog.Index("test", "" + 0, 0, Integer.toString(0).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "" + 0, 0, primaryTerm.get(), Integer.toString(0).getBytes(Charset.forName("UTF-8")))); translog.close(); TranslogConfig config = translog.getConfig(); Path ckp = config.getTranslogPath().resolve(Translog.CHECKPOINT_FILE_NAME); @@ -2198,7 +2203,7 @@ public void testRecoverWithUnbackedNextGenInIllegalState() throws IOException { Files.createFile(config.getTranslogPath().resolve("translog-" + (read.generation + 1) + ".tlog")); try { - Translog tlog = new Translog(config, translog.getTranslogUUID(), translog.getDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED); + Translog tlog = new Translog(config, translog.getTranslogUUID(), translog.getDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get); fail("file already exists?"); } catch (TranslogException ex) { // all is well @@ -2208,7 +2213,7 @@ public void testRecoverWithUnbackedNextGenInIllegalState() throws IOException { } public void testRecoverWithUnbackedNextGenAndFutureFile() throws IOException { - translog.add(new Translog.Index("test", "" + 0, 0, Integer.toString(0).getBytes(Charset.forName("UTF-8")))); + translog.add(new Translog.Index("test", "" + 0, 0, primaryTerm.get(), Integer.toString(0).getBytes(Charset.forName("UTF-8")))); translog.close(); TranslogConfig config = translog.getConfig(); final String translogUUID = translog.getTranslogUUID(); @@ -2220,7 +2225,7 @@ public void testRecoverWithUnbackedNextGenAndFutureFile() throws IOException { Files.createFile(config.getTranslogPath().resolve("translog-" + (read.generation + 1) + ".tlog")); // we add N+1 and N+2 to ensure we only delete the N+1 file and never jump ahead and wipe without the right condition Files.createFile(config.getTranslogPath().resolve("translog-" + (read.generation + 2) + ".tlog")); - try (Translog tlog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED)) { + try (Translog tlog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) { assertFalse(tlog.syncNeeded()); try (Translog.Snapshot snapshot = tlog.newSnapshot()) { for (int i = 0; i < 1; i++) { @@ -2229,11 +2234,11 @@ public void testRecoverWithUnbackedNextGenAndFutureFile() throws IOException { assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.utf8ToString())); } } - tlog.add(new Translog.Index("test", "" + 1, 1, Integer.toString(1).getBytes(Charset.forName("UTF-8")))); + tlog.add(new Translog.Index("test", "" + 1, 1, primaryTerm.get(), Integer.toString(1).getBytes(Charset.forName("UTF-8")))); } try { - Translog tlog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED); + Translog tlog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get); fail("file already exists?"); } catch (TranslogException ex) { // all is well @@ -2269,7 +2274,7 @@ public void testWithRandomException() throws IOException { LineFileDocs lineFileDocs = new LineFileDocs(random()); //writes pretty big docs so we cross buffer boarders regularly for (int opsAdded = 0; opsAdded < numOps; opsAdded++) { String doc = lineFileDocs.nextDoc().toString(); - failableTLog.add(new Translog.Index("test", "" + opsAdded, opsAdded, doc.getBytes(Charset.forName("UTF-8")))); + failableTLog.add(new Translog.Index("test", "" + opsAdded, opsAdded, primaryTerm.get(), doc.getBytes(Charset.forName("UTF-8")))); unsynced.add(doc); if (randomBoolean()) { failableTLog.sync(); @@ -2343,7 +2348,7 @@ public void testWithRandomException() throws IOException { // we never managed to successfully create a translog, make it generationUUID = Translog.createEmptyTranslog(config.getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId); } - try (Translog translog = new Translog(config, generationUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED); + try (Translog translog = new Translog(config, generationUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get); Translog.Snapshot snapshot = translog.newSnapshotFromGen(minGenForRecovery)) { assertEquals(syncedDocs.size(), snapshot.totalOperations()); for (int i = 0; i < syncedDocs.size(); i++) { @@ -2402,20 +2407,20 @@ public void testCheckpointOnDiskFull() throws IOException { * Tests that closing views after the translog is fine and we can reopen the translog */ public void testPendingDelete() throws IOException { - translog.add(new Translog.Index("test", "1", 0, new byte[]{1})); + translog.add(new Translog.Index("test", "1", 0, primaryTerm.get(), new byte[]{1})); translog.rollGeneration(); TranslogConfig config = translog.getConfig(); final String translogUUID = translog.getTranslogUUID(); final TranslogDeletionPolicy deletionPolicy = createTranslogDeletionPolicy(config.getIndexSettings()); translog.close(); - translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED); - translog.add(new Translog.Index("test", "2", 1, new byte[]{2})); + translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get); + translog.add(new Translog.Index("test", "2", 1, primaryTerm.get(), new byte[]{2})); translog.rollGeneration(); Closeable lock = translog.acquireRetentionLock(); - translog.add(new Translog.Index("test", "3", 2, new byte[]{3})); + translog.add(new Translog.Index("test", "3", 2, primaryTerm.get(), new byte[]{3})); translog.close(); IOUtils.close(lock); - translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED); + translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get); } public static Translog.Location randomTranslogLocation() { @@ -2495,20 +2500,31 @@ public void testRollGeneration() throws Exception { final int rolls = randomIntBetween(1, 16); int totalOperations = 0; int seqNo = 0; + final List primaryTerms = new ArrayList<>(); + primaryTerms.add(0L); // We always create an empty translog. + primaryTerms.add(primaryTerm.get()); for (int i = 0; i < rolls; i++) { final int operations = randomIntBetween(1, 128); for (int j = 0; j < operations; j++) { - translog.add(new Translog.NoOp(seqNo++, 0, "test")); + translog.add(new Translog.NoOp(seqNo++, primaryTerm.get(), "test")); totalOperations++; } try (ReleasableLock ignored = translog.writeLock.acquire()) { + if (randomBoolean()){ + primaryTerm.incrementAndGet(); + } translog.rollGeneration(); + primaryTerms.add(primaryTerm.get()); } assertThat(translog.currentFileGeneration(), equalTo(generation + i + 1)); + assertThat(translog.getCurrent().getPrimaryTerm(), equalTo(primaryTerm.get())); assertThat(translog.totalOperations(), equalTo(totalOperations)); } for (int i = 0; i <= rolls; i++) { assertFileIsPresent(translog, generation + i); + final List storedPrimaryTerms = Stream.concat(translog.getReaders().stream(), Stream.of(translog.getCurrent())) + .map(t -> t.getPrimaryTerm()).collect(Collectors.toList()); + assertThat(storedPrimaryTerms, equalTo(primaryTerms)); } long minGenForRecovery = randomLongBetween(generation, generation + rolls); commit(translog, minGenForRecovery, generation + rolls); @@ -2611,8 +2627,11 @@ public void testSimpleCommit() throws IOException { final int operations = randomIntBetween(1, 4096); long seqNo = 0; for (int i = 0; i < operations; i++) { - translog.add(new Translog.NoOp(seqNo++, 0, "test'")); + translog.add(new Translog.NoOp(seqNo++, primaryTerm.get(), "test'")); if (rarely()) { + if (rarely()) { + primaryTerm.incrementAndGet(); + } translog.rollGeneration(); } } @@ -2669,7 +2688,7 @@ public void testSnapshotReadOperationInReverse() throws Exception { for (int gen = 0; gen < generations; gen++) { final int operations = randomIntBetween(1, 100); for (int i = 0; i < operations; i++) { - Translog.Index op = new Translog.Index("doc", randomAlphaOfLength(10), seqNo.getAndIncrement(), new byte[]{1}); + Translog.Index op = new Translog.Index("doc", randomAlphaOfLength(10), seqNo.getAndIncrement(), primaryTerm.get(), new byte[]{1}); translog.add(op); views.peek().add(op); } @@ -2694,7 +2713,7 @@ public void testSnapshotDedupOperations() throws Exception { List batch = LongStream.rangeClosed(0, between(0, 500)).boxed().collect(Collectors.toList()); Randomness.shuffle(batch); for (Long seqNo : batch) { - Translog.Index op = new Translog.Index("doc", randomAlphaOfLength(10), seqNo, new byte[]{1}); + Translog.Index op = new Translog.Index("doc", randomAlphaOfLength(10), seqNo, primaryTerm.get(), new byte[]{1}); translog.add(op); latestOperations.put(op.seqNo(), op); } diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogVersionTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogVersionTests.java deleted file mode 100644 index d57373ebfe349..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogVersionTests.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.translog; - -import org.elasticsearch.index.seqno.SequenceNumbers; -import org.elasticsearch.test.ESTestCase; - -import java.io.IOException; -import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; - -/** - * Tests for reading old and new translog files - */ -public class TranslogVersionTests extends ESTestCase { - - private void checkFailsToOpen(String file, String expectedMessage) throws IOException { - Path translogFile = getDataPath(file); - assertThat("test file should exist", Files.exists(translogFile), equalTo(true)); - try { - openReader(translogFile, 0); - fail("should be able to open an old translog"); - } catch (IllegalStateException e) { - assertThat(e.getMessage(), containsString(expectedMessage)); - } - - } - - public void testV0LegacyTranslogVersion() throws Exception { - checkFailsToOpen("/org/elasticsearch/index/translog/translog-v0.binary", "pre-1.4 translog"); - } - - public void testV1ChecksummedTranslogVersion() throws Exception { - checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1.binary", "pre-2.0 translog"); - } - - public void testCorruptedTranslogs() throws Exception { - try { - Path translogFile = getDataPath("/org/elasticsearch/index/translog/translog-v1-corrupted-magic.binary"); - assertThat("test file should exist", Files.exists(translogFile), equalTo(true)); - openReader(translogFile, 0); - fail("should have thrown an exception about the header being corrupt"); - } catch (TranslogCorruptedException e) { - assertThat("translog corruption from header: " + e.getMessage(), - e.getMessage().contains("translog looks like version 1 or later, but has corrupted header"), equalTo(true)); - } - - try { - Path translogFile = getDataPath("/org/elasticsearch/index/translog/translog-invalid-first-byte.binary"); - assertThat("test file should exist", Files.exists(translogFile), equalTo(true)); - openReader(translogFile, 0); - fail("should have thrown an exception about the header being corrupt"); - } catch (TranslogCorruptedException e) { - assertThat("translog corruption from header: " + e.getMessage(), - e.getMessage().contains("Invalid first byte in translog file, got: 1, expected 0x00 or 0x3f"), equalTo(true)); - } - - checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1-corrupted-body.binary", "pre-2.0 translog"); - } - - public void testTruncatedTranslog() throws Exception { - checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1-truncated.binary", "pre-2.0 translog"); - } - - public TranslogReader openReader(final Path path, final long id) throws IOException { - try (FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) { - final long minSeqNo = SequenceNumbers.NO_OPS_PERFORMED; - final long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED; - final Checkpoint checkpoint = - new Checkpoint(Files.size(path), 1, id, minSeqNo, maxSeqNo, SequenceNumbers.UNASSIGNED_SEQ_NO, id); - return TranslogReader.open(channel, path, checkpoint, null); - } - } -} diff --git a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java index 934222f9e726a..8633a561ad8c5 100644 --- a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java +++ b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java @@ -240,7 +240,7 @@ public void testUnallocatedShardsDoesNotHang() throws InterruptedException { private void indexDoc(Engine engine, String id) throws IOException { final ParsedDocument doc = InternalEngineTests.createParsedDoc(id, null); - final Engine.IndexResult indexResult = engine.index(new Engine.Index(new Term("_id", Uid.encodeId(doc.id())), doc)); + final Engine.IndexResult indexResult = engine.index(new Engine.Index(new Term("_id", Uid.encodeId(doc.id())), 1L, doc)); assertThat(indexResult.getFailure(), nullValue()); } diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index d1dbaf6bc89fe..2489a4d2642a5 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -268,7 +268,7 @@ private Engine.Index getIndex(final String id) { final BytesReference source = new BytesArray(new byte[] { 1 }); final ParsedDocument doc = new ParsedDocument(versionField, seqID, id, type, null, Arrays.asList(document), source, XContentType.JSON, null); - return new Engine.Index(new Term("_uid", Uid.createUidAsBytes(doc.type(), doc.id())), doc); + return new Engine.Index(new Term("_uid", Uid.createUidAsBytes(doc.type(), doc.id())), randomNonNegativeLong(), doc); } public void testHandleCorruptedIndexOnSendSendFiles() throws Throwable { diff --git a/server/src/test/resources/org/elasticsearch/index/translog/translog-invalid-first-byte.binary b/server/src/test/resources/org/elasticsearch/index/translog/translog-invalid-first-byte.binary deleted file mode 100644 index 2eb76cf956f2e08a806b9b2579c2e818186cb164..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 91 zcmZQ%UDz=$xk+}R!YiGsZ_F3P%6(aN=Z}zkx5`O8BC@q#DHZ|W0b)BG^JVw S7+_>zV02#<7S;bB3K#&HAsz|< diff --git a/server/src/test/resources/org/elasticsearch/index/translog/translog-v0.binary b/server/src/test/resources/org/elasticsearch/index/translog/translog-v0.binary deleted file mode 100644 index 303bb2ef50e969469cbfebd09c26ea539b7b9084..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 91 zcmZQzUDz=$xk+}R!YiGsZ_F3P%6(aN=Z}zkx5`O8BC@q#DHZ|W0b)BG^JVw S7+_>zV02#<7S;bB3K#&G#vTa( diff --git a/server/src/test/resources/org/elasticsearch/index/translog/translog-v1-corrupted-body.binary b/server/src/test/resources/org/elasticsearch/index/translog/translog-v1-corrupted-body.binary deleted file mode 100644 index d74970f18b2877102abff14bd8a09be5bea0d26d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 417 zcmcD&o+HjtQk0lioRgoDf!9# z)k;bEDV0iA3QB2-`AW4w4G;j-%IN-Mr^uQAP{8o6M}!9`2QtsoFWsQRy(~M#xU#BH z*D2A%&)my75Z%0dEar*9jQh_}ZD2eNYF<=YgmG%HlSzn+QKUg=Sx$OUWo`yh=1FG2 z3}m>Z&{+sIuQJU)EjuwcG$_)@tDf!9# z)k;bEDV0iA3QB4D`AW4w4G;j-%IN-Mr^uQAP{8o6M}!9`2QtsoFWsQRy(~M#xU#BH z*D2A%&)my7kQnpCV8;Dts5UU31~o4#Ey6f8*vTZs#VFDsv@9pRs4_Q$DDxyUUDf!9# z)k;bEDV0iA3QB4D`AW4w4G;j-%IN-Mr^uQAP{8o6M}!9`2QtsoFWsQRy(~M#xU#BH z*D2A%&)my7kQnpCV8;Dts5UU31~o4#Ey6f8*vTZs#VFDsv@9pRs4_Q$DDxyUUDf!9# z)k;bEDV0iA3QB4D`AW4w4G;j-%IN-Mr^uQAP{8o6M}!9`2QtsoFWsQRy(~M#xU#BH z*D2A%&)my7kQnpCV8;Dts5UU31~o4#Ey6f8*vTZs#VFDsv@9pRs4_Q$DDxyUU SequenceNumbers.NO_OPS_PERFORMED); + return new Translog(translogConfig, translogUUID, createTranslogDeletionPolicy(INDEX_SETTINGS), + () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTermSupplier); } protected InternalEngine createEngine(Store store, Path translogPath) throws IOException { @@ -445,7 +448,7 @@ public void onFailedEngine(String reason, @Nullable Exception e) { new NoneCircuitBreakerService(), globalCheckpointSupplier == null ? new ReplicationTracker(shardId, allocationId.getId(), indexSettings, SequenceNumbers.NO_OPS_PERFORMED) : - globalCheckpointSupplier); + globalCheckpointSupplier, primaryTerm::get); return config; } @@ -471,12 +474,12 @@ protected Engine.Get newGet(boolean realtime, ParsedDocument doc) { } protected Engine.Index indexForDoc(ParsedDocument doc) { - return new Engine.Index(newUid(doc), doc); + return new Engine.Index(newUid(doc), primaryTerm.get(), doc); } protected Engine.Index replicaIndexForDoc(ParsedDocument doc, long version, long seqNo, boolean isRetry) { - return new Engine.Index(newUid(doc), doc, seqNo, 1, version, VersionType.EXTERNAL, + return new Engine.Index(newUid(doc), doc, seqNo, primaryTerm.get(), version, VersionType.EXTERNAL, Engine.Operation.Origin.REPLICA, System.nanoTime(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, isRetry); } From fe025b7aa4655a933715b772ec595c7dc417947a Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Fri, 23 Mar 2018 21:13:41 -0400 Subject: [PATCH 02/13] waits for primary promotion --- .../main/java/org/elasticsearch/index/translog/Translog.java | 2 +- .../index/replication/RecoveryDuringReplicationTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java index f7dc0ea88567c..d9df245ae2c81 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -1405,7 +1405,7 @@ public enum Durability { static void verifyChecksum(BufferedChecksumStreamInput in) throws IOException { // This absolutely must come first, or else reading the checksum becomes part of the checksum long expectedChecksum = in.getChecksum(); - long readChecksum = in.readInt() & 0xFFFF_FFFFL; + long readChecksum = Integer.toUnsignedLong(in.readInt()); if (readChecksum != expectedChecksum) { throw new TranslogCorruptedException("translog stream is corrupted, expected: 0x" + Long.toHexString(expectedChecksum) + ", got: 0x" + Long.toHexString(readChecksum)); diff --git a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java index 66e2a09750a2d..c7469f2432ad3 100644 --- a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java @@ -185,7 +185,7 @@ public void testRecoveryToReplicaThatReceivedExtraDocument() throws Exception { false, SourceToParse.source("index", "type", "replica", new BytesArray("{}"), XContentType.JSON), mapping -> {}); - shards.promoteReplicaToPrimary(promotedReplica); + shards.promoteReplicaToPrimary(promotedReplica).get(); oldPrimary.close("demoted", randomBoolean()); oldPrimary.store().close(); shards.removeReplica(remainingReplica); From 8aac0a1733a4f47ff29fde20754a6edbf9e0af63 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Sat, 24 Mar 2018 21:16:46 -0400 Subject: [PATCH 03/13] Read header when reading global checkpoint --- .../org/elasticsearch/index/translog/Translog.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java index d9df245ae2c81..87410e94c405d 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -274,10 +274,6 @@ private ArrayList recoverFromFiles(Checkpoint checkpoint) throws } TranslogReader openReader(Path path, Checkpoint checkpoint) throws IOException { - return openReader(path, checkpoint, translogUUID); - } - - private static TranslogReader openReader(Path path, Checkpoint checkpoint, String translogUUID) throws IOException { FileChannel channel = FileChannel.open(path, StandardOpenOption.READ); try { assert Translog.parseIdFromFileName(path) == checkpoint.generation : "expected generation: " + Translog.parseIdFromFileName(path) + " but got: " + checkpoint.generation; @@ -1688,10 +1684,10 @@ static Checkpoint readCheckpoint(final Path location) throws IOException { */ public static long readGlobalCheckpoint(final Path location, final String expectedTranslogUUID) throws IOException { final Checkpoint checkpoint = readCheckpoint(location); - // We need to open at least translog reader to validate the translogUUID. + // We need to open at least one translog header to validate the translogUUID. final Path translogFile = location.resolve(getFilename(checkpoint.generation)); - try (TranslogReader reader = openReader(translogFile, checkpoint, expectedTranslogUUID)) { - + try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.READ)) { + TranslogHeader.read(expectedTranslogUUID, translogFile, channel); } catch (TranslogCorruptedException ex) { throw ex; // just bubble up. } catch (Exception ex) { From 46d0c4f0513905779143a862bbecb04ec9109057 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 26 Mar 2018 13:47:28 -0400 Subject: [PATCH 04/13] Restore old versions check --- .../index/translog/TranslogHeader.java | 35 +++++++++++++++--- .../index/translog/TranslogHeaderTests.java | 29 ++++++++++++++- .../translog-invalid-first-byte.binary | Bin 0 -> 91 bytes .../index/translog/translog-v0.binary | Bin 0 -> 91 bytes .../translog-v1-corrupted-body.binary | Bin 0 -> 417 bytes .../translog-v1-corrupted-magic.binary | Bin 0 -> 417 bytes .../translog/translog-v1-truncated.binary | Bin 0 -> 383 bytes .../index/translog/translog-v1.binary | Bin 0 -> 417 bytes 8 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 server/src/test/resources/org/elasticsearch/index/translog/translog-invalid-first-byte.binary create mode 100644 server/src/test/resources/org/elasticsearch/index/translog/translog-v0.binary create mode 100644 server/src/test/resources/org/elasticsearch/index/translog/translog-v1-corrupted-body.binary create mode 100644 server/src/test/resources/org/elasticsearch/index/translog/translog-v1-corrupted-magic.binary create mode 100644 server/src/test/resources/org/elasticsearch/index/translog/translog-v1-truncated.binary create mode 100644 server/src/test/resources/org/elasticsearch/index/translog/translog-v1.binary diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java index 69a26684a97fb..ed9e83fc46f18 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java @@ -26,11 +26,11 @@ import org.apache.lucene.store.InputStreamDataInput; import org.apache.lucene.store.OutputStreamDataOutput; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.Channels; import org.elasticsearch.common.io.stream.InputStreamStreamInput; import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; import java.io.IOException; -import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.file.Path; @@ -40,7 +40,7 @@ final class TranslogHeader { public static final String TRANSLOG_CODEC = "translog"; - // public static final int VERSION_CHECKSUMS = 1; unsupported version + public static final int VERSION_CHECKSUMS = 1; // pre-2.0 - unsupported public static final int VERSION_CHECKPOINTS = 2; // added checkpoints public static final int VERSION_PRIMARY_TERM = 3; // added primary term public static final int CURRENT_VERSION = VERSION_PRIMARY_TERM; @@ -102,13 +102,17 @@ private static int headerSize(int version, int uuidLength) { static TranslogHeader read(final String translogUUID, final Path path, final FileChannel channel) throws IOException { // This input is intentionally not closed because closing it will close the FileChannel. final BufferedChecksumStreamInput in = - new BufferedChecksumStreamInput(new InputStreamStreamInput(Channels.newInputStream(channel), channel.size())); + new BufferedChecksumStreamInput(new InputStreamStreamInput(java.nio.channels.Channels.newInputStream(channel), channel.size())); final int version; try { - version = CodecUtil.checkHeader(new InputStreamDataInput(in), TRANSLOG_CODEC, VERSION_CHECKPOINTS, VERSION_PRIMARY_TERM); + version = CodecUtil.checkHeader(new InputStreamDataInput(in), TRANSLOG_CODEC, VERSION_CHECKSUMS, VERSION_PRIMARY_TERM); } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) { + tryReportOldVersionError(path, channel); throw new TranslogCorruptedException("Translog header corrupted. path:" + path, e); } + if (version == VERSION_CHECKSUMS) { + throw new IllegalStateException("pre-2.0 translog found [" + path + "]"); + } // Read the translogUUID final int uuidLen = in.readInt(); if (uuidLen > channel.size()) { @@ -141,13 +145,31 @@ static TranslogHeader read(final String translogUUID, final Path path, final Fil return new TranslogHeader(translogUUID, primaryTerm, headerSizeInBytes); } + private static void tryReportOldVersionError(final Path path, final FileChannel channel) throws IOException { + // Lucene's CodecUtil writes a magic number of 0x3FD76C17 with the header, in binary this looks like: + // binary: 0011 1111 1101 0111 0110 1100 0001 0111 + // hex : 3 f d 7 6 c 1 7 + // + // With version 0 of the translog, the first byte is the + // Operation.Type, which will always be between 0-4, so we know if + // we grab the first byte, it can be: + // 0x3f => Lucene's magic number, so we can assume it's version 1 or later + // 0x00 => version 0 of the translog + final byte b1 = Channels.readFromFileChannel(channel, 0, 1)[0]; + if (b1 == 0x3f) { //LUCENE_CODEC_HEADER_BYTE + throw new TranslogCorruptedException("translog looks like version 1 or later, but has corrupted header. path:" + path); + } else if (b1 == 0x00) { //UNVERSIONED_TRANSLOG_HEADER_BYTE + throw new IllegalStateException("pre-1.4 translog found [" + path + "]"); + } + } + /** * Writes this header with the latest format into the file channel */ - void write(FileChannel channel) throws IOException { + void write(final FileChannel channel) throws IOException { // This output is intentionally not closed because closing it will close the FileChannel. final BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput( - new OutputStreamStreamOutput(Channels.newOutputStream(channel))); + new OutputStreamStreamOutput(java.nio.channels.Channels.newOutputStream(channel))); CodecUtil.writeHeader(new OutputStreamDataOutput(out), TRANSLOG_CODEC, CURRENT_VERSION); // Write uuid final BytesRef uuid = new BytesRef(translogUUID); @@ -157,6 +179,7 @@ void write(FileChannel channel) throws IOException { out.writeLong(primaryTerm); // Checksum header out.writeInt((int) out.getChecksum()); + out.flush(); channel.force(true); assert channel.position() == headerSizeInBytes : "Header is not fully written; header size [" + headerSizeInBytes + "], channel position [" + channel.position() + "]"; diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java index 22c11e663cc3e..29f8f490ac02c 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java @@ -24,11 +24,13 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.nio.channels.Channels; import java.nio.channels.FileChannel; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -75,7 +77,7 @@ public void testHeaderWithoutPrimaryTerm() throws Exception { final long generation = randomNonNegativeLong(); final Path translogFile = createTempDir().resolve(Translog.getFilename(generation)); try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { - writeLegacyTranslogHeader(channel, translogUUID); + writeHeaderWithoutTerm(channel, translogUUID); assertThat((int)channel.position(), lessThan(TranslogHeader.defaultSizeInBytes(translogUUID))); } try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.READ)) { @@ -91,7 +93,7 @@ public void testHeaderWithoutPrimaryTerm() throws Exception { }); } - static void writeLegacyTranslogHeader(FileChannel channel, String translogUUID) throws IOException { + static void writeHeaderWithoutTerm(FileChannel channel, String translogUUID) throws IOException { final OutputStreamStreamOutput out = new OutputStreamStreamOutput(Channels.newOutputStream(channel)); CodecUtil.writeHeader(new OutputStreamDataOutput(out), TranslogHeader.TRANSLOG_CODEC, TranslogHeader.VERSION_CHECKPOINTS); final BytesRef uuid = new BytesRef(translogUUID); @@ -100,4 +102,27 @@ static void writeLegacyTranslogHeader(FileChannel channel, String translogUUID) channel.force(true); assertThat(channel.position(), equalTo(43L)); } + + public void testLegacyTranslogVersions() throws Exception { + checkFailsToOpen("/org/elasticsearch/index/translog/translog-v0.binary", IllegalStateException.class, "pre-1.4 translog"); + checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1.binary", IllegalStateException.class, "pre-2.0 translog"); + checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1-truncated.binary", IllegalStateException.class, "pre-2.0 translog"); + checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1-corrupted-magic.binary", + TranslogCorruptedException.class, "translog looks like version 1 or later, but has corrupted header"); + checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1-corrupted-body.binary", + IllegalStateException.class, "pre-2.0 translog"); + } + + private void checkFailsToOpen(String file, Class expectedErrorType, String expectedMessage) { + final Path translogFile = getDataPath(file); + assertThat("test file [" + translogFile + "] should exist", Files.exists(translogFile), equalTo(true)); + final E error = expectThrows(expectedErrorType, () -> { + final Checkpoint checkpoint = new Checkpoint(Files.size(translogFile), 1, 1, + SequenceNumbers.NO_OPS_PERFORMED, SequenceNumbers.NO_OPS_PERFORMED, SequenceNumbers.UNASSIGNED_SEQ_NO, 1); + try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.READ)) { + TranslogReader.open(channel, translogFile, checkpoint, null); + } + }); + assertThat(error.getMessage(), containsString(expectedMessage)); + } } diff --git a/server/src/test/resources/org/elasticsearch/index/translog/translog-invalid-first-byte.binary b/server/src/test/resources/org/elasticsearch/index/translog/translog-invalid-first-byte.binary new file mode 100644 index 0000000000000000000000000000000000000000..2eb76cf956f2e08a806b9b2579c2e818186cb164 GIT binary patch literal 91 zcmZQ%UDz=$xk+}R!YiGsZ_F3P%6(aN=Z}zkx5`O8BC@q#DHZ|W0b)BG^JVw S7+_>zV02#<7S;bB3K#&HAsz|< literal 0 HcmV?d00001 diff --git a/server/src/test/resources/org/elasticsearch/index/translog/translog-v0.binary b/server/src/test/resources/org/elasticsearch/index/translog/translog-v0.binary new file mode 100644 index 0000000000000000000000000000000000000000..303bb2ef50e969469cbfebd09c26ea539b7b9084 GIT binary patch literal 91 zcmZQzUDz=$xk+}R!YiGsZ_F3P%6(aN=Z}zkx5`O8BC@q#DHZ|W0b)BG^JVw S7+_>zV02#<7S;bB3K#&G#vTa( literal 0 HcmV?d00001 diff --git a/server/src/test/resources/org/elasticsearch/index/translog/translog-v1-corrupted-body.binary b/server/src/test/resources/org/elasticsearch/index/translog/translog-v1-corrupted-body.binary new file mode 100644 index 0000000000000000000000000000000000000000..d74970f18b2877102abff14bd8a09be5bea0d26d GIT binary patch literal 417 zcmcD&o+HjtQk0lioRgoDf!9# z)k;bEDV0iA3QB2-`AW4w4G;j-%IN-Mr^uQAP{8o6M}!9`2QtsoFWsQRy(~M#xU#BH z*D2A%&)my75Z%0dEar*9jQh_}ZD2eNYF<=YgmG%HlSzn+QKUg=Sx$OUWo`yh=1FG2 z3}m>Z&{+sIuQJU)EjuwcG$_)@tDf!9# z)k;bEDV0iA3QB4D`AW4w4G;j-%IN-Mr^uQAP{8o6M}!9`2QtsoFWsQRy(~M#xU#BH z*D2A%&)my7kQnpCV8;Dts5UU31~o4#Ey6f8*vTZs#VFDsv@9pRs4_Q$DDxyUUDf!9# z)k;bEDV0iA3QB4D`AW4w4G;j-%IN-Mr^uQAP{8o6M}!9`2QtsoFWsQRy(~M#xU#BH z*D2A%&)my7kQnpCV8;Dts5UU31~o4#Ey6f8*vTZs#VFDsv@9pRs4_Q$DDxyUUDf!9# z)k;bEDV0iA3QB4D`AW4w4G;j-%IN-Mr^uQAP{8o6M}!9`2QtsoFWsQRy(~M#xU#BH z*D2A%&)my7kQnpCV8;Dts5UU31~o4#Ey6f8*vTZs#VFDsv@9pRs4_Q$DDxyUU Date: Mon, 26 Mar 2018 14:11:56 -0400 Subject: [PATCH 05/13] add SuppressWarnings for eclipse and intellij --- .../java/org/elasticsearch/index/translog/TranslogHeader.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java index ed9e83fc46f18..abcc1803e7714 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java @@ -168,6 +168,7 @@ private static void tryReportOldVersionError(final Path path, final FileChannel */ void write(final FileChannel channel) throws IOException { // This output is intentionally not closed because closing it will close the FileChannel. + @SuppressWarnings({"IOResourceOpenedButNotSafelyClosed", "resource"}) final BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput( new OutputStreamStreamOutput(java.nio.channels.Channels.newOutputStream(channel))); CodecUtil.writeHeader(new OutputStreamDataOutput(out), TRANSLOG_CODEC, CURRENT_VERSION); From 4999d06ce21b3c917b7e71044a7a54e76a4c4744 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 26 Mar 2018 15:31:37 -0400 Subject: [PATCH 06/13] comments --- .../org/elasticsearch/index/translog/TranslogHeader.java | 9 ++++----- .../elasticsearch/index/engine/EngineDiskUtilsTests.java | 0 .../index/translog/TranslogHeaderTests.java | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) delete mode 100644 server/src/test/java/org/elasticsearch/index/engine/EngineDiskUtilsTests.java diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java index abcc1803e7714..afa71416c2dc6 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java @@ -150,15 +150,14 @@ private static void tryReportOldVersionError(final Path path, final FileChannel // binary: 0011 1111 1101 0111 0110 1100 0001 0111 // hex : 3 f d 7 6 c 1 7 // - // With version 0 of the translog, the first byte is the - // Operation.Type, which will always be between 0-4, so we know if - // we grab the first byte, it can be: + // With version 0 of the translog, the first byte is the Operation.Type, which will always be between 0-4, + // so we know if we grab the first byte, it can be: // 0x3f => Lucene's magic number, so we can assume it's version 1 or later // 0x00 => version 0 of the translog final byte b1 = Channels.readFromFileChannel(channel, 0, 1)[0]; - if (b1 == 0x3f) { //LUCENE_CODEC_HEADER_BYTE + if (b1 == 0x3f) { // LUCENE_CODEC_HEADER_BYTE throw new TranslogCorruptedException("translog looks like version 1 or later, but has corrupted header. path:" + path); - } else if (b1 == 0x00) { //UNVERSIONED_TRANSLOG_HEADER_BYTE + } else if (b1 == 0x00) { // UNVERSIONED_TRANSLOG_HEADER_BYTE throw new IllegalStateException("pre-1.4 translog found [" + path + "]"); } } diff --git a/server/src/test/java/org/elasticsearch/index/engine/EngineDiskUtilsTests.java b/server/src/test/java/org/elasticsearch/index/engine/EngineDiskUtilsTests.java deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java index 29f8f490ac02c..ba0c07c856bfd 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java @@ -118,7 +118,7 @@ private void checkFailsToOpen(String file, Class expect assertThat("test file [" + translogFile + "] should exist", Files.exists(translogFile), equalTo(true)); final E error = expectThrows(expectedErrorType, () -> { final Checkpoint checkpoint = new Checkpoint(Files.size(translogFile), 1, 1, - SequenceNumbers.NO_OPS_PERFORMED, SequenceNumbers.NO_OPS_PERFORMED, SequenceNumbers.UNASSIGNED_SEQ_NO, 1); + SequenceNumbers.NO_OPS_PERFORMED, SequenceNumbers.NO_OPS_PERFORMED, SequenceNumbers.NO_OPS_PERFORMED, 1); try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.READ)) { TranslogReader.open(channel, translogFile, checkpoint, null); } From 374eba8f2aad0191539069419fc614f343252402 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 28 Mar 2018 12:32:26 -0400 Subject: [PATCH 07/13] wip - add doc --- .../java/org/elasticsearch/index/translog/Translog.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java index 87410e94c405d..c1be210c6f085 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -134,10 +134,14 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC * translog file referenced by this generation. The translog creation will fail if this generation can't be opened. * * @param config the configuration of this translog - * @param translogUUID the translog uuid to open, null for a new translog + * @param translogUUID the translog uuid to open, null for a new translog * @param deletionPolicy an instance of {@link TranslogDeletionPolicy} that controls when a translog file can be safely * deleted * @param globalCheckpointSupplier a supplier for the global checkpoint + * @param primaryTermSupplier a supplier for the latest value of primary term of the owning index shard. The latest term value + * is retrieved and stored in the header of the newly rolled generation. It's guaranteed from outside + * that a new generation is rolled when the term is increased. This allows to us to check and reject + * translog operations whose terms are higher than the primary term stored in the translog header. */ public Translog( final TranslogConfig config, final String translogUUID, TranslogDeletionPolicy deletionPolicy, From 4b7c371aba0834c6059cabc4f4bf58ed662eca63 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 28 Mar 2018 18:07:53 -0400 Subject: [PATCH 08/13] javadocs --- .../elasticsearch/index/translog/Translog.java | 12 ++++++------ .../index/translog/TranslogHeader.java | 18 +++++++++++++----- .../index/engine/InternalEngineTests.java | 2 +- .../index/translog/TranslogHeaderTests.java | 2 +- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java index c1be210c6f085..49696c27d9a6f 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -109,7 +109,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC public static final String CHECKPOINT_FILE_NAME = "translog" + CHECKPOINT_SUFFIX; static final Pattern PARSE_STRICT_ID_PATTERN = Pattern.compile("^" + TRANSLOG_FILE_PREFIX + "(\\d+)(\\.tlog)$"); - public static final int DEFAULT_HEADER_SIZE_IN_BYTES = TranslogHeader.defaultSizeInBytes(UUIDs.randomBase64UUID()); + public static final int DEFAULT_HEADER_SIZE_IN_BYTES = TranslogHeader.headerSizeInBytes(UUIDs.randomBase64UUID()); // the list of translog readers is guaranteed to be in order of translog generation private final List readers = new ArrayList<>(); @@ -138,10 +138,10 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC * @param deletionPolicy an instance of {@link TranslogDeletionPolicy} that controls when a translog file can be safely * deleted * @param globalCheckpointSupplier a supplier for the global checkpoint - * @param primaryTermSupplier a supplier for the latest value of primary term of the owning index shard. The latest term value - * is retrieved and stored in the header of the newly rolled generation. It's guaranteed from outside - * that a new generation is rolled when the term is increased. This allows to us to check and reject - * translog operations whose terms are higher than the primary term stored in the translog header. + * @param primaryTermSupplier a supplier for the latest value of primary term of the owning index shard. The latest term value is + * examined and stored in the header whenever a new generation is rolled. It's guaranteed from outside + * that a new generation is rolled when the term is increased. This guarantee allows to us to validate + * and reject operation whose term is higher than the primary term stored in the translog header. */ public Translog( final TranslogConfig config, final String translogUUID, TranslogDeletionPolicy deletionPolicy, @@ -171,7 +171,7 @@ public Translog( // // For this to happen we must have already copied the translog.ckp file into translog-gen.ckp so we first check if that file exists // if not we don't even try to clean it up and wait until we fail creating it - assert Files.exists(nextTranslogFile) == false || Files.size(nextTranslogFile) <= TranslogHeader.defaultSizeInBytes(translogUUID) : "unexpected translog file: [" + nextTranslogFile + "]"; + assert Files.exists(nextTranslogFile) == false || Files.size(nextTranslogFile) <= TranslogHeader.headerSizeInBytes(translogUUID) : "unexpected translog file: [" + nextTranslogFile + "]"; if (Files.exists(currentCheckpointFile) // current checkpoint is already copied && Files.deleteIfExists(nextTranslogFile)) { // delete it and log a warning logger.warn("deleted previously created, but not yet committed, next generation [{}]. This can happen due to a tragic exception when creating a new generation", nextTranslogFile.getFileName()); diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java index afa71416c2dc6..0778769b5d642 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java @@ -51,8 +51,16 @@ final class TranslogHeader { private final long primaryTerm; private final int headerSizeInBytes; + /** + * Creates a new translog header with the given uuid and primary term. + * + * @param translogUUID this UUID is used to prevent accidental recovery from a transaction log that belongs to a + * different engine + * @param primaryTerm the primary term of the owning index shard when creating (eg. rolling) this translog file. + * All operations' terms in this translog file are enforced to be at most this term. + */ TranslogHeader(String translogUUID, long primaryTerm) { - this(translogUUID, primaryTerm, defaultSizeInBytes(translogUUID)); + this(translogUUID, primaryTerm, headerSizeInBytes(translogUUID)); assert primaryTerm >= 0 : "Primary term must be non-negative; term [" + primaryTerm + "]"; } @@ -82,11 +90,11 @@ public int sizeInBytes() { return headerSizeInBytes; } - static int defaultSizeInBytes(String translogUUID) { - return headerSize(CURRENT_VERSION, new BytesRef(translogUUID).length); + static int headerSizeInBytes(String translogUUID) { + return headerSizeInBytes(CURRENT_VERSION, new BytesRef(translogUUID).length); } - private static int headerSize(int version, int uuidLength) { + private static int headerSizeInBytes(int version, int uuidLength) { int size = CodecUtil.headerLength(TRANSLOG_CODEC); size += Integer.BYTES + uuidLength; // uuid if (version >= VERSION_PRIMARY_TERM) { @@ -139,7 +147,7 @@ static TranslogHeader read(final String translogUUID, final Path path, final Fil if (version >= VERSION_PRIMARY_TERM) { Translog.verifyChecksum(in); } - final int headerSizeInBytes = headerSize(version, uuid.length); + final int headerSizeInBytes = headerSizeInBytes(version, uuid.length); assert channel.position() == headerSizeInBytes : "Header is not fully read; header size [" + headerSizeInBytes + "], position [" + channel.position() + "]"; return new TranslogHeader(translogUUID, primaryTerm, headerSizeInBytes); diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 413ff96a8544d..6eef94222d07b 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -4567,7 +4567,7 @@ public void testTrackMaxSeqNoOfNonAppendOnlyOperations() throws Exception { if (randomBoolean()) { engine.index(indexForDoc(parsedDocument)); } else { - engine.delete(new Engine.Delete(parsedDocument.type(), parsedDocument.id(), newUid(parsedDocument.id()))); + engine.delete(new Engine.Delete(parsedDocument.type(), parsedDocument.id(), newUid(parsedDocument.id()), primaryTerm.get())); } } } diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java index ba0c07c856bfd..0dc404767de3c 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java @@ -78,7 +78,7 @@ public void testHeaderWithoutPrimaryTerm() throws Exception { final Path translogFile = createTempDir().resolve(Translog.getFilename(generation)); try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { writeHeaderWithoutTerm(channel, translogUUID); - assertThat((int)channel.position(), lessThan(TranslogHeader.defaultSizeInBytes(translogUUID))); + assertThat((int)channel.position(), lessThan(TranslogHeader.headerSizeInBytes(translogUUID))); } try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.READ)) { final TranslogHeader inHeader = TranslogHeader.read(translogUUID, translogFile, channel); From d50d7ed788c1d209f44ada77c2486a603c0b79aa Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 28 Mar 2018 18:14:59 -0400 Subject: [PATCH 09/13] use 0L for unknown term --- .../java/org/elasticsearch/index/translog/TranslogHeader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java index 0778769b5d642..0fde24d8bb4d5 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java @@ -45,7 +45,7 @@ final class TranslogHeader { public static final int VERSION_PRIMARY_TERM = 3; // added primary term public static final int CURRENT_VERSION = VERSION_PRIMARY_TERM; - public static final long UNKNOWN_PRIMARY_TERM = -1L; + public static final long UNKNOWN_PRIMARY_TERM = 0L; private final String translogUUID; private final long primaryTerm; From ad5a2123a37c4400b1d607669078d9cac09a32d3 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 28 Mar 2018 19:42:56 -0400 Subject: [PATCH 10/13] bootstrap primary term for an empty translog --- .../elasticsearch/index/shard/StoreRecovery.java | 10 ++++++---- .../elasticsearch/index/translog/Translog.java | 11 ++++++----- .../indices/recovery/RecoveryTarget.java | 4 ++-- .../index/engine/InternalEngineTests.java | 16 ++++++++-------- .../index/shard/RefreshListenersTests.java | 4 ++-- .../index/translog/TranslogTests.java | 9 ++++----- .../indices/recovery/RecoveryTests.java | 3 ++- .../index/engine/EngineTestCase.java | 7 ++++--- 8 files changed, 34 insertions(+), 30 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java index 3654aeba2bf8d..54718c545a44e 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java @@ -393,7 +393,8 @@ private void internalRecoverFromStore(IndexShard indexShard) throws IndexShardRe store.bootstrapNewHistory(); final SegmentInfos segmentInfos = store.readLastCommittedSegmentsInfo(); final long maxSeqNo = Long.parseLong(segmentInfos.userData.get(SequenceNumbers.MAX_SEQ_NO)); - final String translogUUID = Translog.createEmptyTranslog(indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId); + final String translogUUID = Translog.createEmptyTranslog( + indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId, indexShard.getPrimaryTerm()); store.associateIndexWithNewTranslog(translogUUID); } else if (indexShouldExists) { // since we recover from local, just fill the files and size @@ -407,8 +408,8 @@ private void internalRecoverFromStore(IndexShard indexShard) throws IndexShardRe } } else { store.createEmpty(); - final String translogUUID = Translog.createEmptyTranslog(indexShard.shardPath().resolveTranslog(), - SequenceNumbers.NO_OPS_PERFORMED, shardId); + final String translogUUID = Translog.createEmptyTranslog( + indexShard.shardPath().resolveTranslog(), SequenceNumbers.NO_OPS_PERFORMED, shardId, indexShard.getPrimaryTerm()); store.associateIndexWithNewTranslog(translogUUID); } indexShard.openEngineAndRecoverFromTranslog(); @@ -456,7 +457,8 @@ private void restore(final IndexShard indexShard, final Repository repository, f store.bootstrapNewHistory(); final SegmentInfos segmentInfos = store.readLastCommittedSegmentsInfo(); final long maxSeqNo = Long.parseLong(segmentInfos.userData.get(SequenceNumbers.MAX_SEQ_NO)); - final String translogUUID = Translog.createEmptyTranslog(indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId); + final String translogUUID = Translog.createEmptyTranslog( + indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId, indexShard.getPrimaryTerm()); store.associateIndexWithNewTranslog(translogUUID); assert indexShard.shardRouting.primary() : "only primary shards can recover from store"; indexShard.openEngineAndRecoverFromTranslog(); diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java index 49696c27d9a6f..b296bde37223d 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -1716,13 +1716,14 @@ List getReaders() { return readers; } - public static String createEmptyTranslog(final Path location, final long initialGlobalCheckpoint, final ShardId shardId) - throws IOException { + public static String createEmptyTranslog(final Path location, final long initialGlobalCheckpoint, + final ShardId shardId, final long primaryTerm) throws IOException { final ChannelFactory channelFactory = FileChannel::open; - return createEmptyTranslog(location, initialGlobalCheckpoint, shardId, channelFactory); + return createEmptyTranslog(location, initialGlobalCheckpoint, shardId, channelFactory, primaryTerm); } - static String createEmptyTranslog(Path location, long initialGlobalCheckpoint, ShardId shardId, ChannelFactory channelFactory) throws IOException { + static String createEmptyTranslog(Path location, long initialGlobalCheckpoint, ShardId shardId, + ChannelFactory channelFactory, long primaryTerm) throws IOException { IOUtils.rm(location); Files.createDirectories(location); final Checkpoint checkpoint = Checkpoint.emptyTranslogCheckpoint(0, 1, initialGlobalCheckpoint, 1); @@ -1732,7 +1733,7 @@ static String createEmptyTranslog(Path location, long initialGlobalCheckpoint, S final String translogUUID = UUIDs.randomBase64UUID(); TranslogWriter writer = TranslogWriter.create(shardId, translogUUID, 1, location.resolve(getFilename(1)), channelFactory, new ByteSizeValue(10), 1, initialGlobalCheckpoint, - () -> { throw new UnsupportedOperationException(); }, () -> { throw new UnsupportedOperationException(); }, 0L + () -> { throw new UnsupportedOperationException(); }, () -> { throw new UnsupportedOperationException(); }, primaryTerm ); writer.close(); return translogUUID; diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java index eb0db395a155f..244bb462df6ae 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java @@ -441,8 +441,8 @@ public void cleanFiles(int totalTranslogOps, Store.MetadataSnapshot sourceMetaDa store.ensureIndexHasHistoryUUID(); } // TODO: Assign the global checkpoint to the max_seqno of the safe commit if the index version >= 6.2 - final String translogUUID = - Translog.createEmptyTranslog(indexShard.shardPath().resolveTranslog(), SequenceNumbers.UNASSIGNED_SEQ_NO, shardId); + final String translogUUID = Translog.createEmptyTranslog( + indexShard.shardPath().resolveTranslog(), SequenceNumbers.UNASSIGNED_SEQ_NO, shardId, indexShard.getPrimaryTerm()); store.associateIndexWithNewTranslog(translogUUID); } catch (CorruptIndexException | IndexFormatTooNewException | IndexFormatTooOldException ex) { // this is a fatal exception at this stage. diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 6eef94222d07b..5af444345fab2 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -1147,7 +1147,7 @@ public void testSyncedFlushSurvivesEngineRestart() throws IOException { } if (randomBoolean()) { final String translogUUID = Translog.createEmptyTranslog(config.getTranslogConfig().getTranslogPath(), - SequenceNumbers.UNASSIGNED_SEQ_NO, shardId); + SequenceNumbers.UNASSIGNED_SEQ_NO, shardId, primaryTerm.get()); store.associateIndexWithNewTranslog(translogUUID); } engine = new InternalEngine(config); @@ -2369,7 +2369,7 @@ public void testCurrentTranslogIDisCommitted() throws IOException { { store.createEmpty(); final String translogUUID = - Translog.createEmptyTranslog(config.getTranslogConfig().getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId); + Translog.createEmptyTranslog(config.getTranslogConfig().getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get()); store.associateIndexWithNewTranslog(translogUUID); ParsedDocument doc = testParsedDocument(Integer.toString(0), null, testDocument(), new BytesArray("{}"), null); Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbers.UNASSIGNED_SEQ_NO, 0, @@ -2409,7 +2409,7 @@ public void testCurrentTranslogIDisCommitted() throws IOException { // open index with new tlog { final String translogUUID = - Translog.createEmptyTranslog(config.getTranslogConfig().getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId); + Translog.createEmptyTranslog(config.getTranslogConfig().getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get()); store.associateIndexWithNewTranslog(translogUUID); try (InternalEngine engine = new InternalEngine(config)) { Map userData = engine.getLastCommittedSegmentInfos().getUserData(); @@ -2454,7 +2454,7 @@ public void testMissingTranslog() throws IOException { // expected } // when a new translog is created it should be ok - final String translogUUID = Translog.createEmptyTranslog(primaryTranslogDir, SequenceNumbers.UNASSIGNED_SEQ_NO, shardId); + final String translogUUID = Translog.createEmptyTranslog(primaryTranslogDir, SequenceNumbers.UNASSIGNED_SEQ_NO, shardId, primaryTerm); store.associateIndexWithNewTranslog(translogUUID); EngineConfig config = config(defaultSettings, store, primaryTranslogDir, newMergePolicy(), null); engine = new InternalEngine(config); @@ -2519,7 +2519,7 @@ public void testTranslogCleanUpPostCommitCrash() throws Exception { final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); final LongSupplier globalCheckpointSupplier = () -> globalCheckpoint.get(); store.createEmpty(); - final String translogUUID = Translog.createEmptyTranslog(translogPath, globalCheckpoint.get(), shardId); + final String translogUUID = Translog.createEmptyTranslog(translogPath, globalCheckpoint.get(), shardId, primaryTerm.get()); store.associateIndexWithNewTranslog(translogUUID); try (InternalEngine engine = new InternalEngine(config(indexSettings, store, translogPath, newMergePolicy(), null, null, @@ -2681,7 +2681,7 @@ public void testRecoverFromForeignTranslog() throws IOException { engine.close(); final Path badTranslogLog = createTempDir(); - final String badUUID = Translog.createEmptyTranslog(badTranslogLog, SequenceNumbers.NO_OPS_PERFORMED, shardId); + final String badUUID = Translog.createEmptyTranslog(badTranslogLog, SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get()); Translog translog = new Translog( new TranslogConfig(shardId, badTranslogLog, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE), badUUID, createTranslogDeletionPolicy(INDEX_SETTINGS), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get); @@ -3311,7 +3311,7 @@ public void testEngineMaxTimestampIsInitialized() throws IOException { } try (Store store = createStore(newFSDirectory(storeDir))) { if (randomBoolean() || true) { - final String translogUUID = Translog.createEmptyTranslog(translogDir, SequenceNumbers.NO_OPS_PERFORMED, shardId); + final String translogUUID = Translog.createEmptyTranslog(translogDir, SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get()); store.associateIndexWithNewTranslog(translogUUID); } try (Engine engine = new InternalEngine(configSupplier.apply(store))) { @@ -4114,7 +4114,7 @@ public void testKeepTranslogAfterGlobalCheckpoint() throws Exception { store = createStore(); final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); store.createEmpty(); - final String translogUUID = Translog.createEmptyTranslog(translogPath, globalCheckpoint.get(), shardId); + final String translogUUID = Translog.createEmptyTranslog(translogPath, globalCheckpoint.get(), shardId, primaryTerm.get()); store.associateIndexWithNewTranslog(translogUUID); final EngineConfig engineConfig = config(indexSettings, store, translogPath, NoMergePolicy.INSTANCE, null, null, diff --git a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java index bf96dfb41cb83..aa98c6224084a 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java @@ -122,10 +122,10 @@ public void onFailedEngine(String reason, @Nullable Exception e) { } }; store.createEmpty(); + final long primaryTerm = randomNonNegativeLong(); final String translogUUID = - Translog.createEmptyTranslog(translogConfig.getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId); + Translog.createEmptyTranslog(translogConfig.getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm); store.associateIndexWithNewTranslog(translogUUID); - final long primaryTerm = randomNonNegativeLong(); EngineConfig config = new EngineConfig(shardId, allocationId, threadPool, indexSettings, null, store, newMergePolicy(), iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), eventListener, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index d8d88146d5108..47dbdd63f58f1 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -106,7 +106,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.LongSupplier; import java.util.stream.Collectors; import java.util.stream.LongStream; import java.util.stream.Stream; @@ -155,7 +154,7 @@ protected void afterIfSuccessful() throws Exception { protected Translog createTranslog(TranslogConfig config) throws IOException { String translogUUID = - Translog.createEmptyTranslog(config.getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId); + Translog.createEmptyTranslog(config.getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get()); return new Translog(config, translogUUID, createTranslogDeletionPolicy(config.getIndexSettings()), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get); } @@ -212,7 +211,7 @@ private Translog create(Path path) throws IOException { globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); final TranslogConfig translogConfig = getTranslogConfig(path); final TranslogDeletionPolicy deletionPolicy = createTranslogDeletionPolicy(translogConfig.getIndexSettings()); - final String translogUUID = Translog.createEmptyTranslog(path, SequenceNumbers.NO_OPS_PERFORMED, shardId); + final String translogUUID = Translog.createEmptyTranslog(path, SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get()); return new Translog(translogConfig, translogUUID, deletionPolicy, () -> globalCheckpoint.get(), primaryTerm::get); } @@ -2032,7 +2031,7 @@ private Translog getFailableTranslog(final FailSwitch fail, final TranslogConfig }; if (translogUUID == null) { translogUUID = Translog.createEmptyTranslog( - config.getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId, channelFactory); + config.getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId, channelFactory, primaryTerm.get()); } return new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get) { @Override @@ -2346,7 +2345,7 @@ public void testWithRandomException() throws IOException { deletionPolicy.setMinTranslogGenerationForRecovery(minGenForRecovery); if (generationUUID == null) { // we never managed to successfully create a translog, make it - generationUUID = Translog.createEmptyTranslog(config.getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId); + generationUUID = Translog.createEmptyTranslog(config.getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get()); } try (Translog translog = new Translog(config, generationUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get); Translog.Snapshot snapshot = translog.newSnapshotFromGen(minGenForRecovery)) { diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java index 49e557c3dde78..f46ab7ebbd603 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java @@ -200,7 +200,8 @@ public void testDifferentHistoryUUIDDisablesOPsRecovery() throws Exception { final String historyUUIDtoUse = UUIDs.randomBase64UUID(random()); if (randomBoolean()) { // create a new translog - translogUUIDtoUse = Translog.createEmptyTranslog(replica.shardPath().resolveTranslog(), flushedDocs, replica.shardId()); + translogUUIDtoUse = Translog.createEmptyTranslog(replica.shardPath().resolveTranslog(), flushedDocs, + replica.shardId(), replica.getPrimaryTerm()); translogGenToUse = 1; } else { translogUUIDtoUse = translogGeneration.translogUUID; diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index be46f0ce00411..6fd72b80534ec 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -269,7 +269,8 @@ protected Translog createTranslog(LongSupplier primaryTermSupplier) throws IOExc protected Translog createTranslog(Path translogPath, LongSupplier primaryTermSupplier) throws IOException { TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE); - String translogUUID = Translog.createEmptyTranslog(translogPath, SequenceNumbers.NO_OPS_PERFORMED, shardId); + String translogUUID = Translog.createEmptyTranslog(translogPath, SequenceNumbers.NO_OPS_PERFORMED, shardId, + primaryTermSupplier.getAsLong()); return new Translog(translogConfig, translogUUID, createTranslogDeletionPolicy(INDEX_SETTINGS), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTermSupplier); } @@ -369,8 +370,8 @@ private InternalEngine createEngine(@Nullable IndexWriterFactory indexWriterFact final Directory directory = store.directory(); if (Lucene.indexExists(directory) == false) { store.createEmpty(); - final String translogUuid = - Translog.createEmptyTranslog(config.getTranslogConfig().getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId); + final String translogUuid = Translog.createEmptyTranslog(config.getTranslogConfig().getTranslogPath(), + SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get()); store.associateIndexWithNewTranslog(translogUuid); } From 6ea0779463b134918d035209d5a852048216cc21 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 28 Mar 2018 21:26:17 -0400 Subject: [PATCH 11/13] fix bootstrap primary term test --- .../java/org/elasticsearch/index/translog/TranslogTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 47dbdd63f58f1..2d323d33c31f5 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -2500,7 +2500,7 @@ public void testRollGeneration() throws Exception { int totalOperations = 0; int seqNo = 0; final List primaryTerms = new ArrayList<>(); - primaryTerms.add(0L); // We always create an empty translog. + primaryTerms.add(primaryTerm.get()); // We always create an empty translog. primaryTerms.add(primaryTerm.get()); for (int i = 0; i < rolls; i++) { final int operations = randomIntBetween(1, 128); From fc8bc0614275ec9f4ffac13eae54753641c4722e Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 11 Apr 2018 11:10:26 -0400 Subject: [PATCH 12/13] style check --- .../action/resync/ResyncReplicationRequestTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/resync/ResyncReplicationRequestTests.java b/server/src/test/java/org/elasticsearch/action/resync/ResyncReplicationRequestTests.java index 3e21f6f6acfa4..d5ad3941a5e8f 100644 --- a/server/src/test/java/org/elasticsearch/action/resync/ResyncReplicationRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/resync/ResyncReplicationRequestTests.java @@ -37,7 +37,8 @@ public class ResyncReplicationRequestTests extends ESTestCase { public void testSerialization() throws IOException { final byte[] bytes = "{}".getBytes(Charset.forName("UTF-8")); - final Translog.Index index = new Translog.Index("type", "id", 0, randomNonNegativeLong(), Versions.MATCH_ANY, VersionType.INTERNAL, bytes, null, -1); + final Translog.Index index = new Translog.Index("type", "id", 0, randomNonNegativeLong(), + Versions.MATCH_ANY, VersionType.INTERNAL, bytes, null, -1); final ShardId shardId = new ShardId(new Index("index", "uuid"), 0); final ResyncReplicationRequest before = new ResyncReplicationRequest(shardId, new Translog.Operation[]{index}); From b61d459ff77161f9cc87ab6ca2acd8addb97a49f Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 12 Apr 2018 12:05:58 -0400 Subject: [PATCH 13/13] Use the primary term constant instead of 0L --- .../elasticsearch/index/translog/TruncateTranslogCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java b/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java index bf4dd762a65f2..b8bd93e05a6f8 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java @@ -213,7 +213,7 @@ static void writeEmptyCheckpoint(Path filename, int translogLength, long translo */ public static int writeEmptyTranslog(Path filename, String translogUUID) throws IOException { try (FileChannel fc = FileChannel.open(filename, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)) { - TranslogHeader header = new TranslogHeader(translogUUID, 0L); + TranslogHeader header = new TranslogHeader(translogUUID, TranslogHeader.UNKNOWN_PRIMARY_TERM); header.write(fc); return header.sizeInBytes(); }