Skip to content

Commit 51bc320

Browse files
dnhatndnhatn
committed
TEST: Randomize soft-deletes settings (#31585)
This change allows us to test our system with/without soft-deletes enabled.
1 parent b3d6340 commit 51bc320

File tree

9 files changed

+74
-17
lines changed

9 files changed

+74
-17
lines changed

‎server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3849,13 +3849,15 @@ protected long doGenerateSeqNoForOperation(Operation operation) {
38493849
assertThat(noOp.seqNo(), equalTo((long) (maxSeqNo + 2)));
38503850
assertThat(noOp.primaryTerm(), equalTo(primaryTerm.get()));
38513851
assertThat(noOp.reason(), equalTo(reason));
3852-
MapperService mapperService = createMapperService("test");
3853-
List<Translog.Operation> operationsFromLucene = readAllOperationsInLucene(noOpEngine, mapperService);
3854-
assertThat(operationsFromLucene, hasSize(maxSeqNo + 2 - localCheckpoint)); // fills n gap and 2 manual noop.
3855-
for (int i = 0; i < operationsFromLucene.size(); i++) {
3856-
assertThat(operationsFromLucene.get(i), equalTo(new Translog.NoOp(localCheckpoint + 1 + i, primaryTerm.get(), "filling gaps")));
3852+
if (engine.engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
3853+
MapperService mapperService = createMapperService("test");
3854+
List<Translog.Operation> operationsFromLucene = readAllOperationsInLucene(noOpEngine, mapperService);
3855+
assertThat(operationsFromLucene, hasSize(maxSeqNo + 2 - localCheckpoint)); // fills n gap and 2 manual noop.
3856+
for (int i = 0; i < operationsFromLucene.size(); i++) {
3857+
assertThat(operationsFromLucene.get(i), equalTo(new Translog.NoOp(localCheckpoint + 1 + i, primaryTerm.get(), "filling gaps")));
3858+
}
3859+
assertConsistentHistoryBetweenTranslogAndLuceneIndex(noOpEngine, mapperService);
38573860
}
3858-
assertConsistentHistoryBetweenTranslogAndLuceneIndex(noOpEngine, mapperService);
38593861
} finally {
38603862
IOUtils.close(noOpEngine);
38613863
}
@@ -3912,8 +3914,10 @@ public void testRandomOperations() throws Exception {
39123914
engine.forceMerge(randomBoolean(), between(1, 10), randomBoolean(), false, false);
39133915
}
39143916
}
3915-
List<Translog.Operation> operations = readAllOperationsInLucene(engine, createMapperService("test"));
3916-
assertThat(operations, hasSize(numOps));
3917+
if (engine.engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
3918+
List<Translog.Operation> operations = readAllOperationsInLucene(engine, createMapperService("test"));
3919+
assertThat(operations, hasSize(numOps));
3920+
}
39173921
}
39183922

39193923
public void testMinGenerationForSeqNo() throws IOException, BrokenBarrierException, InterruptedException {
@@ -4854,9 +4858,15 @@ public void testLuceneHistoryOnReplica() throws Exception {
48544858
private void assertOperationHistoryInLucene(List<Engine.Operation> operations) throws IOException {
48554859
final MergePolicy keepSoftDeleteDocsMP = new SoftDeletesRetentionMergePolicy(
48564860
Lucene.SOFT_DELETE_FIELD, () -> new MatchAllDocsQuery(), engine.config().getMergePolicy());
4861+
Settings.Builder settings = Settings.builder()
4862+
.put(defaultSettings.getSettings())
4863+
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
4864+
.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), randomLongBetween(0, 10));
4865+
final IndexMetaData indexMetaData = IndexMetaData.builder(defaultSettings.getIndexMetaData()).settings(settings).build();
4866+
final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetaData);
48574867
Set<Long> expectedSeqNos = new HashSet<>();
48584868
try (Store store = createStore();
4859-
Engine engine = createEngine(config(defaultSettings, store, createTempDir(), keepSoftDeleteDocsMP, null))) {
4869+
Engine engine = createEngine(config(indexSettings, store, createTempDir(), keepSoftDeleteDocsMP, null))) {
48604870
for (Engine.Operation op : operations) {
48614871
if (op instanceof Engine.Index) {
48624872
Engine.IndexResult indexResult = engine.index((Engine.Index) op);

‎server/src/test/java/org/elasticsearch/index/engine/LuceneChangesSnapshotTests.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@
1919

2020
package org.elasticsearch.index.engine;
2121

22+
import org.elasticsearch.common.settings.Settings;
2223
import org.elasticsearch.core.internal.io.IOUtils;
24+
import org.elasticsearch.index.IndexSettings;
2325
import org.elasticsearch.index.VersionType;
2426
import org.elasticsearch.index.mapper.MapperService;
2527
import org.elasticsearch.index.mapper.ParsedDocument;
@@ -49,6 +51,13 @@ public void createMapper() throws Exception {
4951
mapperService = createMapperService("test");
5052
}
5153

54+
@Override
55+
protected Settings indexSettings() {
56+
return Settings.builder().put(super.indexSettings())
57+
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) // always enable soft-deletes
58+
.build();
59+
}
60+
5261
public void testBasics() throws Exception {
5362
long fromSeqNo = randomNonNegativeLong();
5463
long toSeqNo = randomLongBetween(fromSeqNo, Long.MAX_VALUE);

‎server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2387,7 +2387,8 @@ public void testRecoverFromLocalShard() throws IOException {
23872387
public void testDocStats() throws IOException {
23882388
IndexShard indexShard = null;
23892389
try {
2390-
indexShard = newStartedShard();
2390+
indexShard = newStartedShard(
2391+
Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), 0).build());
23912392
final long numDocs = randomIntBetween(2, 32); // at least two documents so we have docs to delete
23922393
// Delete at least numDocs/10 documents otherwise the number of deleted docs will be below 10%
23932394
// and forceMerge will refuse to expunge deletes

‎server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ public Settings indexSettings() {
118118
return Settings.builder().put(super.indexSettings())
119119
.put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)
120120
.put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), true)
121+
.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), 0)
121122
.build();
122123
}
123124

‎test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,20 @@ protected static void assertVisibleCount(Engine engine, int numDocs, boolean ref
155155
}
156156
}
157157

158+
protected Settings indexSettings() {
159+
// TODO randomize more settings
160+
return Settings.builder()
161+
.put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), "1h") // make sure this doesn't kick in on us
162+
.put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codecName)
163+
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
164+
.put(IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.getKey(),
165+
between(10, 10 * IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.get(Settings.EMPTY)))
166+
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
167+
.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(),
168+
randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000))
169+
.build();
170+
}
171+
158172
@Override
159173
@Before
160174
public void setUp() throws Exception {
@@ -169,13 +183,7 @@ public void setUp() throws Exception {
169183
} else {
170184
codecName = "default";
171185
}
172-
defaultSettings = IndexSettingsModule.newIndexSettings("test", Settings.builder()
173-
.put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), "1h") // make sure this doesn't kick in on us
174-
.put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codecName)
175-
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
176-
.put(IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.getKey(),
177-
between(10, 10 * IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.get(Settings.EMPTY)))
178-
.build()); // TODO randomize more settings
186+
defaultSettings = IndexSettingsModule.newIndexSettings("test", indexSettings());
179187
threadPool = new TestThreadPool(getClass().getName());
180188
store = createStore();
181189
storeReplica = createStore();

‎test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
import org.elasticsearch.common.settings.Settings;
6060
import org.elasticsearch.common.xcontent.XContentType;
6161
import org.elasticsearch.index.Index;
62+
import org.elasticsearch.index.IndexSettings;
6263
import org.elasticsearch.index.engine.EngineFactory;
6364
import org.elasticsearch.index.engine.InternalEngineFactory;
6465
import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction;
@@ -120,6 +121,9 @@ protected IndexMetaData buildIndexMetaData(int replicas, Settings indexSettings,
120121
Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
121122
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas)
122123
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
124+
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
125+
.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(),
126+
randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000))
123127
.put(indexSettings)
124128
.build();
125129
IndexMetaData.Builder metaData = IndexMetaData.builder(index.getName())

‎test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,9 @@ protected IndexShard newShard(
220220
Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
221221
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
222222
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
223+
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
224+
.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(),
225+
randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000))
223226
.put(settings)
224227
.build();
225228
IndexMetaData.Builder metaData = IndexMetaData.builder(shardRouting.getIndexName())
@@ -395,6 +398,14 @@ protected IndexShard newStartedShard() throws IOException {
395398
return newStartedShard(randomBoolean());
396399
}
397400

401+
/**
402+
* Creates a new empty shard and starts it
403+
* @param settings the settings to use for this shard
404+
*/
405+
protected IndexShard newStartedShard(Settings settings) throws IOException {
406+
return newStartedShard(randomBoolean(), settings, new InternalEngineFactory());
407+
}
408+
398409
/**
399410
* Creates a new empty shard and starts it.
400411
*

‎test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -725,6 +725,10 @@ public Settings indexSettings() {
725725
}
726726
// always default delayed allocation to 0 to make sure we have tests are not delayed
727727
builder.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), 0);
728+
builder.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
729+
if (randomBoolean()) {
730+
builder.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000));
731+
}
728732
return builder.build();
729733
}
730734

‎test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.elasticsearch.env.NodeEnvironment;
4343
import org.elasticsearch.index.Index;
4444
import org.elasticsearch.index.IndexService;
45+
import org.elasticsearch.index.IndexSettings;
4546
import org.elasticsearch.indices.IndicesService;
4647
import org.elasticsearch.node.MockNode;
4748
import org.elasticsearch.node.Node;
@@ -87,6 +88,14 @@ protected void startNode(long seed) throws Exception {
8788
.setOrder(0)
8889
.setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
8990
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get();
91+
client().admin().indices()
92+
.preparePutTemplate("random-soft-deletes-template")
93+
.setPatterns(Collections.singletonList("*"))
94+
.setOrder(0)
95+
.setSettings(Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
96+
.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(),
97+
randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000))
98+
).get();
9099
}
91100

92101
private static void stopNode() throws IOException {

0 commit comments

Comments
 (0)