Skip to content

Commit 5923ea5

Browse files
dnhatndnhatn
authored
CCR: Requires soft-deletes on the follower (#34725)
Since #34412 and #34474, a follower must have soft-deletes enabled to work correctly. This change requires soft-deletes on the follower. Relates #34412 Relates #34474
1 parent cadd673 commit 5923ea5

File tree

7 files changed

+38
-9
lines changed

7 files changed

+38
-9
lines changed

‎x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.elasticsearch.common.UUIDs;
2929
import org.elasticsearch.common.inject.Inject;
3030
import org.elasticsearch.common.settings.Settings;
31+
import org.elasticsearch.index.IndexSettings;
3132
import org.elasticsearch.license.LicenseUtils;
3233
import org.elasticsearch.threadpool.ThreadPool;
3334
import org.elasticsearch.transport.TransportService;
@@ -168,6 +169,7 @@ public ClusterState execute(final ClusterState currentState) throws Exception {
168169
settingsBuilder.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
169170
settingsBuilder.put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, followIndex);
170171
settingsBuilder.put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true);
172+
settingsBuilder.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
171173
imdBuilder.settings(settingsBuilder);
172174

173175
// Copy mappings from leader IMD to follow IMD

‎x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,9 @@ static void validate(
240240
if (leaderIndex.getSettings().getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) == false) {
241241
throw new IllegalArgumentException("leader index [" + leaderIndexName + "] does not have soft deletes enabled");
242242
}
243+
if (followIndex.getSettings().getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) == false) {
244+
throw new IllegalArgumentException("follower index [" + request.getFollowerIndex() + "] does not have soft deletes enabled");
245+
}
243246
if (leaderIndex.getNumberOfShards() != followIndex.getNumberOfShards()) {
244247
throw new IllegalArgumentException("leader index primary shards [" + leaderIndex.getNumberOfShards() +
245248
"] does not match with the number of shards of the follow index [" + followIndex.getNumberOfShards() + "]");
@@ -382,7 +385,6 @@ static String[] extractLeaderShardHistoryUUIDs(Map<String, String> ccrIndexMetaD
382385
whiteListedSettings.add(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING);
383386
whiteListedSettings.add(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING);
384387

385-
whiteListedSettings.add(IndexSettings.INDEX_SOFT_DELETES_SETTING);
386388
whiteListedSettings.add(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING);
387389

388390
WHITE_LISTED_SETTINGS = Collections.unmodifiableSet(whiteListedSettings);

‎x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngine.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ private static EngineConfig validateEngineConfig(final EngineConfig engineConfig
4949
if (CcrSettings.CCR_FOLLOWING_INDEX_SETTING.get(engineConfig.getIndexSettings().getSettings()) == false) {
5050
throw new IllegalArgumentException("a following engine can not be constructed for a non-following index");
5151
}
52+
if (engineConfig.getIndexSettings().isSoftDeleteEnabled() == false) {
53+
throw new IllegalArgumentException("a following engine requires soft deletes to be enabled");
54+
}
5255
return engineConfig;
5356
}
5457

‎x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -85,11 +85,20 @@ public void testValidation() throws IOException {
8585
Exception e = expectThrows(IllegalArgumentException.class, () -> validate(request, leaderIMD, followIMD, UUIDs, null));
8686
assertThat(e.getMessage(), equalTo("leader index [leader_cluster:index1] does not have soft deletes enabled"));
8787
}
88+
{
89+
// should fail because the follower index does not have soft deletes enabled
90+
IndexMetaData leaderIMD = createIMD("index1", 5, Settings.builder()
91+
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), null);
92+
IndexMetaData followIMD = createIMD("index2", 5, Settings.EMPTY, customMetaData);
93+
Exception e = expectThrows(IllegalArgumentException.class, () -> validate(request, leaderIMD, followIMD, UUIDs, null));
94+
assertThat(e.getMessage(), equalTo("follower index [index2] does not have soft deletes enabled"));
95+
}
8896
{
8997
// should fail because the number of primary shards between leader and follow index are not equal
9098
IndexMetaData leaderIMD = createIMD("index1", 5, Settings.builder()
9199
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), null);
92-
IndexMetaData followIMD = createIMD("index2", 4, Settings.EMPTY, customMetaData);
100+
IndexMetaData followIMD = createIMD("index2", 4,
101+
Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), customMetaData);
93102
Exception e = expectThrows(IllegalArgumentException.class, () -> validate(request, leaderIMD, followIMD, UUIDs, null));
94103
assertThat(e.getMessage(),
95104
equalTo("leader index primary shards [5] does not match with the number of shards of the follow index [4]"));
@@ -98,16 +107,17 @@ public void testValidation() throws IOException {
98107
// should fail, because leader index is closed
99108
IndexMetaData leaderIMD = createIMD("index1", State.CLOSE, "{}", 5, Settings.builder()
100109
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), null);
101-
IndexMetaData followIMD = createIMD("index2", State.OPEN, "{}", 5, Settings.builder()
102-
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), customMetaData);
110+
IndexMetaData followIMD = createIMD("index2", State.OPEN, "{}", 5,
111+
Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), customMetaData);
103112
Exception e = expectThrows(IllegalArgumentException.class, () -> validate(request, leaderIMD, followIMD, UUIDs, null));
104113
assertThat(e.getMessage(), equalTo("leader and follow index must be open"));
105114
}
106115
{
107116
// should fail, because index.xpack.ccr.following_index setting has not been enabled in leader index
108117
IndexMetaData leaderIMD = createIMD("index1", 1,
109118
Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), null);
110-
IndexMetaData followIMD = createIMD("index2", 1, Settings.EMPTY, customMetaData);
119+
IndexMetaData followIMD = createIMD("index2", 1,
120+
Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), customMetaData);
111121
MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY, "index2");
112122
mapperService.updateMapping(null, followIMD);
113123
Exception e = expectThrows(IllegalArgumentException.class,
@@ -120,7 +130,8 @@ public void testValidation() throws IOException {
120130
IndexMetaData leaderIMD = createIMD("index1", State.OPEN, "{\"properties\": {\"field\": {\"type\": \"keyword\"}}}", 5,
121131
Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), null);
122132
IndexMetaData followIMD = createIMD("index2", State.OPEN, "{\"properties\": {\"field\": {\"type\": \"text\"}}}", 5,
123-
Settings.builder().put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true).build(), customMetaData);
133+
Settings.builder().put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true)
134+
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).build(), customMetaData);
124135
MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY, "index2");
125136
mapperService.updateMapping(null, followIMD);
126137
Exception e = expectThrows(IllegalArgumentException.class, () -> validate(request, leaderIMD, followIMD, UUIDs, mapperService));
@@ -135,6 +146,7 @@ public void testValidation() throws IOException {
135146
.put("index.analysis.analyzer.my_analyzer.tokenizer", "whitespace").build(), null);
136147
IndexMetaData followIMD = createIMD("index2", State.OPEN, mapping, 5, Settings.builder()
137148
.put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true)
149+
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
138150
.put("index.analysis.analyzer.my_analyzer.type", "custom")
139151
.put("index.analysis.analyzer.my_analyzer.tokenizer", "standard").build(), customMetaData);
140152
Exception e = expectThrows(IllegalArgumentException.class, () -> validate(request, leaderIMD, followIMD, UUIDs, null));
@@ -144,8 +156,8 @@ public void testValidation() throws IOException {
144156
// should fail because the following index does not have the following_index settings
145157
IndexMetaData leaderIMD = createIMD("index1", 5,
146158
Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), null);
147-
Settings followingIndexSettings = randomBoolean() ? Settings.EMPTY :
148-
Settings.builder().put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), false).build();
159+
Settings followingIndexSettings = Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
160+
.put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), false).build();
149161
IndexMetaData followIMD = createIMD("index2", 5, followingIndexSettings, customMetaData);
150162
MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(),
151163
followingIndexSettings, "index2");
@@ -160,6 +172,7 @@ public void testValidation() throws IOException {
160172
IndexMetaData leaderIMD = createIMD("index1", 5, Settings.builder()
161173
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), null);
162174
IndexMetaData followIMD = createIMD("index2", 5, Settings.builder()
175+
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
163176
.put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true).build(), customMetaData);
164177
MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY, "index2");
165178
mapperService.updateMapping(null, followIMD);
@@ -174,6 +187,7 @@ public void testValidation() throws IOException {
174187
.put("index.analysis.analyzer.my_analyzer.tokenizer", "standard").build(), null);
175188
IndexMetaData followIMD = createIMD("index2", State.OPEN, mapping, 5, Settings.builder()
176189
.put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true)
190+
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
177191
.put("index.analysis.analyzer.my_analyzer.type", "custom")
178192
.put("index.analysis.analyzer.my_analyzer.tokenizer", "standard").build(), customMetaData);
179193
MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(),
@@ -191,6 +205,7 @@ public void testValidation() throws IOException {
191205
.put("index.analysis.analyzer.my_analyzer.tokenizer", "standard").build(), null);
192206
IndexMetaData followIMD = createIMD("index2", State.OPEN, mapping, 5, Settings.builder()
193207
.put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true)
208+
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
194209
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "10s")
195210
.put("index.analysis.analyzer.my_analyzer.type", "custom")
196211
.put("index.analysis.analyzer.my_analyzer.tokenizer", "standard").build(), customMetaData);

‎x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@ public class BulkShardOperationsTests extends IndexShardTestCase {
4141

4242
// test that we use the primary term on the follower when applying operations from the leader
4343
public void testPrimaryTermFromFollower() throws IOException {
44-
final Settings settings = Settings.builder().put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true).build();
44+
final Settings settings = Settings.builder().put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true)
45+
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).build();
4546
final IndexShard followerPrimary = newStartedShard(true, settings, new FollowingEngineFactory());
4647

4748
// we use this primary on the operations yet we expect the applied operations to have the primary term of the follower

‎x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowEngineIndexShardTests.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.elasticsearch.common.lease.Releasable;
1515
import org.elasticsearch.common.settings.Settings;
1616
import org.elasticsearch.common.xcontent.XContentType;
17+
import org.elasticsearch.index.IndexSettings;
1718
import org.elasticsearch.index.mapper.SourceToParse;
1819
import org.elasticsearch.index.shard.IndexShard;
1920
import org.elasticsearch.index.shard.IndexShardTestCase;
@@ -31,6 +32,7 @@ public class FollowEngineIndexShardTests extends IndexShardTestCase {
3132
public void testDoNotFillGaps() throws Exception {
3233
Settings settings = Settings.builder()
3334
.put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true)
35+
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
3436
.build();
3537
final IndexShard indexShard = newStartedShard(false, settings, new FollowingEngineFactory());
3638

‎x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ public void testOutOfOrderDocuments() throws IOException {
127127
.put("index.number_of_replicas", 0)
128128
.put("index.version.created", Version.CURRENT)
129129
.put("index.xpack.ccr.following_index", true)
130+
.put("index.soft_deletes.enabled", true)
130131
.build();
131132
final IndexMetaData indexMetaData = IndexMetaData.builder(index.getName()).settings(settings).build();
132133
final IndexSettings indexSettings = new IndexSettings(indexMetaData, settings);
@@ -152,6 +153,7 @@ public void runIndexTest(
152153
.put("index.number_of_replicas", 0)
153154
.put("index.version.created", Version.CURRENT)
154155
.put("index.xpack.ccr.following_index", true)
156+
.put("index.soft_deletes.enabled", true)
155157
.build();
156158
final IndexMetaData indexMetaData = IndexMetaData.builder(index.getName()).settings(settings).build();
157159
final IndexSettings indexSettings = new IndexSettings(indexMetaData, settings);
@@ -186,6 +188,7 @@ public void runDeleteTest(
186188
.put("index.number_of_replicas", 0)
187189
.put("index.version.created", Version.CURRENT)
188190
.put("index.xpack.ccr.following_index", true)
191+
.put("index.soft_deletes.enabled", true)
189192
.build();
190193
final IndexMetaData indexMetaData = IndexMetaData.builder(index.getName()).settings(settings).build();
191194
final IndexSettings indexSettings = new IndexSettings(indexMetaData, settings);
@@ -216,6 +219,7 @@ public void testDoNotFillSeqNoGaps() throws Exception {
216219
.put("index.number_of_replicas", 0)
217220
.put("index.version.created", Version.CURRENT)
218221
.put("index.xpack.ccr.following_index", true)
222+
.put("index.soft_deletes.enabled", true)
219223
.build();
220224
final IndexMetaData indexMetaData = IndexMetaData.builder(index.getName()).settings(settings).build();
221225
final IndexSettings indexSettings = new IndexSettings(indexMetaData, settings);

0 commit comments

Comments
 (0)