Include leases in ccr errmsg when ops no longer available (#45681)

The setting index.soft_deletes.retention.operations is no longer needed
nor recommended in CCR. We, therefore, should hint users about the
retention leases period setting instead when operations are no longer
available for replicating.

Author: dnhatn

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.MissingHistoryOperationsException;
+import org.elasticsearch.index.seqno.RetentionLease;
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardNotStartedException;
@@ -44,6 +45,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
@@ -529,8 +531,10 @@ static Translog.Operation[] getOperations(
                 }
             }
         } catch (MissingHistoryOperationsException e) {
-            String message = "Operations are no longer available for replicating. Maybe increase the retention setting [" +
-                IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey() + "]?";
+            final Collection<RetentionLease> retentionLeases = indexShard.getRetentionLeases().leases();
+            final String message = "Operations are no longer available for replicating. " +
+                "Existing retention leases [" + retentionLeases + "]; maybe increase the retention lease period setting " +
+                "[" + IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING.getKey() + "]?";
             // Make it easy to detect this error in ShardFollowNodeTask:
             // (adding a metadata header instead of introducing a new exception that extends ElasticsearchException)
             ResourceNotFoundException wrapper = new ResourceNotFoundException(message, e);

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesTests.java

@@ -17,6 +17,9 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.seqno.RetentionLease;
+import org.elasticsearch.index.seqno.RetentionLeaseActions;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -128,17 +131,22 @@ public void testMissingOperations() throws Exception {
         forceMergeRequest.maxNumSegments(1);
         client().admin().indices().forceMerge(forceMergeRequest).actionGet();
 
+        client().admin().indices().execute(RetentionLeaseActions.Add.INSTANCE, new RetentionLeaseActions.AddRequest(
+            new ShardId(resolveIndex("index"), 0), "test", RetentionLeaseActions.RETAIN_ALL, "ccr")).get();
+
         ShardStats shardStats = client().admin().indices().prepareStats("index").get().getIndex("index").getShards()[0];
         String historyUUID = shardStats.getCommitStats().getUserData().get(Engine.HISTORY_UUID_KEY);
+        Collection<RetentionLease> retentionLeases = shardStats.getRetentionLeaseStats().retentionLeases().leases();
         ShardChangesAction.Request request =  new ShardChangesAction.Request(shardStats.getShardRouting().shardId(), historyUUID);
         request.setFromSeqNo(0L);
         request.setMaxOperationCount(1);
 
         {
             ResourceNotFoundException e =
                 expectThrows(ResourceNotFoundException.class, () -> client().execute(ShardChangesAction.INSTANCE, request).actionGet());
-            assertThat(e.getMessage(), equalTo("Operations are no longer available for replicating. Maybe increase the retention setting " +
-                "[index.soft_deletes.retention.operations]?"));
+            assertThat(e.getMessage(), equalTo("Operations are no longer available for replicating. " +
+                "Existing retention leases [" + retentionLeases + "]; maybe increase the retention lease period setting " +
+                "[index.soft_deletes.retention_lease.period]?"));
 
             assertThat(e.getMetadataKeys().size(), equalTo(1));
             assertThat(e.getMetadata(Ccr.REQUESTED_OPS_MISSING_METADATA_KEY), notNullValue());
@@ -157,7 +165,8 @@ public void testMissingOperations() throws Exception {
 
             ResourceNotFoundException cause = (ResourceNotFoundException) e.getCause();
             assertThat(cause.getMessage(), equalTo("Operations are no longer available for replicating. " +
-                "Maybe increase the retention setting [index.soft_deletes.retention.operations]?"));
+                "Existing retention leases [" + retentionLeases + "]; maybe increase the retention lease period setting " +
+                "[index.soft_deletes.retention_lease.period]?"));
             assertThat(cause.getMetadataKeys().size(), equalTo(1));
             assertThat(cause.getMetadata(Ccr.REQUESTED_OPS_MISSING_METADATA_KEY), notNullValue());
             assertThat(cause.getMetadata(Ccr.REQUESTED_OPS_MISSING_METADATA_KEY), contains("0", "0"));