change default indices.lifecycle.poll_interval to something sane (#32521)

This was originally set to a few seconds while prototyping things.
This interval is for the scheduled trigger of policies. Policies
have this extra trigger beyond just on cluster-state changes because
cluster-state changes may not be happeneing in a cluster for
whatever reason, and we need to continue making progress. Updating
this value to be larger is reasonable since not all operations
are expected to be completed in the span of seconds, but instead in
minutes and hours. 10 minutes is sane.

Author: talevy

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java

@@ -25,9 +25,8 @@ public class LifecycleSettings {
     public static final String LIFECYCLE_STEP_INFO = "index.lifecycle.step_info";
     public static final String LIFECYCLE_SKIP = "index.lifecycle.skip";
 
-    // NORELEASE: we should probably change the default to something other than three seconds for initial release
     public static final Setting<TimeValue> LIFECYCLE_POLL_INTERVAL_SETTING = Setting.positiveTimeSetting(LIFECYCLE_POLL_INTERVAL,
-        TimeValue.timeValueSeconds(3), Setting.Property.Dynamic, Setting.Property.NodeScope);
+        TimeValue.timeValueMinutes(10), Setting.Property.Dynamic, Setting.Property.NodeScope);
     public static final Setting<String> LIFECYCLE_NAME_SETTING = Setting.simpleString(LIFECYCLE_NAME,
         Setting.Property.Dynamic, Setting.Property.IndexScope, Setting.Property.InternalIndex);
     public static final Setting<String> LIFECYCLE_PHASE_SETTING = Setting.simpleString(LIFECYCLE_PHASE,

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleServiceTests.java

@@ -261,7 +261,7 @@ public void testSchedulerInitializationAndUpdate() {
         indexLifecycleService.clusterChanged(noChangeEvent);
         assertThat(indexLifecycleService.getScheduler().jobCount(), equalTo(1));
         assertThat(((TimeValueSchedule) indexLifecycleService.getScheduledJob().getSchedule()).getInterval(),
-                equalTo(TimeValue.timeValueSeconds(3)));
+                equalTo(LifecycleSettings.LIFECYCLE_POLL_INTERVAL_SETTING.getDefault(previousState.metaData().settings())));
         indexLifecycleService.applyClusterState(event);
         indexLifecycleService.clusterChanged(event);
         assertThat(indexLifecycleService.getScheduler().jobCount(), equalTo(1));

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.core.indexlifecycle.AllocateAction;
 import org.elasticsearch.xpack.core.indexlifecycle.DeleteAction;
@@ -47,9 +48,20 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
     private String policy;
 
     @Before
-    public void refreshIndex() {
+    public void refreshIndex() throws IOException {
         index = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
         policy = randomAlphaOfLength(5);
+        Request request = new Request("PUT", "/_cluster/settings");
+        XContentBuilder pollIntervalEntity = JsonXContent.contentBuilder();
+        pollIntervalEntity.startObject();
+        {
+            pollIntervalEntity.startObject("transient");
+            {
+                pollIntervalEntity.field(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s");
+            }pollIntervalEntity.endObject();
+        } pollIntervalEntity.endObject();
+        request.setJsonEntity(Strings.toString(pollIntervalEntity));
+        assertOK(adminClient().performRequest(request));
     }
 
     public static void updatePolicy(String indexName, String policy) throws IOException {