Use a private Directory for split / shrink

server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java

@@ -123,13 +123,17 @@ boolean recoverFromLocalShards(BiConsumer<String, MappingMetaData> mappingUpdate
             return executeRecovery(indexShard, () -> {
                 logger.debug("starting recovery from local shards {}", shards);
                 try {
-                    final Directory directory = indexShard.store().directory(); // don't close this directory!!
                     final Directory[] sources = shards.stream().map(LocalShardSnapshot::getSnapshotDirectory).toArray(Directory[]::new);
                     final long maxSeqNo = shards.stream().mapToLong(LocalShardSnapshot::maxSeqNo).max().getAsLong();
                     final long maxUnsafeAutoIdTimestamp =
                             shards.stream().mapToLong(LocalShardSnapshot::maxUnsafeAutoIdTimestamp).max().getAsLong();
-                    addIndices(indexShard.recoveryState().getIndex(), directory, indexSort, sources, maxSeqNo, maxUnsafeAutoIdTimestamp,
-                        indexShard.indexSettings().getIndexMetaData(), indexShard.shardId().id(), isSplit, hasNested);
+                    try (Directory directory = indexShard.store().createNewDirectory()) {
+                        // we create a new directory since we might delete old files we never write again
+                        // but this might cause issues on windows since Lucenes IW now checks for pending deletes before it
+                        // starts up and we might be still holding on to them on windows. for this reason we use a private directory here
+                        addIndices(indexShard.recoveryState().getIndex(), directory, indexSort, sources, maxSeqNo, maxUnsafeAutoIdTimestamp,
+                            indexShard.indexSettings().getIndexMetaData(), indexShard.shardId().id(), isSplit, hasNested);
+                    }
                     internalRecoverFromStore(indexShard);
                     // just trigger a merge to do housekeeping on the
                     // copied segments - we will also see them in stats etc.

server/src/main/java/org/elasticsearch/index/store/Store.java

@@ -147,6 +147,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
     private final ShardLock shardLock;
     private final OnClose onClose;
     private final SingleObjectCache<StoreStats> statsCache;
+    private final DirectoryService directoryService;
 
     private final AbstractRefCounted refCounter = new AbstractRefCounted("store") {
         @Override
@@ -165,6 +166,7 @@ public Store(ShardId shardId, IndexSettings indexSettings, DirectoryService dire
         super(shardId, indexSettings);
         final Settings settings = indexSettings.getSettings();
         this.directory = new StoreDirectory(directoryService.newDirectory(), Loggers.getLogger("index.store.deletes", settings, shardId));
+        this.directoryService = directoryService;
         this.shardLock = shardLock;
         this.onClose = onClose;
         final TimeValue refreshInterval = indexSettings.getValue(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING);
@@ -176,6 +178,14 @@ public Store(ShardId shardId, IndexSettings indexSettings, DirectoryService dire
         assert shardLock.getShardId().equals(shardId);
     }
 
+    /**
+     * Returns a new directory instance that is not maintained by this store. The caller is responsible for closing it.
+     */
+    public Directory createNewDirectory() throws IOException {
+        ensureOpen();
+        return directoryService.newDirectory();
+    }
+
     public Directory directory() {
         ensureOpen();
         return directory;

server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java

@@ -77,7 +77,6 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
-@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30416")
 public class ShrinkIndexIT extends ESIntegTestCase {
 
     @Override

server/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java

@@ -24,7 +24,6 @@
 import org.apache.lucene.search.SortedSetSelector;
 import org.apache.lucene.search.SortedSetSortField;
 import org.apache.lucene.search.join.ScoreMode;
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
@@ -81,7 +80,6 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 
-@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30416")
 public class SplitIndexIT extends ESIntegTestCase {
 
     @Override