Skip to content

Commit 0e54603

Browse files
martijnvgmartijnvg
committed
Removed static indices and repos and the scripts that create them.
Two tests were still using the static indices: * IndexFolderUpgraderTests#testUpgradeRealIndex() * InternalEngineTests#testUpgradeOldIndex() I removed these tests too, because these tests functionally overlap with the full-cluster-restart qa tests. Relates to #24939
1 parent 20b7258 commit 0e54603

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+0
-1160
lines changed

‎core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java‎

Lines changed: 0 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,7 @@
1919

2020
package org.elasticsearch.common.util;
2121

22-
import org.apache.lucene.util.CollectionUtil;
2322
import org.apache.lucene.util.LuceneTestCase;
24-
import org.apache.lucene.util.TestUtil;
2523
import org.elasticsearch.Version;
2624
import org.elasticsearch.cluster.metadata.IndexMetaData;
2725
import org.elasticsearch.cluster.routing.AllocationId;
@@ -32,32 +30,23 @@
3230
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
3331
import org.elasticsearch.env.Environment;
3432
import org.elasticsearch.env.NodeEnvironment;
35-
import org.elasticsearch.gateway.MetaDataStateFormat;
3633
import org.elasticsearch.index.Index;
3734
import org.elasticsearch.index.IndexSettings;
3835
import org.elasticsearch.index.shard.ShardId;
3936
import org.elasticsearch.index.shard.ShardPath;
4037
import org.elasticsearch.index.shard.ShardStateMetaData;
4138
import org.elasticsearch.test.ESTestCase;
42-
import org.elasticsearch.test.OldIndexUtils;
4339

4440
import java.io.BufferedWriter;
4541
import java.io.FileNotFoundException;
4642
import java.io.IOException;
47-
import java.io.InputStream;
48-
import java.net.URISyntaxException;
4943
import java.nio.charset.StandardCharsets;
50-
import java.nio.file.DirectoryStream;
5144
import java.nio.file.Files;
5245
import java.nio.file.Path;
53-
import java.util.ArrayList;
5446
import java.util.Arrays;
5547
import java.util.HashMap;
5648
import java.util.HashSet;
57-
import java.util.List;
58-
import java.util.Locale;
5949
import java.util.Map;
60-
import java.util.Set;
6150

6251
@LuceneTestCase.SuppressFileSystems("ExtrasFS")
6352
public class IndexFolderUpgraderTests extends ESTestCase {
@@ -181,68 +170,6 @@ public void testUpgradeIndices() throws IOException {
181170
}
182171
}
183172

184-
/**
185-
* Run upgrade on a real bwc index
186-
*/
187-
public void testUpgradeRealIndex() throws IOException, URISyntaxException {
188-
List<Path> indexes = new ArrayList<>();
189-
try (DirectoryStream<Path> stream = Files.newDirectoryStream(getBwcIndicesPath(), "index-*.zip")) {
190-
for (Path path : stream) {
191-
indexes.add(path);
192-
}
193-
}
194-
CollectionUtil.introSort(indexes, (o1, o2) -> o1.getFileName().compareTo(o2.getFileName()));
195-
final Path path = randomFrom(indexes);
196-
final String indexName = path.getFileName().toString().replace(".zip", "").toLowerCase(Locale.ROOT);
197-
try (NodeEnvironment nodeEnvironment = newNodeEnvironment()) {
198-
Path unzipDir = createTempDir();
199-
Path unzipDataDir = unzipDir.resolve("data");
200-
// decompress the index
201-
try (InputStream stream = Files.newInputStream(path)) {
202-
TestUtil.unzip(stream, unzipDir);
203-
}
204-
// check it is unique
205-
assertTrue(Files.exists(unzipDataDir));
206-
Path[] list = FileSystemUtils.files(unzipDataDir);
207-
if (list.length != 1) {
208-
throw new IllegalStateException("Backwards index must contain exactly one cluster but was " + list.length);
209-
}
210-
// the bwc scripts packs the indices under this path
211-
Path src = OldIndexUtils.getIndexDir(logger, indexName, path.getFileName().toString(), list[0]);
212-
assertTrue("[" + path + "] missing index dir: " + src.toString(), Files.exists(src));
213-
final Path indicesPath = randomFrom(nodeEnvironment.nodePaths()).indicesPath;
214-
logger.info("--> injecting index [{}] into [{}]", indexName, indicesPath);
215-
OldIndexUtils.copyIndex(logger, src, src.getFileName().toString(), indicesPath);
216-
IndexFolderUpgrader.upgradeIndicesIfNeeded(Settings.EMPTY, nodeEnvironment);
217-
218-
// ensure old index folder is deleted
219-
Set<String> indexFolders = nodeEnvironment.availableIndexFolders();
220-
assertEquals(indexFolders.size(), 1);
221-
222-
// ensure index metadata is moved
223-
IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY,
224-
nodeEnvironment.resolveIndexFolder(indexFolders.iterator().next()));
225-
assertNotNull(indexMetaData);
226-
Index index = indexMetaData.getIndex();
227-
assertEquals(index.getName(), indexName);
228-
229-
Set<ShardId> shardIds = nodeEnvironment.findAllShardIds(index);
230-
// ensure all shards are moved
231-
assertEquals(shardIds.size(), indexMetaData.getNumberOfShards());
232-
for (ShardId shardId : shardIds) {
233-
final ShardPath shardPath = ShardPath.loadShardPath(logger, nodeEnvironment, shardId,
234-
new IndexSettings(indexMetaData, Settings.EMPTY));
235-
final Path translog = shardPath.resolveTranslog();
236-
final Path idx = shardPath.resolveIndex();
237-
final Path state = shardPath.getShardStatePath().resolve(MetaDataStateFormat.STATE_DIR_NAME);
238-
assertTrue(shardPath.exists());
239-
assertTrue(Files.exists(translog));
240-
assertTrue(Files.exists(idx));
241-
assertTrue(Files.exists(state));
242-
}
243-
}
244-
}
245-
246173
public void testNeedsUpgrade() throws IOException {
247174
final Index index = new Index("foo", UUIDs.randomBase64UUID());
248175
IndexMetaData indexState = IndexMetaData.builder(index.getName())

‎core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java‎

Lines changed: 0 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@
6767
import org.apache.lucene.util.BytesRef;
6868
import org.apache.lucene.util.FixedBitSet;
6969
import org.apache.lucene.util.IOUtils;
70-
import org.apache.lucene.util.TestUtil;
7170
import org.elasticsearch.ElasticsearchException;
7271
import org.elasticsearch.Version;
7372
import org.elasticsearch.action.index.IndexRequest;
@@ -83,7 +82,6 @@
8382
import org.elasticsearch.common.bytes.BytesArray;
8483
import org.elasticsearch.common.bytes.BytesReference;
8584
import org.elasticsearch.common.collect.Tuple;
86-
import org.elasticsearch.common.io.FileSystemUtils;
8785
import org.elasticsearch.common.logging.Loggers;
8886
import org.elasticsearch.common.lucene.Lucene;
8987
import org.elasticsearch.common.lucene.uid.Versions;
@@ -120,7 +118,6 @@
120118
import org.elasticsearch.index.mapper.SeqNoFieldMapper;
121119
import org.elasticsearch.index.mapper.SourceFieldMapper;
122120
import org.elasticsearch.index.mapper.Uid;
123-
import org.elasticsearch.index.mapper.UidFieldMapper;
124121
import org.elasticsearch.index.seqno.SequenceNumbers;
125122
import org.elasticsearch.index.seqno.SequenceNumbersService;
126123
import org.elasticsearch.index.shard.IndexSearcherWrapper;
@@ -138,18 +135,15 @@
138135
import org.elasticsearch.test.DummyShardLock;
139136
import org.elasticsearch.test.ESTestCase;
140137
import org.elasticsearch.test.IndexSettingsModule;
141-
import org.elasticsearch.test.OldIndexUtils;
142138
import org.elasticsearch.threadpool.TestThreadPool;
143139
import org.elasticsearch.threadpool.ThreadPool;
144140
import org.hamcrest.MatcherAssert;
145141
import org.junit.After;
146142
import org.junit.Before;
147143

148144
import java.io.IOException;
149-
import java.io.InputStream;
150145
import java.io.UncheckedIOException;
151146
import java.nio.charset.Charset;
152-
import java.nio.file.DirectoryStream;
153147
import java.nio.file.Files;
154148
import java.nio.file.Path;
155149
import java.util.ArrayList;
@@ -161,7 +155,6 @@
161155
import java.util.HashSet;
162156
import java.util.LinkedHashMap;
163157
import java.util.List;
164-
import java.util.Locale;
165158
import java.util.Map;
166159
import java.util.Queue;
167160
import java.util.Set;
@@ -2592,93 +2585,6 @@ private Mapping dynamicUpdate() {
25922585
return new Mapping(Version.CURRENT, root, new MetadataFieldMapper[0], emptyMap());
25932586
}
25942587

2595-
public void testUpgradeOldIndex() throws IOException {
2596-
List<Path> indexes = new ArrayList<>();
2597-
try (DirectoryStream<Path> stream = Files.newDirectoryStream(getBwcIndicesPath(), "index-*.zip")) {
2598-
for (Path path : stream) {
2599-
indexes.add(path);
2600-
}
2601-
}
2602-
Collections.shuffle(indexes, random());
2603-
for (Path indexFile : indexes.subList(0, scaledRandomIntBetween(1, indexes.size() / 2))) {
2604-
final String indexName = indexFile.getFileName().toString().replace(".zip", "").toLowerCase(Locale.ROOT);
2605-
Path unzipDir = createTempDir();
2606-
Path unzipDataDir = unzipDir.resolve("data");
2607-
// decompress the index
2608-
try (InputStream stream = Files.newInputStream(indexFile)) {
2609-
TestUtil.unzip(stream, unzipDir);
2610-
}
2611-
// check it is unique
2612-
assertTrue(Files.exists(unzipDataDir));
2613-
Path[] list = filterExtraFSFiles(FileSystemUtils.files(unzipDataDir));
2614-
2615-
if (list.length != 1) {
2616-
throw new IllegalStateException("Backwards index must contain exactly one cluster but was " + list.length
2617-
+ " " + Arrays.toString(list));
2618-
}
2619-
2620-
// the bwc scripts packs the indices under this path
2621-
Path src = OldIndexUtils.getIndexDir(logger, indexName, indexFile.toString(), list[0]);
2622-
Path translog = src.resolve("0").resolve("translog");
2623-
assertTrue("[" + indexFile + "] missing translog dir: " + translog.toString(), Files.exists(translog));
2624-
Path[] tlogFiles = filterExtraFSFiles(FileSystemUtils.files(translog));
2625-
assertEquals(Arrays.toString(tlogFiles), tlogFiles.length, 2); // ckp & tlog
2626-
Path tlogFile = tlogFiles[0].getFileName().toString().endsWith("tlog") ? tlogFiles[0] : tlogFiles[1];
2627-
final long size = Files.size(tlogFile);
2628-
logger.debug("upgrading index {} file: {} size: {}", indexName, tlogFiles[0].getFileName(), size);
2629-
Directory directory = newFSDirectory(src.resolve("0").resolve("index"));
2630-
final IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, xContentRegistry(), src);
2631-
final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetaData);
2632-
final Store store = createStore(indexSettings, directory);
2633-
final int iters = randomIntBetween(0, 2);
2634-
int numDocs = -1;
2635-
for (int i = 0; i < iters; i++) { // make sure we can restart on an upgraded index
2636-
try (InternalEngine engine = createEngine(indexSettings, store, translog, newMergePolicy())) {
2637-
try (Searcher searcher = engine.acquireSearcher("test")) {
2638-
if (i > 0) {
2639-
assertEquals(numDocs, searcher.reader().numDocs());
2640-
}
2641-
TopDocs search = searcher.searcher().search(new MatchAllDocsQuery(), 1);
2642-
numDocs = searcher.reader().numDocs();
2643-
assertTrue(search.totalHits > 1);
2644-
}
2645-
CommitStats commitStats = engine.commitStats();
2646-
Map<String, String> userData = commitStats.getUserData();
2647-
assertTrue("user data doesn't contain uuid", userData.containsKey(Translog.TRANSLOG_UUID_KEY));
2648-
assertTrue("user data doesn't contain generation key", userData.containsKey(Translog.TRANSLOG_GENERATION_KEY));
2649-
assertFalse("user data contains legacy marker", userData.containsKey("translog_id"));
2650-
}
2651-
}
2652-
2653-
try (InternalEngine engine = createEngine(indexSettings, store, translog, newMergePolicy())) {
2654-
if (numDocs == -1) {
2655-
try (Searcher searcher = engine.acquireSearcher("test")) {
2656-
numDocs = searcher.reader().numDocs();
2657-
}
2658-
}
2659-
final int numExtraDocs = randomIntBetween(1, 10);
2660-
for (int i = 0; i < numExtraDocs; i++) {
2661-
ParsedDocument doc = testParsedDocument("extra" + Integer.toString(i), null, testDocument(), new BytesArray("{}"), null);
2662-
Term uid;
2663-
if (indexMetaData.getCreationVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
2664-
uid = new Term(IdFieldMapper.NAME, doc.id());
2665-
} else {
2666-
uid = new Term(UidFieldMapper.NAME, Uid.createUid(doc.type(), doc.id()));
2667-
}
2668-
Engine.Index firstIndexRequest = new Engine.Index(uid, doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
2669-
Engine.IndexResult indexResult = engine.index(firstIndexRequest);
2670-
assertThat(indexResult.getVersion(), equalTo(1L));
2671-
}
2672-
engine.refresh("test");
2673-
try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
2674-
TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + numExtraDocs));
2675-
assertThat(topDocs.totalHits, equalTo((long) numDocs + numExtraDocs));
2676-
}
2677-
}
2678-
IOUtils.close(store, directory);
2679-
}
2680-
}
2681-
26822588
private Path[] filterExtraFSFiles(Path[] files) {
26832589
List<Path> paths = new ArrayList<>();
26842590
for (Path p : files) {

‎core/src/test/resources/indices/bwc/index-5.0.0.zip‎

-198 KB
Binary file not shown.

‎core/src/test/resources/indices/bwc/index-5.0.1.zip‎

-356 KB
Binary file not shown.

‎core/src/test/resources/indices/bwc/index-5.0.2.zip‎

-367 KB
Binary file not shown.

‎core/src/test/resources/indices/bwc/index-5.1.1.zip‎

-227 KB
Binary file not shown.

‎core/src/test/resources/indices/bwc/index-5.1.2.zip‎

-478 KB
Binary file not shown.

‎core/src/test/resources/indices/bwc/index-5.2.0.zip‎

-435 KB
Binary file not shown.

‎core/src/test/resources/indices/bwc/index-5.2.1.zip‎

-489 KB
Binary file not shown.

‎core/src/test/resources/indices/bwc/index-5.2.2.zip‎

-492 KB
Binary file not shown.

0 commit comments

Comments
 (0)