Skip to content

Commit 54ad800

Browse files
brwebrwe
committed
[Test] move methods from bwc test to test package for use in plugins (#19738)
* [Test] move methods from bwc test to test package for use in other plugins
1 parent cbb6b82 commit 54ad800

File tree

3 files changed

+308
-122
lines changed

3 files changed

+308
-122
lines changed

‎core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java‎

Lines changed: 104 additions & 121 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,6 @@
2121

2222
import com.google.common.base.Predicate;
2323
import com.google.common.util.concurrent.ListenableFuture;
24-
25-
import org.apache.lucene.index.IndexWriter;
2624
import org.apache.lucene.util.LuceneTestCase;
2725
import org.apache.lucene.util.TestUtil;
2826
import org.elasticsearch.Version;
@@ -36,25 +34,28 @@
3634
import org.elasticsearch.action.search.SearchResponse;
3735
import org.elasticsearch.cluster.ClusterState;
3836
import org.elasticsearch.cluster.metadata.IndexMetaData;
37+
import org.elasticsearch.cluster.metadata.MetaData;
3938
import org.elasticsearch.common.io.FileSystemUtils;
40-
import org.elasticsearch.common.logging.ESLogger;
4139
import org.elasticsearch.common.settings.Settings;
4240
import org.elasticsearch.common.unit.TimeValue;
4341
import org.elasticsearch.common.util.MultiDataPathUpgrader;
42+
import org.elasticsearch.common.xcontent.XContentBuilder;
4443
import org.elasticsearch.common.xcontent.XContentHelper;
44+
import org.elasticsearch.common.xcontent.XContentParser;
45+
import org.elasticsearch.common.xcontent.XContentType;
4546
import org.elasticsearch.env.NodeEnvironment;
47+
import org.elasticsearch.gateway.MetaDataStateFormat;
4648
import org.elasticsearch.index.engine.EngineConfig;
4749
import org.elasticsearch.index.engine.Segment;
4850
import org.elasticsearch.index.mapper.string.StringFieldMapperPositionIncrementGapTests;
4951
import org.elasticsearch.index.query.QueryBuilders;
50-
import org.elasticsearch.index.shard.MergePolicyConfig;
51-
import org.elasticsearch.indices.recovery.RecoverySettings;
5252
import org.elasticsearch.search.SearchHit;
5353
import org.elasticsearch.search.aggregations.AggregationBuilders;
5454
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
5555
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
5656
import org.elasticsearch.search.sort.SortOrder;
5757
import org.elasticsearch.test.ESIntegTestCase;
58+
import org.elasticsearch.test.OldIndexUtils;
5859
import org.elasticsearch.test.VersionUtils;
5960
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
6061
import org.hamcrest.Matchers;
@@ -64,24 +65,17 @@
6465

6566
import java.io.IOException;
6667
import java.io.InputStream;
67-
import java.nio.file.DirectoryStream;
68-
import java.nio.file.FileVisitResult;
6968
import java.nio.file.Files;
7069
import java.nio.file.Path;
71-
import java.nio.file.SimpleFileVisitor;
72-
import java.nio.file.attribute.BasicFileAttributes;
73-
import java.util.ArrayList;
7470
import java.util.Collections;
7571
import java.util.List;
7672
import java.util.Locale;
7773
import java.util.Map;
7874
import java.util.SortedSet;
7975
import java.util.TreeSet;
8076

81-
import static org.elasticsearch.action.admin.indices.upgrade.UpgradeAssertions.assertNotUpgraded;
82-
import static org.elasticsearch.action.admin.indices.upgrade.UpgradeAssertions.assertUpgraded;
77+
import static org.elasticsearch.test.OldIndexUtils.assertUpgradeWorks;
8378
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
84-
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
8579
import static org.hamcrest.CoreMatchers.containsString;
8680
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
8781

@@ -96,22 +90,13 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
9690
List<String> unsupportedIndexes;
9791
static Path singleDataPath;
9892
static Path[] multiDataPath;
93+
private String singleDataPathNodeName;
94+
private String multiDataPathNodeName;
9995

10096
@Before
10197
public void initIndexesList() throws Exception {
102-
indexes = loadIndexesList("index");
103-
unsupportedIndexes = loadIndexesList("unsupported");
104-
}
105-
106-
private List<String> loadIndexesList(String prefix) throws IOException {
107-
List<String> indexes = new ArrayList<>();
108-
try (DirectoryStream<Path> stream = Files.newDirectoryStream(getBwcIndicesPath(), prefix + "-*.zip")) {
109-
for (Path path : stream) {
110-
indexes.add(path.getFileName().toString());
111-
}
112-
}
113-
Collections.sort(indexes);
114-
return indexes;
98+
indexes = OldIndexUtils.loadIndexesList("index", getBwcIndicesPath());
99+
unsupportedIndexes = OldIndexUtils.loadIndexesList("unsupported", getBwcIndicesPath());
115100
}
116101

117102
@AfterClass
@@ -122,10 +107,9 @@ public static void tearDownStatics() {
122107

123108
@Override
124109
public Settings nodeSettings(int ord) {
125-
return Settings.builder()
126-
.put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) // disable merging so no segments will be upgraded
127-
.put(RecoverySettings.INDICES_RECOVERY_CONCURRENT_SMALL_FILE_STREAMS, 30) // increase recovery speed for small files
128-
.build();
110+
111+
return OldIndexUtils.getSettings();
112+
129113
}
130114

131115
void setupCluster() throws Exception {
@@ -155,47 +139,17 @@ void setupCluster() throws Exception {
155139
// find multi data path dirs
156140
nodePaths = internalCluster().getInstance(NodeEnvironment.class, multiDataPathNode.get()).nodeDataPaths();
157141
assertEquals(2, nodePaths.length);
158-
multiDataPath = new Path[] {nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER),
159-
nodePaths[1].resolve(NodeEnvironment.INDICES_FOLDER)};
142+
multiDataPath = new Path[]{nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER),
143+
nodePaths[1].resolve(NodeEnvironment.INDICES_FOLDER)};
160144
assertFalse(Files.exists(multiDataPath[0]));
161145
assertFalse(Files.exists(multiDataPath[1]));
162146
Files.createDirectories(multiDataPath[0]);
163147
Files.createDirectories(multiDataPath[1]);
164148
logger.info("--> Multi data paths: " + multiDataPath[0].toString() + ", " + multiDataPath[1].toString());
165149

166150
replicas.get(); // wait for replicas
167-
}
168-
169-
String loadIndex(String indexFile) throws Exception {
170-
Path unzipDir = createTempDir();
171-
Path unzipDataDir = unzipDir.resolve("data");
172-
String indexName = indexFile.replace(".zip", "").toLowerCase(Locale.ROOT).replace("unsupported-", "index-");
173-
174-
// decompress the index
175-
Path backwardsIndex = getBwcIndicesPath().resolve(indexFile);
176-
try (InputStream stream = Files.newInputStream(backwardsIndex)) {
177-
TestUtil.unzip(stream, unzipDir);
178-
}
179-
180-
// check it is unique
181-
assertTrue(Files.exists(unzipDataDir));
182-
Path[] list = FileSystemUtils.files(unzipDataDir);
183-
if (list.length != 1) {
184-
throw new IllegalStateException("Backwards index must contain exactly one cluster");
185-
}
186-
187-
// the bwc scripts packs the indices under this path
188-
Path src = list[0].resolve("nodes/0/indices/" + indexName);
189-
assertTrue("[" + indexFile + "] missing index dir: " + src.toString(), Files.exists(src));
190-
191-
if (randomBoolean()) {
192-
logger.info("--> injecting index [{}] into single data path", indexName);
193-
copyIndex(logger, src, indexName, singleDataPath);
194-
} else {
195-
logger.info("--> injecting index [{}] into multi data path", indexName);
196-
copyIndex(logger, src, indexName, multiDataPath);
197-
}
198-
return indexName;
151+
singleDataPathNodeName = singleDataPathNode.get();
152+
multiDataPathNodeName = multiDataPathNode.get();
199153
}
200154

201155
void importIndex(String indexName) throws IOException {
@@ -208,43 +162,6 @@ void importIndex(String indexName) throws IOException {
208162
ensureGreen(indexName);
209163
}
210164

211-
// randomly distribute the files from src over dests paths
212-
public static void copyIndex(final ESLogger logger, final Path src, final String indexName, final Path... dests) throws IOException {
213-
for (Path dest : dests) {
214-
Path indexDir = dest.resolve(indexName);
215-
assertFalse(Files.exists(indexDir));
216-
Files.createDirectories(indexDir);
217-
}
218-
Files.walkFileTree(src, new SimpleFileVisitor<Path>() {
219-
@Override
220-
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
221-
Path relativeDir = src.relativize(dir);
222-
for (Path dest : dests) {
223-
Path destDir = dest.resolve(indexName).resolve(relativeDir);
224-
Files.createDirectories(destDir);
225-
}
226-
return FileVisitResult.CONTINUE;
227-
}
228-
229-
@Override
230-
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
231-
if (file.getFileName().toString().equals(IndexWriter.WRITE_LOCK_NAME)) {
232-
// skip lock file, we don't need it
233-
logger.trace("Skipping lock file: " + file.toString());
234-
return FileVisitResult.CONTINUE;
235-
}
236-
237-
Path relativeFile = src.relativize(file);
238-
Path destFile = dests[randomInt(dests.length - 1)].resolve(indexName).resolve(relativeFile);
239-
logger.trace("--> Moving " + relativeFile.toString() + " to " + destFile.toString());
240-
Files.move(file, destFile);
241-
assertFalse(Files.exists(file));
242-
assertTrue(Files.exists(destFile));
243-
return FileVisitResult.CONTINUE;
244-
}
245-
});
246-
}
247-
248165
void unloadIndex(String indexName) throws Exception {
249166
assertAcked(client().admin().indices().prepareDelete(indexName).get());
250167
}
@@ -312,7 +229,18 @@ public boolean apply(Object o) {
312229
void assertUnsupportedIndexHandling(String index) throws Exception {
313230
long startTime = System.currentTimeMillis();
314231
logger.info("--> Testing old index " + index);
315-
String indexName = loadIndex(index);
232+
Version version = OldIndexUtils.extractVersion(index);
233+
Path[] paths;
234+
if (randomBoolean()) {
235+
logger.info("--> injecting index [{}] into single data path", index);
236+
paths = new Path[]{singleDataPath};
237+
} else {
238+
logger.info("--> injecting index [{}] into multi data path", index);
239+
paths = multiDataPath;
240+
}
241+
242+
String indexName = index.replace(".zip", "").toLowerCase(Locale.ROOT).replace("unsupported-", "index-");
243+
OldIndexUtils.loadIndex(indexName, index, createTempDir(), getBwcIndicesPath(), logger, paths);
316244
// force reloading dangling indices with a cluster state republish
317245
client().admin().cluster().prepareReroute().get();
318246
ensureClosed(indexName);
@@ -327,29 +255,33 @@ void assertUnsupportedIndexHandling(String index) throws Exception {
327255
}
328256

329257
void assertOldIndexWorks(String index) throws Exception {
330-
Version version = extractVersion(index);
331-
String indexName = loadIndex(index);
258+
Version version = OldIndexUtils.extractVersion(index);
259+
Path[] paths;
260+
if (randomBoolean()) {
261+
logger.info("--> injecting index [{}] into single data path", index);
262+
paths = new Path[]{singleDataPath};
263+
} else {
264+
logger.info("--> injecting index [{}] into multi data path", index);
265+
paths = multiDataPath;
266+
}
267+
268+
String indexName = index.replace(".zip", "").toLowerCase(Locale.ROOT).replace("unsupported-", "index-");
269+
OldIndexUtils.loadIndex(indexName, index, createTempDir(), getBwcIndicesPath(), logger, paths);
270+
// we explicitly upgrade the index folders as these indices
271+
// are imported as dangling indices and not available on
272+
// node startup
332273
importIndex(indexName);
333274
assertIndexSanity(indexName, version);
334275
assertBasicSearchWorks(indexName);
335276
assertBasicAggregationWorks(indexName);
336277
assertRealtimeGetWorks(indexName);
337278
assertNewReplicasWork(indexName);
338-
assertUpgradeWorks(indexName, isLatestLuceneVersion(version));
279+
assertUpgradeWorks(client(), indexName, version);
339280
assertDeleteByQueryWorked(indexName, version);
340281
assertPositionIncrementGapDefaults(indexName, version);
341282
unloadIndex(indexName);
342283
}
343284

344-
Version extractVersion(String index) {
345-
return Version.fromString(index.substring(index.indexOf('-') + 1, index.lastIndexOf('.')));
346-
}
347-
348-
boolean isLatestLuceneVersion(Version version) {
349-
return version.luceneVersion.major == Version.CURRENT.luceneVersion.major &&
350-
version.luceneVersion.minor == Version.CURRENT.luceneVersion.minor;
351-
}
352-
353285
void assertIndexSanity(String indexName, Version indexCreated) {
354286
GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().addIndices(indexName).get();
355287
assertEquals(1, getIndexResponse.indices().length);
@@ -398,7 +330,8 @@ void assertBasicSearchWorks(String indexName) {
398330

399331
void assertBasicAggregationWorks(String indexName) {
400332
// histogram on a long
401-
SearchResponse searchRsp = client().prepareSearch(indexName).addAggregation(AggregationBuilders.histogram("histo").field("long_sort").interval(10)).get();
333+
SearchResponse searchRsp = client().prepareSearch(indexName).addAggregation(AggregationBuilders.histogram("histo").field
334+
("long_sort").interval(10)).get();
402335
ElasticsearchAssertions.assertSearchResponse(searchRsp);
403336
Histogram histo = searchRsp.getAggregations().get("histo");
404337
assertNotNull(histo);
@@ -441,7 +374,7 @@ void assertNewReplicasWork(String indexName) throws Exception {
441374
final long startTime = System.currentTimeMillis();
442375
logger.debug("--> creating [{}] replicas for index [{}]", numReplicas, indexName);
443376
assertAcked(client().admin().indices().prepareUpdateSettings(indexName).setSettings(Settings.builder()
444-
.put("number_of_replicas", numReplicas)
377+
.put("number_of_replicas", numReplicas)
445378
).execute().actionGet());
446379
ensureGreen(TimeValue.timeValueMinutes(2), indexName);
447380
logger.debug("--> index [{}] is green, took [{}]", indexName, TimeValue.timeValueMillis(System.currentTimeMillis() - startTime));
@@ -469,12 +402,62 @@ void assertPositionIncrementGapDefaults(String indexName, Version version) throw
469402
}
470403
}
471404

472-
void assertUpgradeWorks(String indexName, boolean alreadyLatest) throws Exception {
473-
if (alreadyLatest == false) {
474-
assertNotUpgraded(client(), indexName);
405+
private Path getNodeDir(String indexFile) throws IOException {
406+
Path unzipDir = createTempDir();
407+
Path unzipDataDir = unzipDir.resolve("data");
408+
409+
// decompress the index
410+
Path backwardsIndex = getBwcIndicesPath().resolve(indexFile);
411+
try (InputStream stream = Files.newInputStream(backwardsIndex)) {
412+
TestUtil.unzip(stream, unzipDir);
475413
}
476-
assertNoFailures(client().admin().indices().prepareUpgrade(indexName).get());
477-
assertUpgraded(client(), indexName);
414+
415+
// check it is unique
416+
assertTrue(Files.exists(unzipDataDir));
417+
Path[] list = FileSystemUtils.files(unzipDataDir);
418+
if (list.length != 1) {
419+
throw new IllegalStateException("Backwards index must contain exactly one cluster");
420+
}
421+
422+
// the bwc scripts packs the indices under this path
423+
return list[0].resolve("nodes/0/");
478424
}
479425

426+
public void testOldClusterStates() throws Exception {
427+
// dangling indices do not load the global state, only the per-index states
428+
// so we make sure we can read them separately
429+
MetaDataStateFormat<MetaData> globalFormat = new MetaDataStateFormat<MetaData>(XContentType.JSON, "global-") {
430+
431+
@Override
432+
public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
433+
throw new UnsupportedOperationException();
434+
}
435+
436+
@Override
437+
public MetaData fromXContent(XContentParser parser) throws IOException {
438+
return MetaData.Builder.fromXContent(parser);
439+
}
440+
};
441+
MetaDataStateFormat<IndexMetaData> indexFormat = new MetaDataStateFormat<IndexMetaData>(XContentType.JSON, "state-") {
442+
443+
@Override
444+
public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOException {
445+
throw new UnsupportedOperationException();
446+
}
447+
448+
@Override
449+
public IndexMetaData fromXContent(XContentParser parser) throws IOException {
450+
return IndexMetaData.Builder.fromXContent(parser);
451+
}
452+
};
453+
Collections.shuffle(indexes, random());
454+
for (String indexFile : indexes) {
455+
String indexName = indexFile.replace(".zip", "").toLowerCase(Locale.ROOT).replace("unsupported-", "index-");
456+
Path nodeDir = getNodeDir(indexFile);
457+
logger.info("Parsing cluster state files from index [{}]", indexName);
458+
assertNotNull(globalFormat.loadLatestState(logger, nodeDir)); // no exception
459+
Path indexDir = nodeDir.resolve("indices").resolve(indexName);
460+
assertNotNull(indexFormat.loadLatestState(logger, indexDir)); // no exception
461+
}
462+
}
480463
}

‎core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.elasticsearch.index.shard.ShardPath;
3535
import org.elasticsearch.index.shard.ShardStateMetaData;
3636
import org.elasticsearch.test.ESTestCase;
37+
import org.elasticsearch.test.OldIndexUtils;
3738

3839
import java.io.BufferedWriter;
3940
import java.io.IOException;
@@ -175,7 +176,7 @@ public int compare(Path o1, Path o2) {
175176
multiDataPath[i++] = nodePath.indicesPath;
176177
}
177178
logger.info("--> injecting index [{}] into multiple data paths", indexName);
178-
OldIndexBackwardsCompatibilityIT.copyIndex(logger, src, indexName, multiDataPath);
179+
OldIndexUtils.copyIndex(logger, src, indexName, multiDataPath);
179180
final ShardPath shardPath = new ShardPath(false, nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], IndexMetaData.INDEX_UUID_NA_VALUE, new ShardId(indexName, 0));
180181

181182
logger.info("{}", FileSystemUtils.files(shardPath.resolveIndex()));

0 commit comments

Comments
 (0)