From 4fdee8bdc1f9c87529f95d73e5e447b997dc7e8a Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 20 Apr 2017 09:22:21 +0200 Subject: [PATCH 1/2] ESIntegTestCase.indexRandom should not introduce types. Since we plan on removing types, `indexRandom` should not introduce new types. This commit refactors `indexRandom` to reuse existing types. --- .../elasticsearch/test/ESIntegTestCase.java | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 6d15a5e164ef5..9c5f6c4799080 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1338,9 +1338,6 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, IndexReque indexRandom(forceRefresh, dummyDocuments, Arrays.asList(builders)); } - - private static final String RANDOM_BOGUS_TYPE = "RANDOM_BOGUS_TYPE______"; - /** * Indexes the given {@link IndexRequestBuilder} instances randomly. It shuffles the given builders and either * indexes them in a blocking or async fashion. This is very useful to catch problems that relate to internal document @@ -1388,31 +1385,33 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, List builders) throws InterruptedException, ExecutionException { - Random random = random(); - Set indicesSet = new HashSet<>(); + Map> indicesAndTypes = new HashMap<>(); for (IndexRequestBuilder builder : builders) { - indicesSet.add(builder.request().index()); + final Set types = indicesAndTypes.computeIfAbsent(builder.request().index(), index -> new HashSet<>()); + types.add(builder.request().type()); } - Set> bogusIds = new HashSet<>(); + Set> bogusIds = new HashSet<>(); if (random.nextBoolean() && !builders.isEmpty() && dummyDocuments) { builders = new ArrayList<>(builders); - final String[] indices = indicesSet.toArray(new String[indicesSet.size()]); // inject some bogus docs final int numBogusDocs = scaledRandomIntBetween(1, builders.size() * 2); final int unicodeLen = between(1, 10); for (int i = 0; i < numBogusDocs; i++) { - String id = randomRealisticUnicodeOfLength(unicodeLen) + Integer.toString(dummmyDocIdGenerator.incrementAndGet()); - String index = RandomPicks.randomFrom(random, indices); - bogusIds.add(new Tuple<>(index, id)); - builders.add(client().prepareIndex(index, RANDOM_BOGUS_TYPE, id).setSource("{}", XContentType.JSON)); + String id = "bogus_doc_" + randomRealisticUnicodeOfLength(unicodeLen) + Integer.toString(dummmyDocIdGenerator.incrementAndGet()); + Map.Entry> indexAndTypes = RandomPicks.randomFrom(random, indicesAndTypes.entrySet()); + String index = indexAndTypes.getKey(); + String type = RandomPicks.randomFrom(random, indexAndTypes.getValue()); + bogusIds.add(Arrays.asList(index, type, id)); + // We configure a routing key in case the mapping requires it + builders.add(client().prepareIndex(index, type, id).setSource("{}", XContentType.JSON).setRouting(id)); } } - final String[] indices = indicesSet.toArray(new String[indicesSet.size()]); Collections.shuffle(builders, random()); final CopyOnWriteArrayList> errors = new CopyOnWriteArrayList<>(); List inFlightAsyncOperations = new ArrayList<>(); // If you are indexing just a few documents then frequently do it one at a time. If many then frequently in bulk. + final String[] indices = indicesAndTypes.keySet().toArray(new String[0]); if (builders.size() < FREQUENT_BULK_THRESHOLD ? frequently() : builders.size() < ALWAYS_BULK_THRESHOLD ? rarely() : false) { if (frequently()) { logger.info("Index [{}] docs async: [{}] bulk: [{}]", builders.size(), true, false); @@ -1454,10 +1453,10 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma assertThat(actualErrors, emptyIterable()); if (!bogusIds.isEmpty()) { // delete the bogus types again - it might trigger merges or at least holes in the segments and enforces deleted docs! - for (Tuple doc : bogusIds) { - assertEquals("failed to delete a dummy doc [" + doc.v1() + "][" + doc.v2() + "]", + for (List doc : bogusIds) { + assertEquals("failed to delete a dummy doc [" + doc.get(0) + "][" + doc.get(2) + "]", DocWriteResponse.Result.DELETED, - client().prepareDelete(doc.v1(), RANDOM_BOGUS_TYPE, doc.v2()).get().getResult()); + client().prepareDelete(doc.get(0), doc.get(1), doc.get(2)).setRouting(doc.get(2)).get().getResult()); } } if (forceRefresh) { From ac1565d7edceab4cf6fd81bbb1608c0c77a3a004 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 21 Apr 2017 10:38:08 +0200 Subject: [PATCH 2/2] Document structure. --- .../src/main/java/org/elasticsearch/test/ESIntegTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 9c5f6c4799080..7c8d8cd1a551f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1391,7 +1391,7 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma final Set types = indicesAndTypes.computeIfAbsent(builder.request().index(), index -> new HashSet<>()); types.add(builder.request().type()); } - Set> bogusIds = new HashSet<>(); + Set> bogusIds = new HashSet<>(); // (index, type, id) if (random.nextBoolean() && !builders.isEmpty() && dummyDocuments) { builders = new ArrayList<>(builders); // inject some bogus docs