Skip to content

Commit 54e9dea

Browse files
Merge remote-tracking branch 'elastic/master' into more-efficient-snapshot-info-api
2 parents b874ef5 + 15fc541 commit 54e9dea

File tree

12 files changed

+96
-134
lines changed

12 files changed

+96
-134
lines changed

rest-api-spec/build.gradle

Lines changed: 42 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -28,106 +28,52 @@ testClusters.all {
2828

2929
tasks.named("test").configure { enabled = false }
3030
tasks.named("jarHell").configure { enabled = false }
31-
31+
def v7compatiblityNotSupportedTests = {
32+
return [
33+
// Cat API are meant to be consumed by humans, so will not be supported by Compatible REST API
34+
'cat*/*/*',
35+
// type information about the type is removed and not passed down. The logic to check for this is also removed.
36+
'delete/70_mix_typeless_typeful/DELETE with typeless API on an index that has types',
37+
// WILL NOT BE FIXED - failing due to not recognising missing type (the type path param is ignored)
38+
'get/100_mix_typeless_typeful/GET with typeless API on an index that has types',
39+
// type information about the type is removed and not passed down. The logic to check for this is also removed.
40+
'indices.create/20_mix_typeless_typeful/Implicitly create a typed index while there is a typeless template',
41+
'indices.create/20_mix_typeless_typeful/Implicitly create a typeless index while there is a typed template',
42+
//
43+
// This test returns test_index.mappings:{} when {} was expected. difference between 20_missing_field and 21_missing_field_with_types?
44+
'indices.get_field_mapping/21_missing_field_with_types/Return empty object if field doesn\'t exist, but type and index do',
45+
// The information about the type is not present in the index. hence it cannot know if the type exist or not.
46+
'indices.get_field_mapping/30_missing_type/Raise 404 when type doesn\'t exist',
47+
// The information about the type is not present in the index. hence it cannot know if the type exist or not.
48+
'indices.get_mapping/20_missing_type/Existent and non-existent type returns 404 and the existing type',
49+
'indices.get_mapping/20_missing_type/Existent and non-existent types returns 404 and the existing type',
50+
'indices.get_mapping/20_missing_type/No type matching pattern returns 404',
51+
'indices.get_mapping/20_missing_type/Non-existent type returns 404',
52+
'indices.get_mapping/20_missing_type/Type missing when no types exist',
53+
//
54+
// The information about the type is not present in the index. hence it cannot know if the type was already used or not
55+
'indices.put_mapping/20_mix_typeless_typeful/PUT mapping with _doc on an index that has types',
56+
'indices.put_mapping/20_mix_typeless_typeful/PUT mapping with typeless API on an index that has types',
57+
// there is a small distinction between empty mappings and no mappings at all. The code to implement this test was refactored #54003
58+
// field search on _type field- not implementing. The data for _type is considered incorrect in this search
59+
'search/160_exists_query/Test exists query on _type field',
60+
//type information is not stored, hence the the index will be found
61+
'termvectors/50_mix_typeless_typeful/Term vectors with typeless API on an index that has types',
62+
// mget - these use cases are no longer valid, because we always default to _doc.
63+
// This mean test cases where there is assertion on not finging by type won't work
64+
'mget/11_default_index_type/Default index/type',
65+
'mget/16_basic_with_types/Basic multi-get',
66+
// asserting about type not found won't work as we ignore the type information
67+
'explain/40_mix_typeless_typeful/Explain with typeless API on an index that has types',
68+
]
69+
}
3270
tasks.named("yamlRestCompatTest").configure {
3371
onlyIf {
3472
// Skip these tests on Windows since the blacklist exceeds Windows CLI limits
3573
OS.current() != OS.WINDOWS
3674
}
3775

38-
systemProperty 'tests.rest.blacklist', [
39-
'cat.aliases/10_basic/Alias against closed index',
40-
'cat.aliases/10_basic/Alias name',
41-
'cat.aliases/10_basic/Alias sorting',
42-
'cat.aliases/10_basic/Column headers',
43-
'cat.aliases/10_basic/Complex alias',
44-
'cat.aliases/10_basic/Empty cluster',
45-
'cat.aliases/10_basic/Multiple alias names',
46-
'cat.aliases/10_basic/Select columns',
47-
'cat.aliases/10_basic/Simple alias',
48-
'cat.aliases/40_hidden/Test cat aliases output with a hidden index with a hidden alias',
49-
'cat.aliases/40_hidden/Test cat aliases output with a hidden index with a visible alias',
50-
'cat.aliases/40_hidden/Test cat aliases output with a visible index with a hidden alias',
51-
'cat.allocation/10_basic/All Nodes',
52-
'cat.allocation/10_basic/Bytes',
53-
'cat.allocation/10_basic/Column headers',
54-
'cat.allocation/10_basic/Empty cluster',
55-
'cat.allocation/10_basic/Node ID',
56-
'cat.allocation/10_basic/One index',
57-
'cat.allocation/10_basic/Select columns',
58-
'cat.count/10_basic/Test cat count output',
59-
'cat.fielddata/10_basic/Test cat fielddata output',
60-
'cat.health/10_basic/Empty cluster',
61-
'cat.health/10_basic/With ts parameter',
62-
'cat.indices/10_basic/Test cat indices output',
63-
'cat.indices/10_basic/Test cat indices output (no indices)',
64-
'cat.indices/10_basic/Test cat indices output for closed index',
65-
'cat.indices/10_basic/Test cat indices sort',
66-
'cat.indices/10_basic/Test cat indices using health status',
67-
'cat.indices/10_basic/Test cat indices using wildcards',
68-
'cat.indices/20_hidden/Test cat indices output for dot-hidden index and dot-prefixed pattern',
69-
'cat.indices/20_hidden/Test cat indices output for hidden index',
70-
'cat.indices/20_hidden/Test cat indices output with a hidden index with a hidden alias',
71-
'cat.indices/20_hidden/Test cat indices output with a hidden index with a visible alias',
72-
'cat.indices/20_hidden/Test cat indices output with a hidden index, dot-hidden alias and dot pattern',
73-
'cat.nodeattrs/10_basic/Test cat nodes attrs output',
74-
'cat.nodes/10_basic/Additional disk information',
75-
'cat.nodes/10_basic/Test cat nodes output',
76-
'cat.nodes/10_basic/Test cat nodes output with full_id set',
77-
'cat.recovery/10_basic/Test cat recovery output',
78-
'cat.recovery/10_basic/Test cat recovery output for closed index',
79-
'cat.repositories/10_basic/Test cat repositories output',
80-
'cat.repositories/10_basic/Test cat repositories sort',
81-
'cat.segments/10_basic/Test cat segments output',
82-
'cat.segments/10_basic/Test cat segments using wildcards',
83-
'cat.shards/10_basic/Help',
84-
'cat.shards/10_basic/Test cat shards output',
85-
'cat.shards/10_basic/Test cat shards sort',
86-
'cat.shards/10_basic/Test cat shards using wildcards',
87-
'cat.snapshots/10_basic/Help',
88-
'cat.snapshots/10_basic/Test cat snapshots output',
89-
'cat.tasks/10_basic/Test cat tasks output',
90-
'cat.tasks/10_basic/Test cat tasks output with X-Opaque-Id',
91-
'cat.templates/10_basic/Column headers',
92-
'cat.templates/10_basic/Filtered templates',
93-
'cat.templates/10_basic/Mixture of legacy and composable templates',
94-
'cat.templates/10_basic/Normal templates',
95-
'cat.templates/10_basic/Select columns',
96-
'cat.thread_pool/10_basic/Test cat thread_pool output',
97-
// type information about the type is removed and not passed down. The logic to check for this is also removed.
98-
'delete/70_mix_typeless_typeful/DELETE with typeless API on an index that has types',
99-
// WILL NOT BE FIXED - failing due to not recognising missing type (the type path param is ignored)
100-
'get/100_mix_typeless_typeful/GET with typeless API on an index that has types',
101-
// type information about the type is removed and not passed down. The logic to check for this is also removed.
102-
'indices.create/20_mix_typeless_typeful/Implicitly create a typed index while there is a typeless template',
103-
'indices.create/20_mix_typeless_typeful/Implicitly create a typeless index while there is a typed template',
104-
//
105-
// This test returns test_index.mappings:{} when {} was expected. difference between 20_missing_field and 21_missing_field_with_types?
106-
'indices.get_field_mapping/21_missing_field_with_types/Return empty object if field doesn\'t exist, but type and index do',
107-
// The information about the type is not present in the index. hence it cannot know if the type exist or not.
108-
'indices.get_field_mapping/30_missing_type/Raise 404 when type doesn\'t exist',
109-
// The information about the type is not present in the index. hence it cannot know if the type exist or not.
110-
'indices.get_mapping/20_missing_type/Existent and non-existent type returns 404 and the existing type',
111-
'indices.get_mapping/20_missing_type/Existent and non-existent types returns 404 and the existing type',
112-
'indices.get_mapping/20_missing_type/No type matching pattern returns 404',
113-
'indices.get_mapping/20_missing_type/Non-existent type returns 404',
114-
'indices.get_mapping/20_missing_type/Type missing when no types exist',
115-
//
116-
// The information about the type is not present in the index. hence it cannot know if the type was already used or not
117-
'indices.put_mapping/20_mix_typeless_typeful/PUT mapping with _doc on an index that has types',
118-
'indices.put_mapping/20_mix_typeless_typeful/PUT mapping with typeless API on an index that has types',
119-
// there is a small distinction between empty mappings and no mappings at all. The code to implement this test was refactored #54003
120-
// field search on _type field- not implementing. The data for _type is considered incorrect in this search
121-
'search/160_exists_query/Test exists query on _type field',
122-
//type information is not stored, hence the the index will be found
123-
'termvectors/50_mix_typeless_typeful/Term vectors with typeless API on an index that has types',
124-
// mget - these use cases are no longer valid, because we always default to _doc.
125-
// This mean test cases where there is assertion on not finging by type won't work
126-
'mget/11_default_index_type/Default index/type',
127-
'mget/16_basic_with_types/Basic multi-get',
128-
// asserting about type not found won't work as we ignore the type information
129-
'explain/40_mix_typeless_typeful/Explain with typeless API on an index that has types',
130-
// 89 - 15 = 74 tests won't be fixed
76+
systemProperty 'tests.rest.blacklist', ([
13177
'cluster.voting_config_exclusions/10_basic/Throw exception when adding voting config exclusion and specifying both node_ids and node_names',
13278
'cluster.voting_config_exclusions/10_basic/Throw exception when adding voting config exclusion without specifying nodes',
13379
'count/11_basic_with_types/count body without query element',
@@ -192,7 +138,8 @@ tasks.named("yamlRestCompatTest").configure {
192138
'snapshot.get/10_basic/Get snapshot info with metadata',
193139
'snapshot.get/10_basic/Get snapshot info with index details',
194140
'suggest/20_completion/Suggestions with source should work'
195-
].join(',')
141+
] + v7compatiblityNotSupportedTests())
142+
.join(',')
196143
}
197144

198145
tasks.named("transformV7RestTests").configure({ task ->

server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
package org.elasticsearch.search.fetch.subphase.highlight;
99

1010
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
11+
1112
import org.apache.lucene.analysis.Analyzer;
1213
import org.apache.lucene.analysis.MockAnalyzer;
1314
import org.apache.lucene.analysis.MockTokenizer;
@@ -74,6 +75,7 @@
7475
import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
7576
import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
7677
import static org.elasticsearch.index.query.QueryBuilders.fuzzyQuery;
78+
import static org.elasticsearch.index.query.QueryBuilders.matchPhrasePrefixQuery;
7779
import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery;
7880
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
7981
import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery;
@@ -2908,7 +2910,7 @@ public void testWithNestedQuery() throws Exception {
29082910

29092911
client().prepareIndex("test").setId("1").setSource(jsonBuilder().startObject()
29102912
.startArray("foo")
2911-
.startObject().field("text", "brown").endObject()
2913+
.startObject().field("text", "brown shoes").endObject()
29122914
.startObject().field("text", "cow").endObject()
29132915
.endArray()
29142916
.field("text", "brown")
@@ -2924,7 +2926,7 @@ public void testWithNestedQuery() throws Exception {
29242926
assertHitCount(searchResponse, 1);
29252927
HighlightField field = searchResponse.getHits().getAt(0).getHighlightFields().get("foo.text");
29262928
assertThat(field.getFragments().length, equalTo(2));
2927-
assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em>"));
2929+
assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em> shoes"));
29282930
assertThat(field.getFragments()[1].string(), equalTo("<em>cow</em>"));
29292931

29302932
searchResponse = client().prepareSearch()
@@ -2935,17 +2937,27 @@ public void testWithNestedQuery() throws Exception {
29352937
assertHitCount(searchResponse, 1);
29362938
field = searchResponse.getHits().getAt(0).getHighlightFields().get("foo.text");
29372939
assertThat(field.getFragments().length, equalTo(1));
2938-
assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em>"));
2940+
assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em> shoes"));
29392941

29402942
searchResponse = client().prepareSearch()
2941-
.setQuery(nestedQuery("foo", prefixQuery("foo.text", "bro"), ScoreMode.None))
2943+
.setQuery(nestedQuery("foo", matchPhraseQuery("foo.text", "brown shoes"), ScoreMode.None))
29422944
.highlighter(new HighlightBuilder()
2943-
.field(new Field("foo.text").highlighterType("plain")))
2945+
.field(new Field("foo.text").highlighterType(type)))
29442946
.get();
29452947
assertHitCount(searchResponse, 1);
29462948
field = searchResponse.getHits().getAt(0).getHighlightFields().get("foo.text");
29472949
assertThat(field.getFragments().length, equalTo(1));
2948-
assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em>"));
2950+
assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em> <em>shoes</em>"));
2951+
2952+
searchResponse = client().prepareSearch()
2953+
.setQuery(nestedQuery("foo", matchPhrasePrefixQuery("foo.text", "bro"), ScoreMode.None))
2954+
.highlighter(new HighlightBuilder()
2955+
.field(new Field("foo.text").highlighterType(type)))
2956+
.get();
2957+
assertHitCount(searchResponse, 1);
2958+
field = searchResponse.getHits().getAt(0).getHighlightFields().get("foo.text");
2959+
assertThat(field.getFragments().length, equalTo(1));
2960+
assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em> shoes"));
29492961
}
29502962

29512963
// For unified and fvh highlighters we just check that the nested query is correctly extracted

server/src/main/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighter.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.elasticsearch.core.Nullable;
2525
import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
2626
import org.elasticsearch.index.IndexSettings;
27+
import org.elasticsearch.index.search.ESToParentBlockJoinQuery;
2728

2829
import java.io.IOException;
2930
import java.text.BreakIterator;
@@ -204,6 +205,8 @@ private Collection<Query> rewriteCustomQuery(Query query) {
204205
boolean inorder = (mpq.getSlop() == 0);
205206
return Collections.singletonList(new SpanNearQuery(positionSpanQueries,
206207
mpq.getSlop() + positionGaps, inorder));
208+
} else if (query instanceof ESToParentBlockJoinQuery) {
209+
return Collections.singletonList(((ESToParentBlockJoinQuery) query).getChildQuery());
207210
} else {
208211
return null;
209212
}

x-pack/plugin/core/build.gradle

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -148,10 +148,5 @@ testClusters.all {
148148
tasks.named('splitPackagesAudit').configure {
149149
// engine is owned by server, these should be renamed
150150
ignoreClasses 'org.elasticsearch.index.engine.FrozenEngine',
151-
'org.elasticsearch.index.engine.RewriteCachingDirectoryReader',
152-
153-
// snapshots are part of server, these should be renamed
154-
'org.elasticsearch.snapshots.SeqIdGeneratingFilterReader',
155-
'org.elasticsearch.snapshots.SourceOnlySnapshot',
156-
'org.elasticsearch.snapshots.SourceOnlySnapshotRepository'
151+
'org.elasticsearch.index.engine.RewriteCachingDirectoryReader'
157152
}
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
* 2.0; you may not use this file except in compliance with the Elastic License
55
* 2.0.
66
*/
7-
package org.elasticsearch.snapshots;
7+
package org.elasticsearch.snapshots.sourceonly;
88

99
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
1010
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
@@ -38,6 +38,7 @@
3838
import org.elasticsearch.search.SearchHits;
3939
import org.elasticsearch.search.slice.SliceBuilder;
4040
import org.elasticsearch.search.sort.SortOrder;
41+
import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
4142
import org.elasticsearch.test.ESIntegTestCase;
4243
import org.hamcrest.Matchers;
4344

x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SeqIdGeneratingFilterReader.java renamed to x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SeqIdGeneratingFilterReader.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
* 2.0; you may not use this file except in compliance with the Elastic License
55
* 2.0.
66
*/
7-
package org.elasticsearch.snapshots;
7+
package org.elasticsearch.snapshots.sourceonly;
88

99
import org.apache.lucene.codecs.StoredFieldsReader;
1010
import org.apache.lucene.index.DirectoryReader;

x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java renamed to x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
* 2.0; you may not use this file except in compliance with the Elastic License
55
* 2.0.
66
*/
7-
package org.elasticsearch.snapshots;
7+
package org.elasticsearch.snapshots.sourceonly;
88

99
import org.apache.lucene.codecs.Codec;
1010
import org.apache.lucene.index.CheckIndex;

x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java renamed to x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotRepository.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
* 2.0; you may not use this file except in compliance with the Elastic License
55
* 2.0.
66
*/
7-
package org.elasticsearch.snapshots;
7+
package org.elasticsearch.snapshots.sourceonly;
88

99
import org.apache.logging.log4j.LogManager;
1010
import org.apache.logging.log4j.Logger;
@@ -45,6 +45,7 @@
4545
import org.elasticsearch.repositories.RepositoryData;
4646
import org.elasticsearch.repositories.ShardGenerations;
4747
import org.elasticsearch.repositories.SnapshotShardContext;
48+
import org.elasticsearch.snapshots.SnapshotInfo;
4849

4950
import java.io.Closeable;
5051
import java.io.IOException;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@
5858
import org.elasticsearch.rest.RestController;
5959
import org.elasticsearch.rest.RestHandler;
6060
import org.elasticsearch.script.ScriptService;
61-
import org.elasticsearch.snapshots.SourceOnlySnapshotRepository;
61+
import org.elasticsearch.snapshots.sourceonly.SourceOnlySnapshotRepository;
6262
import org.elasticsearch.threadpool.ThreadPool;
6363
import org.elasticsearch.watcher.ResourceWatcherService;
6464
import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;

x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java renamed to x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
* 2.0; you may not use this file except in compliance with the Elastic License
55
* 2.0.
66
*/
7-
package org.elasticsearch.snapshots;
7+
package org.elasticsearch.snapshots.sourceonly;
88

99
import org.apache.lucene.document.Document;
1010
import org.apache.lucene.index.DirectoryReader;
@@ -70,6 +70,9 @@
7070
import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil;
7171
import org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase;
7272
import org.elasticsearch.repositories.fs.FsRepository;
73+
import org.elasticsearch.snapshots.Snapshot;
74+
import org.elasticsearch.snapshots.SnapshotId;
75+
import org.elasticsearch.snapshots.SnapshotInfo;
7376
import org.elasticsearch.threadpool.ThreadPool;
7477
import org.hamcrest.Matchers;
7578

0 commit comments

Comments
 (0)