From b428b0cf906299d862dd9609116a689d463a0120 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Tue, 19 Jun 2018 18:12:06 -0400 Subject: [PATCH 1/3] Remove QueryCachingPolicy#ALWAYS_CACHE The QueryCachingPolicy#ALWAYS_CACHE was deprecated in Lucene-7.4 and will be removed in Lucene-8.0. This change replaces it with UsageTrackingQueryCachingPolicy. This commit also makes INDEX_QUERY_CACHE_EVERYTHING_SETTING visible in testing only. --- .../common/settings/IndexScopedSettings.java | 1 - .../elasticsearch/index/shard/IndexShard.java | 8 +++++++- .../indices/IndicesQueryCacheTests.java | 20 ++++++++++++++----- .../elasticsearch/test/ESIntegTestCase.java | 4 ---- .../test/InternalSettingsPlugin.java | 5 ++++- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index bd6bba7b784cd..6f45a5d462bd8 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -151,7 +151,6 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexModule.INDEX_STORE_TYPE_SETTING, IndexModule.INDEX_STORE_PRE_LOAD_SETTING, IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING, - IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING, FsDirectoryService.INDEX_LOCK_FACTOR_SETTING, EngineConfig.INDEX_CODEC_SETTING, EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS, diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 34230be14cb7e..affe265e3c4b5 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -29,6 +29,7 @@ import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.ReferenceManager; import org.apache.lucene.search.Sort; @@ -299,7 +300,12 @@ public IndexShard( // the query cache is a node-level thing, however we want the most popular filters // to be computed on a per-shard basis if (IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.get(settings)) { - cachingPolicy = QueryCachingPolicy.ALWAYS_CACHE; + cachingPolicy = new UsageTrackingQueryCachingPolicy() { + @Override + public boolean shouldCache(Query query) { + return true; + } + }; } else { cachingPolicy = new UsageTrackingQueryCachingPolicy(); } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index 83bde66e3bd21..61b490e30d684 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -34,6 +34,7 @@ import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.UsageTrackingQueryCachingPolicy; import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; import org.elasticsearch.core.internal.io.IOUtils; @@ -89,6 +90,15 @@ public boolean isCacheable(LeafReaderContext ctx) { } + private static QueryCachingPolicy alwaysCachePolicy() { + return new UsageTrackingQueryCachingPolicy() { + @Override + public boolean shouldCache(Query query) { + return true; + } + }; + } + public void testBasics() throws IOException { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig()); @@ -98,7 +108,7 @@ public void testBasics() throws IOException { ShardId shard = new ShardId("index", "_na_", 0); r = ElasticsearchDirectoryReader.wrap(r, shard); IndexSearcher s = new IndexSearcher(r); - s.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); + s.setQueryCachingPolicy(alwaysCachePolicy()); Settings settings = Settings.builder() .put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING.getKey(), 10) @@ -169,7 +179,7 @@ public void testTwoShards() throws IOException { ShardId shard1 = new ShardId("index", "_na_", 0); r1 = ElasticsearchDirectoryReader.wrap(r1, shard1); IndexSearcher s1 = new IndexSearcher(r1); - s1.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); + s1.setQueryCachingPolicy(alwaysCachePolicy()); Directory dir2 = newDirectory(); IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig()); @@ -179,7 +189,7 @@ public void testTwoShards() throws IOException { ShardId shard2 = new ShardId("index", "_na_", 1); r2 = ElasticsearchDirectoryReader.wrap(r2, shard2); IndexSearcher s2 = new IndexSearcher(r2); - s2.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); + s2.setQueryCachingPolicy(alwaysCachePolicy()); Settings settings = Settings.builder() .put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING.getKey(), 10) @@ -295,7 +305,7 @@ public void testStatsOnEviction() throws IOException { ShardId shard1 = new ShardId("index", "_na_", 0); r1 = ElasticsearchDirectoryReader.wrap(r1, shard1); IndexSearcher s1 = new IndexSearcher(r1); - s1.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); + s1.setQueryCachingPolicy(alwaysCachePolicy()); Directory dir2 = newDirectory(); IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig()); @@ -305,7 +315,7 @@ public void testStatsOnEviction() throws IOException { ShardId shard2 = new ShardId("index", "_na_", 1); r2 = ElasticsearchDirectoryReader.wrap(r2, shard2); IndexSearcher s2 = new IndexSearcher(r2); - s2.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); + s2.setQueryCachingPolicy(alwaysCachePolicy()); Settings settings = Settings.builder() .put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING.getKey(), 10) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 979bfccdb64b8..c63a1c9c6e68f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -434,10 +434,6 @@ public void randomIndexTemplate() throws IOException { if (randomBoolean()) { randomSettingsBuilder.put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), randomBoolean()); } - - if (randomBoolean()) { - randomSettingsBuilder.put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), randomBoolean()); - } PutIndexTemplateRequestBuilder putTemplate = client().admin().indices() .preparePutTemplate("random_index_template") .setPatterns(Collections.singletonList("*")) diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java b/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java index e1c555b811064..be8c824f0f790 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexService; import org.elasticsearch.plugins.Plugin; @@ -51,6 +52,8 @@ public List> getSettings() { INDEX_CREATION_DATE_SETTING, PROVIDED_NAME_SETTING, TRANSLOG_RETENTION_CHECK_INTERVAL_SETTING, - IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING); + IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING, + IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING + ); } } From f7c3cbcf8602074dfcbad171c2102f20f6b207b9 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Tue, 19 Jun 2018 21:53:14 -0400 Subject: [PATCH 2/3] Fix tests --- .../search/scriptfilter/ScriptQuerySearchIT.java | 4 +++- .../java/org/elasticsearch/test/SecuritySettingsSource.java | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java b/server/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java index 45e374b8697a2..16a9d99b78341 100644 --- a/server/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java +++ b/server/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java @@ -31,8 +31,10 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; import java.io.IOException; +import java.util.Arrays; import java.util.Base64; import java.util.Collection; import java.util.Collections; @@ -52,7 +54,7 @@ public class ScriptQuerySearchIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Collections.singleton(CustomScriptPlugin.class); + return Arrays.asList(CustomScriptPlugin.class, InternalSettingsPlugin.class); } public static class CustomScriptPlugin extends MockScriptPlugin { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java index 1690ab652c067..8ad1c61029a97 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java @@ -169,7 +169,8 @@ protected void addDefaultSecurityTransportType(Settings.Builder builder, Setting @Override public Collection> nodePlugins() { - return Arrays.asList(LocalStateSecurity.class, Netty4Plugin.class, ReindexPlugin.class, CommonAnalysisPlugin.class); + return Arrays.asList(LocalStateSecurity.class, Netty4Plugin.class, ReindexPlugin.class, CommonAnalysisPlugin.class, + InternalSettingsPlugin.class); } @Override From 28efc8b39fe7a2379d00cb55e7e62e19f293fdf9 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 20 Jun 2018 08:06:28 -0400 Subject: [PATCH 3/3] Extends from QueryCachingPolicy --- .../java/org/elasticsearch/index/shard/IndexShard.java | 6 +++++- .../org/elasticsearch/indices/IndicesQueryCacheTests.java | 7 +++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index affe265e3c4b5..fb987fe035afa 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -300,7 +300,11 @@ public IndexShard( // the query cache is a node-level thing, however we want the most popular filters // to be computed on a per-shard basis if (IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.get(settings)) { - cachingPolicy = new UsageTrackingQueryCachingPolicy() { + cachingPolicy = new QueryCachingPolicy() { + @Override + public void onUse(Query query) { + + } @Override public boolean shouldCache(Query query) { return true; diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index 61b490e30d684..e155639f143c6 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -34,7 +34,6 @@ import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; -import org.apache.lucene.search.UsageTrackingQueryCachingPolicy; import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; import org.elasticsearch.core.internal.io.IOUtils; @@ -91,7 +90,11 @@ public boolean isCacheable(LeafReaderContext ctx) { } private static QueryCachingPolicy alwaysCachePolicy() { - return new UsageTrackingQueryCachingPolicy() { + return new QueryCachingPolicy() { + @Override + public void onUse(Query query) { + + } @Override public boolean shouldCache(Query query) { return true;