diff --git a/build.gradle b/build.gradle index 5c1fe80668283..8ee0712383367 100644 --- a/build.gradle +++ b/build.gradle @@ -160,8 +160,8 @@ task verifyVersions { * after the backport of the backcompat code is complete. */ -boolean bwc_tests_enabled = true -final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ +boolean bwc_tests_enabled = false +final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/40416" /* place a PR link here when committing bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java index f68a731edf8f7..568ddfe97df16 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java @@ -51,10 +51,12 @@ import java.nio.file.Path; import java.security.GeneralSecurityException; import java.security.KeyStore; +import java.security.MessageDigest; import java.security.SecureRandom; import java.util.ArrayList; import java.util.Base64; import java.util.List; +import java.util.Locale; import java.util.Set; import static org.hamcrest.Matchers.containsString; @@ -126,6 +128,27 @@ public void testCannotReadStringFromClosedKeystore() throws Exception { assertThat(exception.getMessage(), containsString("closed")); } + public void testValueSHA256Digest() throws Exception { + final KeyStoreWrapper keystore = KeyStoreWrapper.create(); + final String stringSettingKeyName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT) + "1"; + final String stringSettingValue = randomAlphaOfLength(32); + keystore.setString(stringSettingKeyName, stringSettingValue.toCharArray()); + final String fileSettingKeyName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT) + "2"; + final byte[] fileSettingValue = randomByteArrayOfLength(32); + keystore.setFile(fileSettingKeyName, fileSettingValue); + + final byte[] stringSettingHash = MessageDigest.getInstance("SHA-256").digest(stringSettingValue.getBytes(StandardCharsets.UTF_8)); + assertThat(keystore.getSHA256Digest(stringSettingKeyName), equalTo(stringSettingHash)); + final byte[] fileSettingHash = MessageDigest.getInstance("SHA-256").digest(fileSettingValue); + assertThat(keystore.getSHA256Digest(fileSettingKeyName), equalTo(fileSettingHash)); + + keystore.close(); + + // value hashes accessible even when the keystore is closed + assertThat(keystore.getSHA256Digest(stringSettingKeyName), equalTo(stringSettingHash)); + assertThat(keystore.getSHA256Digest(fileSettingKeyName), equalTo(fileSettingHash)); + } + public void testUpgradeNoop() throws Exception { KeyStoreWrapper keystore = KeyStoreWrapper.create(); SecureString seed = keystore.getString(KeyStoreWrapper.SEED_SETTING.getKey()); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DiffableStringMap.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DiffableStringMap.java index 46433eed8a657..b6e31e92698e6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DiffableStringMap.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DiffableStringMap.java @@ -39,6 +39,8 @@ */ public class DiffableStringMap extends AbstractMap implements Diffable { + public static final DiffableStringMap EMPTY = new DiffableStringMap(Collections.emptyMap()); + private final Map innerMap; DiffableStringMap(final Map map) { @@ -75,6 +77,8 @@ public static Diff readDiffFrom(StreamInput in) throws IOExce */ public static class DiffableStringMapDiff implements Diff { + public static final DiffableStringMapDiff EMPTY = new DiffableStringMapDiff(DiffableStringMap.EMPTY, DiffableStringMap.EMPTY); + private final List deletes; private final Map upserts; // diffs also become upserts diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index c0c2bd82295ec..4c2eeec72a970 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -25,6 +25,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.Version; import org.elasticsearch.action.AliasesRequest; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterState.FeatureAware; @@ -168,6 +169,7 @@ public interface Custom extends NamedDiffable, ToXContentFragment, Clust private final Settings transientSettings; private final Settings persistentSettings; private final Settings settings; + private final DiffableStringMap hashesOfConsistentSettings; private final ImmutableOpenMap indices; private final ImmutableOpenMap templates; private final ImmutableOpenMap customs; @@ -182,7 +184,7 @@ public interface Custom extends NamedDiffable, ToXContentFragment, Clust private final SortedMap aliasAndIndexLookup; MetaData(String clusterUUID, boolean clusterUUIDCommitted, long version, CoordinationMetaData coordinationMetaData, - Settings transientSettings, Settings persistentSettings, + Settings transientSettings, Settings persistentSettings, DiffableStringMap hashesOfConsistentSettings, ImmutableOpenMap indices, ImmutableOpenMap templates, ImmutableOpenMap customs, String[] allIndices, String[] allOpenIndices, String[] allClosedIndices, SortedMap aliasAndIndexLookup) { @@ -193,6 +195,7 @@ public interface Custom extends NamedDiffable, ToXContentFragment, Clust this.transientSettings = transientSettings; this.persistentSettings = persistentSettings; this.settings = Settings.builder().put(persistentSettings).put(transientSettings).build(); + this.hashesOfConsistentSettings = hashesOfConsistentSettings; this.indices = indices; this.customs = customs; this.templates = templates; @@ -244,6 +247,10 @@ public Settings persistentSettings() { return this.persistentSettings; } + public Map hashesOfConsistentSettings() { + return this.hashesOfConsistentSettings; + } + public CoordinationMetaData coordinationMetaData() { return this.coordinationMetaData; } @@ -733,6 +740,9 @@ public static boolean isGlobalStateEquals(MetaData metaData1, MetaData metaData2 if (!metaData1.persistentSettings.equals(metaData2.persistentSettings)) { return false; } + if (!metaData1.hashesOfConsistentSettings.equals(metaData2.hashesOfConsistentSettings)) { + return false; + } if (!metaData1.templates.equals(metaData2.templates())) { return false; } @@ -787,6 +797,7 @@ private static class MetaDataDiff implements Diff { private CoordinationMetaData coordinationMetaData; private Settings transientSettings; private Settings persistentSettings; + private Diff hashesOfConsistentSettings; private Diff> indices; private Diff> templates; private Diff> customs; @@ -798,6 +809,7 @@ private static class MetaDataDiff implements Diff { coordinationMetaData = after.coordinationMetaData; transientSettings = after.transientSettings; persistentSettings = after.persistentSettings; + hashesOfConsistentSettings = after.hashesOfConsistentSettings.diff(before.hashesOfConsistentSettings); indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer()); templates = DiffableUtils.diff(before.templates, after.templates, DiffableUtils.getStringKeySerializer()); customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER); @@ -810,6 +822,11 @@ private static class MetaDataDiff implements Diff { coordinationMetaData = new CoordinationMetaData(in); transientSettings = Settings.readSettingsFromStream(in); persistentSettings = Settings.readSettingsFromStream(in); + if (in.getVersion().onOrAfter(Version.V_7_3_0)) { + hashesOfConsistentSettings = DiffableStringMap.readDiffFrom(in); + } else { + hashesOfConsistentSettings = DiffableStringMap.DiffableStringMapDiff.EMPTY; + } indices = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexMetaData::readFrom, IndexMetaData::readDiffFrom); templates = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexTemplateMetaData::readFrom, @@ -825,6 +842,9 @@ public void writeTo(StreamOutput out) throws IOException { coordinationMetaData.writeTo(out); Settings.writeSettingsToStream(transientSettings, out); Settings.writeSettingsToStream(persistentSettings, out); + if (out.getVersion().onOrAfter(Version.V_7_3_0)) { + hashesOfConsistentSettings.writeTo(out); + } indices.writeTo(out); templates.writeTo(out); customs.writeTo(out); @@ -839,6 +859,7 @@ public MetaData apply(MetaData part) { builder.coordinationMetaData(coordinationMetaData); builder.transientSettings(transientSettings); builder.persistentSettings(persistentSettings); + builder.hashesOfConsistentSettings(hashesOfConsistentSettings.apply(part.hashesOfConsistentSettings)); builder.indices(indices.apply(part.indices)); builder.templates(templates.apply(part.templates)); builder.customs(customs.apply(part.customs)); @@ -854,6 +875,9 @@ public static MetaData readFrom(StreamInput in) throws IOException { builder.coordinationMetaData(new CoordinationMetaData(in)); builder.transientSettings(readSettingsFromStream(in)); builder.persistentSettings(readSettingsFromStream(in)); + if (in.getVersion().onOrAfter(Version.V_7_3_0)) { + builder.hashesOfConsistentSettings(new DiffableStringMap(in)); + } int size = in.readVInt(); for (int i = 0; i < size; i++) { builder.put(IndexMetaData.readFrom(in), false); @@ -878,6 +902,9 @@ public void writeTo(StreamOutput out) throws IOException { coordinationMetaData.writeTo(out); writeSettingsToStream(transientSettings, out); writeSettingsToStream(persistentSettings, out); + if (out.getVersion().onOrAfter(Version.V_7_3_0)) { + hashesOfConsistentSettings.writeTo(out); + } out.writeVInt(indices.size()); for (IndexMetaData indexMetaData : this) { indexMetaData.writeTo(out); @@ -918,6 +945,7 @@ public static class Builder { private CoordinationMetaData coordinationMetaData = CoordinationMetaData.EMPTY_META_DATA; private Settings transientSettings = Settings.Builder.EMPTY_SETTINGS; private Settings persistentSettings = Settings.Builder.EMPTY_SETTINGS; + private DiffableStringMap hashesOfConsistentSettings = new DiffableStringMap(Collections.emptyMap()); private final ImmutableOpenMap.Builder indices; private final ImmutableOpenMap.Builder templates; @@ -937,6 +965,7 @@ public Builder(MetaData metaData) { this.coordinationMetaData = metaData.coordinationMetaData; this.transientSettings = metaData.transientSettings; this.persistentSettings = metaData.persistentSettings; + this.hashesOfConsistentSettings = metaData.hashesOfConsistentSettings; this.version = metaData.version; this.indices = ImmutableOpenMap.builder(metaData.indices); this.templates = ImmutableOpenMap.builder(metaData.templates); @@ -1100,6 +1129,20 @@ public Builder persistentSettings(Settings settings) { return this; } + public DiffableStringMap hashesOfConsistentSettings() { + return this.hashesOfConsistentSettings; + } + + public Builder hashesOfConsistentSettings(DiffableStringMap hashesOfConsistentSettings) { + this.hashesOfConsistentSettings = hashesOfConsistentSettings; + return this; + } + + public Builder hashesOfConsistentSettings(Map hashesOfConsistentSettings) { + this.hashesOfConsistentSettings = new DiffableStringMap(hashesOfConsistentSettings); + return this; + } + public Builder version(long version) { this.version = version; return this; @@ -1173,8 +1216,8 @@ public MetaData build() { String[] allClosedIndicesArray = allClosedIndices.toArray(new String[allClosedIndices.size()]); return new MetaData(clusterUUID, clusterUUIDCommitted, version, coordinationMetaData, transientSettings, persistentSettings, - indices.build(), templates.build(), customs.build(), allIndicesArray, allOpenIndicesArray, allClosedIndicesArray, - aliasAndIndexLookup); + hashesOfConsistentSettings, indices.build(), templates.build(), customs.build(), allIndicesArray, allOpenIndicesArray, + allClosedIndicesArray, aliasAndIndexLookup); } private SortedMap buildAliasAndIndexLookup() { @@ -1298,6 +1341,8 @@ public static MetaData fromXContent(XContentParser parser) throws IOException { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { builder.put(IndexMetaData.Builder.fromXContent(parser), false); } + } else if ("hashes_of_consistent_settings".equals(currentFieldName)) { + builder.hashesOfConsistentSettings(parser.mapStrings()); } else if ("templates".equals(currentFieldName)) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { builder.put(IndexTemplateMetaData.Builder.fromXContent(parser, parser.currentName())); diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java index f83f2606b14b6..fded43a4bdd19 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java @@ -73,7 +73,7 @@ public ClusterService(Settings settings, ClusterSettings clusterSettings, Thread } public ClusterService(Settings settings, ClusterSettings clusterSettings, MasterService masterService, - ClusterApplierService clusterApplierService) { + ClusterApplierService clusterApplierService) { this.settings = settings; this.nodeName = Node.NODE_NAME_SETTING.get(settings); this.masterService = masterService; diff --git a/server/src/main/java/org/elasticsearch/common/hash/MessageDigests.java b/server/src/main/java/org/elasticsearch/common/hash/MessageDigests.java index 8bcef7b8ff4cb..df8f3e2fa7f43 100644 --- a/server/src/main/java/org/elasticsearch/common/hash/MessageDigests.java +++ b/server/src/main/java/org/elasticsearch/common/hash/MessageDigests.java @@ -95,15 +95,24 @@ private static MessageDigest get(ThreadLocal messageDigest) { * @return a hex representation of the input as a String. */ public static String toHexString(byte[] bytes) { - Objects.requireNonNull(bytes); - StringBuilder sb = new StringBuilder(2 * bytes.length); + return new String(toHexCharArray(bytes)); + } + /** + * Encodes the byte array into a newly created hex char array, without allocating any other temporary variables. + * + * @param bytes the input to be encoded as hex. + * @return the hex encoding of the input as a char array. + */ + public static char[] toHexCharArray(byte[] bytes) { + Objects.requireNonNull(bytes); + final char[] result = new char[2 * bytes.length]; for (int i = 0; i < bytes.length; i++) { byte b = bytes[i]; - sb.append(HEX_DIGITS[b >> 4 & 0xf]).append(HEX_DIGITS[b & 0xf]); + result[2 * i] = HEX_DIGITS[b >> 4 & 0xf]; + result[2 * i + 1] = HEX_DIGITS[b & 0xf]; } - - return sb.toString(); + return result; } } diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index b1bb84cff4d6f..867b628a5f97c 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -112,12 +112,12 @@ * Encapsulates all valid cluster level settings. */ public final class ClusterSettings extends AbstractScopedSettings { + public ClusterSettings(final Settings nodeSettings, final Set> settingsSet) { this(nodeSettings, settingsSet, Collections.emptySet()); } - public ClusterSettings( - final Settings nodeSettings, final Set> settingsSet, final Set> settingUpgraders) { + public ClusterSettings(final Settings nodeSettings, final Set> settingsSet, final Set> settingUpgraders) { super(nodeSettings, settingsSet, settingUpgraders, Property.NodeScope); addSettingsUpdater(new LoggingSettingUpdater(nodeSettings)); } diff --git a/server/src/main/java/org/elasticsearch/common/settings/ConsistentSettingsService.java b/server/src/main/java/org/elasticsearch/common/settings/ConsistentSettingsService.java new file mode 100644 index 0000000000000..411a470238638 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/settings/ConsistentSettingsService.java @@ -0,0 +1,256 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.settings; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.LocalNodeMasterListener; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.threadpool.ThreadPool; + +import java.nio.charset.StandardCharsets; +import java.security.NoSuchAlgorithmException; +import java.security.spec.InvalidKeySpecException; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; + +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; + +/** + * Used to publish secure setting hashes in the cluster state and to validate those hashes against the local values of those same settings. + * This is colloquially referred to as the secure setting consistency check. It will publish and verify hashes only for the collection + * of settings passed in the constructor. The settings have to have the {@link Setting.Property#Consistent} property. + */ +public final class ConsistentSettingsService { + private static final Logger logger = LogManager.getLogger(ConsistentSettingsService.class); + + private final Settings settings; + private final ClusterService clusterService; + private final Collection> secureSettingsCollection; + private final SecretKeyFactory pbkdf2KeyFactory; + + public ConsistentSettingsService(Settings settings, ClusterService clusterService, + Collection> secureSettingsCollection) { + this.settings = settings; + this.clusterService = clusterService; + this.secureSettingsCollection = secureSettingsCollection; + // this is used to compute the PBKDF2 hash (the published one) + try { + this.pbkdf2KeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA512"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("The \"PBKDF2WithHmacSHA512\" algorithm is required for consistent secure settings' hashes", e); + } + } + + /** + * Returns a {@link LocalNodeMasterListener} that will publish hashes of all the settings passed in the constructor. These hashes are + * published by the master node only. Note that this is not designed for {@link SecureSettings} implementations that are mutable. + */ + public LocalNodeMasterListener newHashPublisher() { + // eagerly compute hashes to be published + final Map computedHashesOfConsistentSettings = computeHashesOfConsistentSecureSettings(); + return new HashesPublisher(computedHashesOfConsistentSettings, clusterService); + } + + /** + * Verifies that the hashes of consistent secure settings in the latest {@code ClusterState} verify for the values of those same + * settings on the local node. The settings to be checked are passed in the constructor. Also, validates that a missing local + * value is also missing in the published set, and vice-versa. + */ + public boolean areAllConsistent() { + final ClusterState state = clusterService.state(); + final Map publishedHashesOfConsistentSettings = state.metaData().hashesOfConsistentSettings(); + final Set publishedSettingKeysToVerify = new HashSet<>(); + publishedSettingKeysToVerify.addAll(publishedHashesOfConsistentSettings.keySet()); + final AtomicBoolean allConsistent = new AtomicBoolean(true); + forEachConcreteSecureSettingDo(concreteSecureSetting -> { + final String publishedSaltAndHash = publishedHashesOfConsistentSettings.get(concreteSecureSetting.getKey()); + final byte[] localHash = concreteSecureSetting.getSecretDigest(settings); + if (publishedSaltAndHash == null && localHash == null) { + // consistency of missing + logger.debug("no published hash for the consistent secure setting [{}] but it also does NOT exist on the local node", + concreteSecureSetting.getKey()); + } else if (publishedSaltAndHash == null && localHash != null) { + // setting missing on master but present locally + logger.warn("no published hash for the consistent secure setting [{}] but it exists on the local node", + concreteSecureSetting.getKey()); + if (state.nodes().isLocalNodeElectedMaster()) { + throw new IllegalStateException("Master node cannot validate consistent setting. No published hash for [" + + concreteSecureSetting.getKey() + "] but setting exists."); + } + allConsistent.set(false); + } else if (publishedSaltAndHash != null && localHash == null) { + // setting missing locally but present on master + logger.warn("the consistent secure setting [{}] does not exist on the local node but there is a published hash for it", + concreteSecureSetting.getKey()); + allConsistent.set(false); + } else { + assert publishedSaltAndHash != null; + assert localHash != null; + final String[] parts = publishedSaltAndHash.split(":"); + if (parts == null || parts.length != 2) { + throw new IllegalArgumentException("published hash [" + publishedSaltAndHash + " ] for secure setting [" + + concreteSecureSetting.getKey() + "] is invalid"); + } + final String publishedSalt = parts[0]; + final String publishedHash = parts[1]; + final byte[] computedSaltedHashBytes = computeSaltedPBKDF2Hash(localHash, publishedSalt.getBytes(StandardCharsets.UTF_8)); + final String computedSaltedHash = new String(Base64.getEncoder().encode(computedSaltedHashBytes), StandardCharsets.UTF_8); + if (false == publishedHash.equals(computedSaltedHash)) { + logger.warn("the published hash [{}] of the consistent secure setting [{}] differs from the locally computed one [{}]", + publishedHash, concreteSecureSetting.getKey(), computedSaltedHash); + if (state.nodes().isLocalNodeElectedMaster()) { + throw new IllegalStateException("Master node cannot validate consistent setting. The published hash [" + + publishedHash + "] of the consistent secure setting [" + concreteSecureSetting.getKey() + + "] differs from the locally computed one [" + computedSaltedHash + "]."); + } + allConsistent.set(false); + } + } + publishedSettingKeysToVerify.remove(concreteSecureSetting.getKey()); + }); + // another case of settings missing locally, when group settings have not expanded to all the keys published + for (String publishedSettingKey : publishedSettingKeysToVerify) { + for (Setting setting : secureSettingsCollection) { + if (setting.match(publishedSettingKey)) { + // setting missing locally but present on master + logger.warn("the consistent secure setting [{}] does not exist on the local node but there is a published hash for it", + publishedSettingKey); + allConsistent.set(false); + } + } + } + return allConsistent.get(); + } + + /** + * Iterate over the passed in secure settings, expanding {@link Setting.AffixSetting} to concrete settings, in the scope of the local + * settings. + */ + private void forEachConcreteSecureSettingDo(Consumer> secureSettingConsumer) { + for (Setting setting : secureSettingsCollection) { + assert setting.isConsistent() : "[" + setting.getKey() + "] is not a consistent setting"; + if (setting instanceof Setting.AffixSetting) { + ((Setting.AffixSetting)setting).getAllConcreteSettings(settings).forEach(concreteSetting -> { + assert concreteSetting instanceof SecureSetting : "[" + concreteSetting.getKey() + "] is not a secure setting"; + secureSettingConsumer.accept((SecureSetting)concreteSetting); + }); + } else if (setting instanceof SecureSetting) { + secureSettingConsumer.accept((SecureSetting) setting); + } else { + assert false : "Unrecognized consistent secure setting [" + setting.getKey() + "]"; + } + } + } + + private Map computeHashesOfConsistentSecureSettings() { + final Map hashesBySettingKey = new HashMap<>(); + forEachConcreteSecureSettingDo(concreteSecureSetting -> { + final byte[] localHash = concreteSecureSetting.getSecretDigest(settings); + if (localHash != null) { + final String salt = UUIDs.randomBase64UUID(); + final byte[] publicHash = computeSaltedPBKDF2Hash(localHash, salt.getBytes(StandardCharsets.UTF_8)); + final String encodedPublicHash = new String(Base64.getEncoder().encode(publicHash), StandardCharsets.UTF_8); + hashesBySettingKey.put(concreteSecureSetting.getKey(), salt + ":" + encodedPublicHash); + } + }); + return hashesBySettingKey; + } + + private byte[] computeSaltedPBKDF2Hash(byte[] bytes, byte[] salt) { + final int iterations = 5000; + final int keyLength = 512; + char[] value = null; + try { + value = MessageDigests.toHexCharArray(bytes); + final PBEKeySpec spec = new PBEKeySpec(value, salt, iterations, keyLength); + final SecretKey key = pbkdf2KeyFactory.generateSecret(spec); + return key.getEncoded(); + } catch (InvalidKeySpecException e) { + throw new RuntimeException("Unexpected exception when computing PBKDF2 hash", e); + } finally { + if (value != null) { + Arrays.fill(value, '0'); + } + } + } + + static final class HashesPublisher implements LocalNodeMasterListener { + + // eagerly compute hashes to be published + final Map computedHashesOfConsistentSettings; + final ClusterService clusterService; + + HashesPublisher(Map computedHashesOfConsistentSettings, ClusterService clusterService) { + this.computedHashesOfConsistentSettings = Map.copyOf(computedHashesOfConsistentSettings); + this.clusterService = clusterService; + } + + @Override + public void onMaster() { + clusterService.submitStateUpdateTask("publish-secure-settings-hashes", new ClusterStateUpdateTask(Priority.URGENT) { + @Override + public ClusterState execute(ClusterState currentState) { + final Map publishedHashesOfConsistentSettings = currentState.metaData() + .hashesOfConsistentSettings(); + if (computedHashesOfConsistentSettings.equals(publishedHashesOfConsistentSettings)) { + logger.debug("Nothing to publish. What is already published matches this node's view."); + return currentState; + } else { + return ClusterState.builder(currentState).metaData(MetaData.builder(currentState.metaData()) + .hashesOfConsistentSettings(computedHashesOfConsistentSettings)).build(); + } + } + + @Override + public void onFailure(String source, Exception e) { + logger.error("unable to publish secure settings hashes", e); + } + + }); + } + + @Override + public void offMaster() { + logger.trace("I am no longer master, nothing to do"); + } + + @Override + public String executorName() { + return ThreadPool.Names.SAME; + } + } + +} diff --git a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java index 64cdd7165f2a3..7ad69c1eebe0c 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java +++ b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java @@ -30,6 +30,7 @@ import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.cli.UserException; import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.hash.MessageDigests; import javax.crypto.Cipher; import javax.crypto.CipherInputStream; @@ -85,6 +86,17 @@ private enum EntryType { FILE } + /** An entry in the keystore. The bytes are opaque and interpreted based on the entry type. */ + private static class Entry { + final byte[] bytes; + final byte[] sha256Digest; + + Entry(byte[] bytes) { + this.bytes = bytes; + this.sha256Digest = MessageDigests.sha256().digest(bytes); + } + } + /** * A regex for the valid characters that a setting name in the keystore may use. */ @@ -148,7 +160,7 @@ private enum EntryType { private final byte[] dataBytes; /** The decrypted secret data. See {@link #decrypt(char[])}. */ - private final SetOnce> entries = new SetOnce<>(); + private final SetOnce> entries = new SetOnce<>(); private volatile boolean closed; private KeyStoreWrapper(int formatVersion, boolean hasPassword, byte[] dataBytes) { @@ -350,7 +362,7 @@ public void decrypt(char[] password) throws GeneralSecurityException, IOExceptio int entrySize = input.readInt(); byte[] entryBytes = new byte[entrySize]; input.readFully(entryBytes); - entries.get().put(setting, entryBytes); + entries.get().put(setting, new Entry(entryBytes)); } if (input.read() != -1) { throw new SecurityException("Keystore has been corrupted or tampered with"); @@ -369,11 +381,11 @@ private byte[] encrypt(char[] password, byte[] salt, byte[] iv) throws GeneralSe try (CipherOutputStream cipherStream = new CipherOutputStream(bytes, cipher); DataOutputStream output = new DataOutputStream(cipherStream)) { output.writeInt(entries.get().size()); - for (Map.Entry mapEntry : entries.get().entrySet()) { + for (Map.Entry mapEntry : entries.get().entrySet()) { output.writeUTF(mapEntry.getKey()); - byte[] entry = mapEntry.getValue(); - output.writeInt(entry.length); - output.write(entry); + byte[] entryBytes = mapEntry.getValue().bytes; + output.writeInt(entryBytes.length); + output.write(entryBytes); } } return bytes.toByteArray(); @@ -448,7 +460,7 @@ private void decryptLegacyEntries() throws GeneralSecurityException, IOException } Arrays.fill(chars, '\0'); - entries.get().put(setting, bytes); + entries.get().put(setting, new Entry(bytes)); } } @@ -521,8 +533,8 @@ public Set getSettingNames() { @Override public synchronized SecureString getString(String setting) { ensureOpen(); - byte[] entry = entries.get().get(setting); - ByteBuffer byteBuffer = ByteBuffer.wrap(entry); + Entry entry = entries.get().get(setting); + ByteBuffer byteBuffer = ByteBuffer.wrap(entry.bytes); CharBuffer charBuffer = StandardCharsets.UTF_8.decode(byteBuffer); return new SecureString(Arrays.copyOfRange(charBuffer.array(), charBuffer.position(), charBuffer.limit())); } @@ -530,8 +542,19 @@ public synchronized SecureString getString(String setting) { @Override public synchronized InputStream getFile(String setting) { ensureOpen(); - byte[] entry = entries.get().get(setting); - return new ByteArrayInputStream(entry); + Entry entry = entries.get().get(setting); + return new ByteArrayInputStream(entry.bytes); + } + + /** + * Returns the SHA256 digest for the setting's value, even after {@code #close()} has been called. The setting must exist. The digest is + * used to check for value changes without actually storing the value. + */ + @Override + public byte[] getSHA256Digest(String setting) { + assert entries.get() != null : "Keystore is not loaded"; + Entry entry = entries.get().get(setting); + return entry.sha256Digest; } /** @@ -553,9 +576,9 @@ synchronized void setString(String setting, char[] value) { ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(CharBuffer.wrap(value)); byte[] bytes = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.position(), byteBuffer.limit()); - byte[] oldEntry = entries.get().put(setting, bytes); + Entry oldEntry = entries.get().put(setting, new Entry(bytes)); if (oldEntry != null) { - Arrays.fill(oldEntry, (byte)0); + Arrays.fill(oldEntry.bytes, (byte)0); } } @@ -564,18 +587,18 @@ synchronized void setFile(String setting, byte[] bytes) { ensureOpen(); validateSettingName(setting); - byte[] oldEntry = entries.get().put(setting, Arrays.copyOf(bytes, bytes.length)); + Entry oldEntry = entries.get().put(setting, new Entry(Arrays.copyOf(bytes, bytes.length))); if (oldEntry != null) { - Arrays.fill(oldEntry, (byte)0); + Arrays.fill(oldEntry.bytes, (byte)0); } } /** Remove the given setting from the keystore. */ void remove(String setting) { ensureOpen(); - byte[] oldEntry = entries.get().remove(setting); + Entry oldEntry = entries.get().remove(setting); if (oldEntry != null) { - Arrays.fill(oldEntry, (byte)0); + Arrays.fill(oldEntry.bytes, (byte)0); } } @@ -590,8 +613,8 @@ private void ensureOpen() { public synchronized void close() { this.closed = true; if (null != entries.get() && entries.get().isEmpty() == false) { - for (byte[] entry : entries.get().values()) { - Arrays.fill(entry, (byte) 0); + for (Entry entry : entries.get().values()) { + Arrays.fill(entry.bytes, (byte) 0); } } } diff --git a/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java b/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java index 33f4718aa45e4..e022e4e3760a5 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java @@ -37,7 +37,7 @@ public abstract class SecureSetting extends Setting { /** Determines whether legacy settings with sensitive values should be allowed. */ private static final boolean ALLOW_INSECURE_SETTINGS = Booleans.parseBoolean(System.getProperty("es.allow_insecure_settings", "false")); - private static final Set ALLOWED_PROPERTIES = EnumSet.of(Property.Deprecated); + private static final Set ALLOWED_PROPERTIES = EnumSet.of(Property.Deprecated, Property.Consistent); private static final Property[] FIXED_PROPERTIES = { Property.NodeScope @@ -97,6 +97,23 @@ public T get(Settings settings) { } } + /** + * Returns the digest of this secure setting's value or {@code null} if the setting is missing (inside the keystore). This method can be + * called even after the {@code SecureSettings} have been closed, unlike {@code #get(Settings)}. The digest is used to check for changes + * of the value (by re-reading the {@code SecureSettings}), without actually transmitting the value to compare with. + */ + public byte[] getSecretDigest(Settings settings) { + final SecureSettings secureSettings = settings.getSecureSettings(); + if (secureSettings == null || false == secureSettings.getSettingNames().contains(getKey())) { + return null; + } + try { + return secureSettings.getSHA256Digest(getKey()); + } catch (GeneralSecurityException e) { + throw new RuntimeException("failed to read secure setting " + getKey(), e); + } + } + /** Returns the secret setting from the keyStoreReader store. */ abstract T getSecret(SecureSettings secureSettings) throws GeneralSecurityException; diff --git a/server/src/main/java/org/elasticsearch/common/settings/SecureSettings.java b/server/src/main/java/org/elasticsearch/common/settings/SecureSettings.java index 98f980c1ec6c8..7f92b382dd7b1 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/SecureSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/SecureSettings.java @@ -42,6 +42,8 @@ public interface SecureSettings extends Closeable { /** Return a file setting. The {@link InputStream} should be closed once it is used. */ InputStream getFile(String setting) throws GeneralSecurityException; + byte[] getSHA256Digest(String setting) throws GeneralSecurityException; + @Override void close() throws IOException; } diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index 1e5079124c345..d4164b474de0b 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -112,6 +112,11 @@ public enum Property { */ NodeScope, + /** + * Secure setting values equal on all nodes + */ + Consistent, + /** * Index scope */ @@ -167,6 +172,7 @@ private Setting(Key key, @Nullable Setting fallbackSetting, Function properties, } } + private void checkPropertyRequiresNodeScope(final EnumSet properties, final Property property) { + if (properties.contains(property) && properties.contains(Property.NodeScope) == false) { + throw new IllegalArgumentException("non-node-scoped setting [" + key + "] can not have property [" + property + "]"); + } + } + /** * Creates a new Setting instance * @param key the settings key for this setting. @@ -321,6 +333,14 @@ public boolean hasNodeScope() { return properties.contains(Property.NodeScope); } + /** + * Returns true if this setting's value can be checked for equality across all nodes. Only {@link SecureSetting} instances + * may have this qualifier. + */ + public boolean isConsistent() { + return properties.contains(Property.Consistent); + } + /** * Returns true if this setting has an index scope, otherwise false */ diff --git a/server/src/main/java/org/elasticsearch/common/settings/Settings.java b/server/src/main/java/org/elasticsearch/common/settings/Settings.java index 5789abf76d8b0..e92d53f801183 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -1324,15 +1324,20 @@ public Set getSettingNames() { } @Override - public SecureString getString(String setting) throws GeneralSecurityException{ + public SecureString getString(String setting) throws GeneralSecurityException { return delegate.getString(addPrefix.apply(setting)); } @Override - public InputStream getFile(String setting) throws GeneralSecurityException{ + public InputStream getFile(String setting) throws GeneralSecurityException { return delegate.getFile(addPrefix.apply(setting)); } + @Override + public byte[] getSHA256Digest(String setting) throws GeneralSecurityException { + return delegate.getSHA256Digest(addPrefix.apply(setting)); + } + @Override public void close() throws IOException { delegate.close(); diff --git a/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java b/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java index 6a78e81d7f3f4..58c9cbc520456 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java +++ b/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java @@ -49,6 +49,7 @@ public class SettingsModule implements Module { private final Set settingsFilterPattern = new HashSet<>(); private final Map> nodeSettings = new HashMap<>(); private final Map> indexSettings = new HashMap<>(); + private final Set> consistentSettings = new HashSet<>(); private final IndexScopedSettings indexScopedSettings; private final ClusterSettings clusterSettings; private final SettingsFilter settingsFilter; @@ -157,7 +158,6 @@ public void configure(Binder binder) { binder.bind(IndexScopedSettings.class).toInstance(indexScopedSettings); } - /** * Registers a new setting. This method should be used by plugins in order to expose any custom settings the plugin defines. * Unless a setting is registered the setting is unusable. If a setting is never the less specified the node will reject @@ -175,6 +175,19 @@ private void registerSetting(Setting setting) { if (existingSetting != null) { throw new IllegalArgumentException("Cannot register setting [" + setting.getKey() + "] twice"); } + if (setting.isConsistent()) { + if (setting instanceof Setting.AffixSetting) { + if (((Setting.AffixSetting)setting).getConcreteSettingForNamespace("_na_") instanceof SecureSetting) { + consistentSettings.add(setting); + } else { + throw new IllegalArgumentException("Invalid consistent secure setting [" + setting.getKey() + "]"); + } + } else if (setting instanceof SecureSetting) { + consistentSettings.add(setting); + } else { + throw new IllegalArgumentException("Invalid consistent secure setting [" + setting.getKey() + "]"); + } + } nodeSettings.put(setting.getKey(), setting); } if (setting.hasIndexScope()) { @@ -182,6 +195,9 @@ private void registerSetting(Setting setting) { if (existingSetting != null) { throw new IllegalArgumentException("Cannot register setting [" + setting.getKey() + "] twice"); } + if (setting.isConsistent()) { + throw new IllegalStateException("Consistent setting [" + setting.getKey() + "] cannot be index scoped"); + } indexSettings.put(setting.getKey(), setting); } } else { @@ -215,6 +231,10 @@ public ClusterSettings getClusterSettings() { return clusterSettings; } + public Set> getConsistentSettings() { + return consistentSettings; + } + public SettingsFilter getSettingsFilter() { return settingsFilter; } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 3fab82c3e9eb9..8b8c051eef9a0 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -74,6 +74,7 @@ import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.ConsistentSettingsService; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.SettingUpgrader; @@ -363,6 +364,9 @@ protected Node( final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool); clusterService.addStateApplier(scriptModule.getScriptService()); resourcesToClose.add(clusterService); + clusterService.addLocalNodeMasterListener( + new ConsistentSettingsService(settings, clusterService, settingsModule.getConsistentSettings()) + .newHashPublisher()); final IngestService ingestService = new IngestService(clusterService, threadPool, this.environment, scriptModule.getScriptService(), analysisModule.getAnalysisRegistry(), pluginsService.filterPlugins(IngestPlugin.class)); final DiskThresholdMonitor listener = new DiskThresholdMonitor(settings, clusterService::state, diff --git a/server/src/test/java/org/elasticsearch/common/settings/ConsistentSettingsIT.java b/server/src/test/java/org/elasticsearch/common/settings/ConsistentSettingsIT.java new file mode 100644 index 0000000000000..4ee0c6849c2bc --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/settings/ConsistentSettingsIT.java @@ -0,0 +1,188 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.settings; + +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Setting.AffixSetting; +import org.elasticsearch.env.Environment; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) +public class ConsistentSettingsIT extends ESIntegTestCase { + + static final Setting DUMMY_STRING_CONSISTENT_SETTING = SecureSetting + .secureString("dummy.consistent.secure.string.setting", null, Setting.Property.Consistent); + static final AffixSetting DUMMY_AFFIX_STRING_CONSISTENT_SETTING = Setting.affixKeySetting( + "dummy.consistent.secure.string.affix.setting.", "suffix", + key -> SecureSetting.secureString(key, null, Setting.Property.Consistent)); + private final AtomicReference> nodeSettingsOverride = new AtomicReference<>(null); + + public void testAllConsistentOnAllNodesSuccess() throws Exception { + for (String nodeName : internalCluster().getNodeNames()) { + Environment environment = internalCluster().getInstance(Environment.class, nodeName); + ClusterService clusterService = internalCluster().getInstance(ClusterService.class, nodeName); + assertTrue("Empty settings list always consistent.", + new ConsistentSettingsService(environment.settings(), clusterService, Collections.emptyList()).areAllConsistent()); + assertTrue( + "Simple consistent secure setting is consistent [" + clusterService.state().metaData().hashesOfConsistentSettings() + + "].", + new ConsistentSettingsService(environment.settings(), clusterService, + Collections.singletonList(DUMMY_STRING_CONSISTENT_SETTING)).areAllConsistent()); + assertTrue( + "Affix consistent secure setting is consistent [" + clusterService.state().metaData().hashesOfConsistentSettings() + + "].", + new ConsistentSettingsService(environment.settings(), clusterService, + Collections.singletonList(DUMMY_AFFIX_STRING_CONSISTENT_SETTING)).areAllConsistent()); + assertTrue("All secure settings are consistent [" + clusterService.state().metaData().hashesOfConsistentSettings() + "].", + new ConsistentSettingsService(environment.settings(), clusterService, + List.of(DUMMY_STRING_CONSISTENT_SETTING, DUMMY_AFFIX_STRING_CONSISTENT_SETTING)).areAllConsistent()); + } + } + + public void testConsistencyFailures() throws Exception { + nodeSettingsOverride.set(nodeOrdinal -> { + Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)); + MockSecureSettings secureSettings = new MockSecureSettings(); + if (randomBoolean()) { + // different value + secureSettings.setString("dummy.consistent.secure.string.setting", "DIFFERENT_VALUE"); + } else { + // missing value + // secureSettings.setString("dummy.consistent.secure.string.setting", "string_value"); + } + secureSettings.setString("dummy.consistent.secure.string.affix.setting." + "affix1" + ".suffix", "affix_value_1"); + secureSettings.setString("dummy.consistent.secure.string.affix.setting." + "affix2" + ".suffix", "affix_value_2"); + assert builder.getSecureSettings() == null : "Deal with the settings merge"; + builder.setSecureSettings(secureSettings); + return builder.build(); + }); + String newNodeName = internalCluster().startNode(); + Environment environment = internalCluster().getInstance(Environment.class, newNodeName); + ClusterService clusterService = internalCluster().getInstance(ClusterService.class, newNodeName); + assertTrue("Empty settings list always consistent.", + new ConsistentSettingsService(environment.settings(), clusterService, Collections.emptyList()).areAllConsistent()); + assertFalse( + "Simple consistent secure setting is NOT consistent [" + clusterService.state().metaData().hashesOfConsistentSettings() + + "].", + new ConsistentSettingsService(environment.settings(), clusterService, + Collections.singletonList(DUMMY_STRING_CONSISTENT_SETTING)).areAllConsistent()); + assertTrue( + "Affix consistent secure setting is consistent [" + clusterService.state().metaData().hashesOfConsistentSettings() + + "].", + new ConsistentSettingsService(environment.settings(), clusterService, + Collections.singletonList(DUMMY_AFFIX_STRING_CONSISTENT_SETTING)).areAllConsistent()); + assertFalse("All secure settings are NOT consistent [" + clusterService.state().metaData().hashesOfConsistentSettings() + "].", + new ConsistentSettingsService(environment.settings(), clusterService, + List.of(DUMMY_STRING_CONSISTENT_SETTING, DUMMY_AFFIX_STRING_CONSISTENT_SETTING)).areAllConsistent()); + nodeSettingsOverride.set(nodeOrdinal -> { + Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)); + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("dummy.consistent.secure.string.setting", "string_value"); + if (randomBoolean()) { + secureSettings.setString("dummy.consistent.secure.string.affix.setting." + "affix1" + ".suffix", "affix_value_1"); + if (randomBoolean()) { + secureSettings.setString("dummy.consistent.secure.string.affix.setting." + "affix2" + ".suffix", "DIFFERENT_VALUE"); + } else { + // missing value + // "dummy.consistent.secure.string.affix.setting.affix2.suffix" + } + } else { + if (randomBoolean()) { + secureSettings.setString("dummy.consistent.secure.string.affix.setting." + "affix1" + ".suffix", "DIFFERENT_VALUE_1"); + secureSettings.setString("dummy.consistent.secure.string.affix.setting." + "affix2" + ".suffix", "DIFFERENT_VALUE_2"); + } else { + // missing values + // dummy.consistent.secure.string.affix.setting.affix1.suffix + // dummy.consistent.secure.string.affix.setting.affix2.suffix + } + } + assert builder.getSecureSettings() == null : "Deal with the settings merge"; + builder.setSecureSettings(secureSettings); + return builder.build(); + }); + newNodeName = internalCluster().startNode(); + environment = internalCluster().getInstance(Environment.class, newNodeName); + clusterService = internalCluster().getInstance(ClusterService.class, newNodeName); + assertTrue("Empty settings list always consistent.", + new ConsistentSettingsService(environment.settings(), clusterService, Collections.emptyList()).areAllConsistent()); + assertTrue( + "Simple consistent secure setting is consistent [" + clusterService.state().metaData().hashesOfConsistentSettings() + + "].", + new ConsistentSettingsService(environment.settings(), clusterService, + Collections.singletonList(DUMMY_STRING_CONSISTENT_SETTING)).areAllConsistent()); + assertFalse( + "Affix consistent secure setting is NOT consistent [" + clusterService.state().metaData().hashesOfConsistentSettings() + + "].", + new ConsistentSettingsService(environment.settings(), clusterService, + Collections.singletonList(DUMMY_AFFIX_STRING_CONSISTENT_SETTING)).areAllConsistent()); + assertFalse("All secure settings are NOT consistent [" + clusterService.state().metaData().hashesOfConsistentSettings() + "].", + new ConsistentSettingsService(environment.settings(), clusterService, + List.of(DUMMY_STRING_CONSISTENT_SETTING, DUMMY_AFFIX_STRING_CONSISTENT_SETTING)).areAllConsistent()); + nodeSettingsOverride.set(null); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + Function nodeSettingsOverrideFunction = nodeSettingsOverride.get(); + if (nodeSettingsOverrideFunction != null) { + final Settings overrideSettings = nodeSettingsOverrideFunction.apply(nodeOrdinal); + if (overrideSettings != null) { + return overrideSettings; + } + } + Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)); + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("dummy.consistent.secure.string.setting", "string_value"); + secureSettings.setString("dummy.consistent.secure.string.affix.setting." + "affix1" + ".suffix", "affix_value_1"); + secureSettings.setString("dummy.consistent.secure.string.affix.setting." + "affix2" + ".suffix", "affix_value_2"); + assert builder.getSecureSettings() == null : "Deal with the settings merge"; + builder.setSecureSettings(secureSettings); + return builder.build(); + } + + @Override + protected Collection> nodePlugins() { + Collection> classes = new ArrayList<>(super.nodePlugins()); + classes.add(DummyPlugin.class); + return classes; + } + + public static final class DummyPlugin extends Plugin { + + public DummyPlugin() { + } + + @Override + public List> getSettings() { + List> settings = new ArrayList<>(super.getSettings()); + settings.add(DUMMY_STRING_CONSISTENT_SETTING); + settings.add(DUMMY_AFFIX_STRING_CONSISTENT_SETTING); + return settings; + } + } +} diff --git a/server/src/test/java/org/elasticsearch/common/settings/ConsistentSettingsServiceTests.java b/server/src/test/java/org/elasticsearch/common/settings/ConsistentSettingsServiceTests.java new file mode 100644 index 0000000000000..687b74e3397cb --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/settings/ConsistentSettingsServiceTests.java @@ -0,0 +1,159 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.settings; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.mock.orig.Mockito; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; +import org.mockito.stubbing.Answer; + +import java.util.List; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicReference; + +import static org.mockito.Mockito.mock; +import static org.hamcrest.Matchers.is; + +public class ConsistentSettingsServiceTests extends ESTestCase { + + private AtomicReference clusterState = new AtomicReference<>(); + private ClusterService clusterService; + + @Before + public void init() throws Exception { + clusterState.set(ClusterState.EMPTY_STATE); + clusterService = mock(ClusterService.class); + Mockito.doAnswer((Answer) invocation -> { + return clusterState.get(); + }).when(clusterService).state(); + Mockito.doAnswer((Answer) invocation -> { + final ClusterStateUpdateTask arg0 = (ClusterStateUpdateTask) invocation.getArguments()[1]; + this.clusterState.set(arg0.execute(this.clusterState.get())); + return null; + }).when(clusterService).submitStateUpdateTask(Mockito.isA(String.class), Mockito.isA(ClusterStateUpdateTask.class)); + } + + public void testSingleStringSetting() throws Exception { + Setting stringSetting = SecureSetting.secureString("test.simple.foo", null, Setting.Property.Consistent); + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString(stringSetting.getKey(), "somethingsecure"); + secureSettings.setString("test.noise.setting", "noise"); + Settings.Builder builder = Settings.builder(); + builder.setSecureSettings(secureSettings); + Settings settings = builder.build(); + // hashes not yet published + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).areAllConsistent(), is(false)); + // publish + new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).newHashPublisher().onMaster(); + ConsistentSettingsService consistentService = new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)); + assertThat(consistentService.areAllConsistent(), is(true)); + // change value + secureSettings.setString(stringSetting.getKey(), "_TYPO_somethingsecure"); + assertThat(consistentService.areAllConsistent(), is(false)); + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).areAllConsistent(), is(false)); + // publish change + new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).newHashPublisher().onMaster(); + assertThat(consistentService.areAllConsistent(), is(true)); + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).areAllConsistent(), is(true)); + } + + public void testSingleAffixSetting() throws Exception { + Setting.AffixSetting affixStringSetting = Setting.affixKeySetting("test.affix.", "bar", + (key) -> SecureSetting.secureString(key, null, Setting.Property.Consistent)); + // add two affix settings to the keystore + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("test.noise.setting", "noise"); + secureSettings.setString("test.affix.first.bar", "first_secure"); + secureSettings.setString("test.affix.second.bar", "second_secure"); + Settings.Builder builder = Settings.builder(); + builder.setSecureSettings(secureSettings); + Settings settings = builder.build(); + // hashes not yet published + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(false)); + // publish + new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).newHashPublisher().onMaster(); + ConsistentSettingsService consistentService = new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)); + assertThat(consistentService.areAllConsistent(), is(true)); + // change value + secureSettings.setString("test.affix.second.bar", "_TYPO_second_secure"); + assertThat(consistentService.areAllConsistent(), is(false)); + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(false)); + // publish change + new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).newHashPublisher().onMaster(); + assertThat(consistentService.areAllConsistent(), is(true)); + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(true)); + // add value + secureSettings.setString("test.affix.third.bar", "third_secure"); + builder = Settings.builder(); + builder.setSecureSettings(secureSettings); + settings = builder.build(); + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(false)); + // publish + new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).newHashPublisher().onMaster(); + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(true)); + // remove value + secureSettings = new MockSecureSettings(); + secureSettings.setString("test.another.noise.setting", "noise"); + // missing value test.affix.first.bar + secureSettings.setString("test.affix.second.bar", "second_secure"); + secureSettings.setString("test.affix.third.bar", "third_secure"); + builder = Settings.builder(); + builder.setSecureSettings(secureSettings); + settings = builder.build(); + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(false)); + } + + public void testStringAndAffixSettings() throws Exception { + Setting stringSetting = SecureSetting.secureString("mock.simple.foo", null, Setting.Property.Consistent); + Setting.AffixSetting affixStringSetting = Setting.affixKeySetting("mock.affix.", "bar", + (key) -> SecureSetting.secureString(key, null, Setting.Property.Consistent)); + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString(randomAlphaOfLength(8).toLowerCase(Locale.ROOT), "noise"); + secureSettings.setString(stringSetting.getKey(), "somethingsecure"); + secureSettings.setString("mock.affix.foo.bar", "another_secure"); + Settings.Builder builder = Settings.builder(); + builder.setSecureSettings(secureSettings); + Settings settings = builder.build(); + // hashes not yet published + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting, affixStringSetting)).areAllConsistent(), + is(false)); + // publish only the simple string setting + new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).newHashPublisher().onMaster(); + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).areAllConsistent(), is(true)); + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(false)); + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting, affixStringSetting)).areAllConsistent(), + is(false)); + // publish only the affix string setting + new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).newHashPublisher().onMaster(); + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).areAllConsistent(), is(false)); + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(true)); + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting, affixStringSetting)).areAllConsistent(), + is(false)); + // publish both settings + new ConsistentSettingsService(settings, clusterService, List.of(stringSetting, affixStringSetting)).newHashPublisher().onMaster(); + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).areAllConsistent(), is(true)); + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(true)); + assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting, affixStringSetting)).areAllConsistent(), + is(true)); + } +} diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java index c6182eac8f680..c374984eb5d15 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java @@ -21,11 +21,13 @@ import org.elasticsearch.common.inject.ModuleTestCase; import org.elasticsearch.common.settings.Setting.Property; +import org.hamcrest.Matchers; import java.util.Arrays; import static java.util.Collections.emptySet; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; public class SettingsModuleTests extends ModuleTestCase { @@ -85,6 +87,40 @@ public void testRegisterSettings() { } } + public void testRegisterConsistentSettings() { + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("some.custom.secure.consistent.setting", "secure_value"); + final Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); + final Setting concreteConsistentSetting = SecureSetting.secureString("some.custom.secure.consistent.setting", null, + Setting.Property.Consistent); + SettingsModule module = new SettingsModule(settings, concreteConsistentSetting); + assertInstanceBinding(module, Settings.class, (s) -> s == settings); + assertThat(module.getConsistentSettings(), Matchers.containsInAnyOrder(concreteConsistentSetting)); + + final Setting concreteUnsecureConsistentSetting = Setting.simpleString("some.custom.UNSECURE.consistent.setting", + Property.Consistent, Property.NodeScope); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new SettingsModule(Settings.builder().build(), concreteUnsecureConsistentSetting)); + assertThat(e.getMessage(), is("Invalid consistent secure setting [some.custom.UNSECURE.consistent.setting]")); + + secureSettings = new MockSecureSettings(); + secureSettings.setString("some.custom.secure.consistent.afix.wow.setting", "secure_value"); + final Settings settings2 = Settings.builder().setSecureSettings(secureSettings).build(); + final Setting afixConcreteConsistentSetting = Setting.affixKeySetting( + "some.custom.secure.consistent.afix.", "setting", + key -> SecureSetting.secureString(key, null, Setting.Property.Consistent)); + module = new SettingsModule(settings2,afixConcreteConsistentSetting); + assertInstanceBinding(module, Settings.class, (s) -> s == settings2); + assertThat(module.getConsistentSettings(), Matchers.containsInAnyOrder(afixConcreteConsistentSetting)); + + final Setting concreteUnsecureConsistentAfixSetting = Setting.affixKeySetting( + "some.custom.secure.consistent.afix.", "setting", + key -> Setting.simpleString(key, Setting.Property.Consistent, Property.NodeScope)); + e = expectThrows(IllegalArgumentException.class, + () -> new SettingsModule(Settings.builder().build(), concreteUnsecureConsistentAfixSetting)); + assertThat(e.getMessage(), is("Invalid consistent secure setting [some.custom.secure.consistent.afix.*.setting]")); + } + public void testLoggerSettings() { { Settings settings = Settings.builder().put("logger._root", "TRACE").put("logger.transport", "INFO").build(); diff --git a/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java b/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java index 3a6161a9f7fa0..84689cf223d20 100644 --- a/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java +++ b/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java @@ -19,9 +19,12 @@ package org.elasticsearch.common.settings; +import org.elasticsearch.common.hash.MessageDigests; + import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -35,6 +38,7 @@ public class MockSecureSettings implements SecureSettings { private Map secureStrings = new HashMap<>(); private Map files = new HashMap<>(); + private Map sha256Digests = new HashMap<>(); private Set settingNames = new HashSet<>(); private final AtomicBoolean closed = new AtomicBoolean(false); @@ -44,6 +48,7 @@ public MockSecureSettings() { private MockSecureSettings(MockSecureSettings source) { secureStrings.putAll(source.secureStrings); files.putAll(source.files); + sha256Digests.putAll(source.sha256Digests); settingNames.addAll(source.settingNames); } @@ -69,15 +74,22 @@ public InputStream getFile(String setting) { return new ByteArrayInputStream(files.get(setting)); } + @Override + public byte[] getSHA256Digest(String setting) { + return sha256Digests.get(setting); + } + public void setString(String setting, String value) { ensureOpen(); secureStrings.put(setting, new SecureString(value.toCharArray())); + sha256Digests.put(setting, MessageDigests.sha256().digest(value.getBytes(StandardCharsets.UTF_8))); settingNames.add(setting); } public void setFile(String setting, byte[] value) { ensureOpen(); files.put(setting, value); + sha256Digests.put(setting, MessageDigests.sha256().digest(value)); settingNames.add(setting); } @@ -90,6 +102,7 @@ public void merge(MockSecureSettings secureSettings) { } settingNames.addAll(secureSettings.settingNames); secureStrings.putAll(secureSettings.secureStrings); + sha256Digests.putAll(secureSettings.sha256Digests); files.putAll(secureSettings.files); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java index c2a079e519f0f..c6c041a6571b1 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.SecureSettings; import org.elasticsearch.common.settings.SecureString; @@ -179,12 +180,13 @@ private static SecureSettings extractSecureSettings(Settings source, List cache = new HashMap<>(); + final Map> cache = new HashMap<>(); if (sourceSecureSettings != null && securePluginSettings != null) { for (final String settingKey : sourceSecureSettings.getSettingNames()) { for (final Setting secureSetting : securePluginSettings) { if (secureSetting.match(settingKey)) { - cache.put(settingKey, sourceSecureSettings.getString(settingKey)); + cache.put(settingKey, + new Tuple<>(sourceSecureSettings.getString(settingKey), sourceSecureSettings.getSHA256Digest(settingKey))); } } } @@ -197,8 +199,8 @@ public boolean isLoaded() { } @Override - public SecureString getString(String setting) throws GeneralSecurityException { - return cache.get(setting); + public SecureString getString(String setting) { + return cache.get(setting).v1(); } @Override @@ -207,10 +209,15 @@ public Set getSettingNames() { } @Override - public InputStream getFile(String setting) throws GeneralSecurityException { + public InputStream getFile(String setting) { throw new IllegalStateException("A NotificationService setting cannot be File."); } + @Override + public byte[] getSHA256Digest(String setting) { + return cache.get(setting).v2(); + } + @Override public void close() throws IOException { } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java index efbefdd640893..0fa05e900e518 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.watcher.notification; +import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureSettings; import org.elasticsearch.common.settings.SecureString; @@ -16,6 +17,7 @@ import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.security.GeneralSecurityException; import java.util.Arrays; import java.util.Collections; @@ -247,6 +249,11 @@ public InputStream getFile(String setting) throws GeneralSecurityException { return null; } + @Override + public byte[] getSHA256Digest(String setting) throws GeneralSecurityException { + return MessageDigests.sha256().digest(new String(secureSettingsMap.get(setting)).getBytes(StandardCharsets.UTF_8)); + } + @Override public void close() throws IOException { }