Skip to content

Commit 26c2347

Browse files
albertzaharovitsalbertzaharovits
authored
Reload secure settings for plugins (#31481)
Adds the ability to reread and decrypt the local node keystore. Commonly, the contents of the keystore, backing the `SecureSettings`, are not retrievable except during node initialization. This changes that by adding a new API which broadcasts a password to every node. The password is used to decrypt the local keystore and use it to populate a `Settings` object that is passes to all the plugins implementing the `ReloadablePlugin` interface. The plugin is then responsible to do whatever "reload" means in his case. When the `reload`handler returns, the keystore is closed and its contents are no longer retrievable. Password is never stored persistently on any node. Plugins that have been moded in this commit are: `repository-azure`, `repository-s3`, `repository-gcs` and `discovery-ec2`.
1 parent afff380 commit 26c2347

File tree

69 files changed

+3551
-1286
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+3551
-1286
lines changed

‎plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AmazonEc2Reference.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.discovery.ec2;
21+
22+
import com.amazonaws.services.ec2.AmazonEC2;
23+
24+
import org.elasticsearch.common.lease.Releasable;
25+
import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
26+
27+
/**
28+
* Handles the shutdown of the wrapped {@link AmazonEC2} using reference
29+
* counting.
30+
*/
31+
public class AmazonEc2Reference extends AbstractRefCounted implements Releasable {
32+
33+
private final AmazonEC2 client;
34+
35+
AmazonEc2Reference(AmazonEC2 client) {
36+
super("AWS_EC2_CLIENT");
37+
this.client = client;
38+
}
39+
40+
/**
41+
* Call when the client is not needed anymore.
42+
*/
43+
@Override
44+
public void close() {
45+
decRef();
46+
}
47+
48+
/**
49+
* Returns the underlying `AmazonEC2` client. All method calls are permitted BUT
50+
* NOT shutdown. Shutdown is called when reference count reaches 0.
51+
*/
52+
public AmazonEC2 client() {
53+
return client;
54+
}
55+
56+
@Override
57+
protected void closeInternal() {
58+
client.shutdown();
59+
}
60+
61+
}

‎plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Service.java‎

Lines changed: 26 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -19,22 +19,17 @@
1919

2020
package org.elasticsearch.discovery.ec2;
2121

22-
import com.amazonaws.ClientConfiguration;
23-
import com.amazonaws.Protocol;
24-
import com.amazonaws.services.ec2.AmazonEC2;
25-
import org.elasticsearch.common.settings.SecureSetting;
26-
import org.elasticsearch.common.settings.SecureString;
2722
import org.elasticsearch.common.settings.Setting;
2823
import org.elasticsearch.common.settings.Setting.Property;
2924
import org.elasticsearch.common.unit.TimeValue;
3025

26+
import java.io.Closeable;
3127
import java.util.ArrayList;
3228
import java.util.Collections;
3329
import java.util.List;
34-
import java.util.Locale;
3530
import java.util.function.Function;
3631

37-
interface AwsEc2Service {
32+
interface AwsEc2Service extends Closeable {
3833
Setting<Boolean> AUTO_ATTRIBUTE_SETTING = Setting.boolSetting("cloud.node.auto_attributes", false, Property.NodeScope);
3934

4035
class HostType {
@@ -45,36 +40,6 @@ class HostType {
4540
public static final String TAG_PREFIX = "tag:";
4641
}
4742

48-
/** The access key (ie login id) for connecting to ec2. */
49-
Setting<SecureString> ACCESS_KEY_SETTING = SecureSetting.secureString("discovery.ec2.access_key", null);
50-
51-
/** The secret key (ie password) for connecting to ec2. */
52-
Setting<SecureString> SECRET_KEY_SETTING = SecureSetting.secureString("discovery.ec2.secret_key", null);
53-
54-
/** An override for the ec2 endpoint to connect to. */
55-
Setting<String> ENDPOINT_SETTING = new Setting<>("discovery.ec2.endpoint", "",
56-
s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
57-
58-
/** The protocol to use to connect to to ec2. */
59-
Setting<Protocol> PROTOCOL_SETTING = new Setting<>("discovery.ec2.protocol", "https",
60-
s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope);
61-
62-
/** The host name of a proxy to connect to ec2 through. */
63-
Setting<String> PROXY_HOST_SETTING = Setting.simpleString("discovery.ec2.proxy.host", Property.NodeScope);
64-
65-
/** The port of a proxy to connect to ec2 through. */
66-
Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("discovery.ec2.proxy.port", 80, 0, 1<<16, Property.NodeScope);
67-
68-
/** The username of a proxy to connect to s3 through. */
69-
Setting<SecureString> PROXY_USERNAME_SETTING = SecureSetting.secureString("discovery.ec2.proxy.username", null);
70-
71-
/** The password of a proxy to connect to s3 through. */
72-
Setting<SecureString> PROXY_PASSWORD_SETTING = SecureSetting.secureString("discovery.ec2.proxy.password", null);
73-
74-
/** The socket timeout for connecting to s3. */
75-
Setting<TimeValue> READ_TIMEOUT_SETTING = Setting.timeSetting("discovery.ec2.read_timeout",
76-
TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope);
77-
7843
/**
7944
* discovery.ec2.host_type: The type of host type to use to communicate with other instances.
8045
* Can be one of private_ip, public_ip, private_dns, public_dns or tag:XXXX where
@@ -87,26 +52,24 @@ class HostType {
8752
* discovery.ec2.any_group: If set to false, will require all security groups to be present for the instance to be used for the
8853
* discovery. Defaults to true.
8954
*/
90-
Setting<Boolean> ANY_GROUP_SETTING =
91-
Setting.boolSetting("discovery.ec2.any_group", true, Property.NodeScope);
55+
Setting<Boolean> ANY_GROUP_SETTING = Setting.boolSetting("discovery.ec2.any_group", true, Property.NodeScope);
9256
/**
9357
* discovery.ec2.groups: Either a comma separated list or array based list of (security) groups. Only instances with the provided
9458
* security groups will be used in the cluster discovery. (NOTE: You could provide either group NAME or group ID.)
9559
*/
96-
Setting<List<String>> GROUPS_SETTING =
97-
Setting.listSetting("discovery.ec2.groups", new ArrayList<>(), s -> s.toString(), Property.NodeScope);
60+
Setting<List<String>> GROUPS_SETTING = Setting.listSetting("discovery.ec2.groups", new ArrayList<>(), s -> s.toString(),
61+
Property.NodeScope);
9862
/**
9963
* discovery.ec2.availability_zones: Either a comma separated list or array based list of availability zones. Only instances within
10064
* the provided availability zones will be used in the cluster discovery.
10165
*/
102-
Setting<List<String>> AVAILABILITY_ZONES_SETTING =
103-
Setting.listSetting("discovery.ec2.availability_zones", Collections.emptyList(), s -> s.toString(),
104-
Property.NodeScope);
66+
Setting<List<String>> AVAILABILITY_ZONES_SETTING = Setting.listSetting("discovery.ec2.availability_zones", Collections.emptyList(),
67+
s -> s.toString(), Property.NodeScope);
10568
/**
10669
* discovery.ec2.node_cache_time: How long the list of hosts is cached to prevent further requests to the AWS API. Defaults to 10s.
10770
*/
108-
Setting<TimeValue> NODE_CACHE_TIME_SETTING =
109-
Setting.timeSetting("discovery.ec2.node_cache_time", TimeValue.timeValueSeconds(10), Property.NodeScope);
71+
Setting<TimeValue> NODE_CACHE_TIME_SETTING = Setting.timeSetting("discovery.ec2.node_cache_time", TimeValue.timeValueSeconds(10),
72+
Property.NodeScope);
11073

11174
/**
11275
* discovery.ec2.tag.*: The ec2 discovery can filter machines to include in the cluster based on tags (and not just groups).
@@ -115,7 +78,22 @@ class HostType {
11578
* instance to be included.
11679
*/
11780
Setting.AffixSetting<List<String>> TAG_SETTING = Setting.prefixKeySetting("discovery.ec2.tag.",
118-
key -> Setting.listSetting(key, Collections.emptyList(), Function.identity(), Property.NodeScope));
81+
key -> Setting.listSetting(key, Collections.emptyList(), Function.identity(), Property.NodeScope));
82+
83+
/**
84+
* Builds then caches an {@code AmazonEC2} client using the current client
85+
* settings. Returns an {@code AmazonEc2Reference} wrapper which should be
86+
* released as soon as it is not required anymore.
87+
*/
88+
AmazonEc2Reference client();
89+
90+
/**
91+
* Updates the settings for building the client and releases the cached one.
92+
* Future client requests will use the new settings to lazily built the new
93+
* client.
94+
*
95+
* @param clientSettings the new refreshed settings
96+
*/
97+
void refreshAndClearCache(Ec2ClientSettings clientSettings);
11998

120-
AmazonEC2 client();
12199
}

‎plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java‎

Lines changed: 75 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,9 @@
1919

2020
package org.elasticsearch.discovery.ec2;
2121

22-
import java.io.Closeable;
23-
import java.io.IOException;
2422
import java.util.Random;
23+
import java.util.concurrent.atomic.AtomicReference;
2524

26-
import com.amazonaws.AmazonClientException;
27-
import com.amazonaws.AmazonWebServiceRequest;
2825
import com.amazonaws.ClientConfiguration;
2926
import com.amazonaws.auth.AWSCredentialsProvider;
3027
import com.amazonaws.auth.BasicAWSCredentials;
@@ -35,112 +32,117 @@
3532
import com.amazonaws.services.ec2.AmazonEC2;
3633
import com.amazonaws.services.ec2.AmazonEC2Client;
3734
import org.apache.logging.log4j.Logger;
35+
import org.elasticsearch.ElasticsearchException;
3836
import org.elasticsearch.common.Randomness;
37+
import org.elasticsearch.common.Strings;
3938
import org.elasticsearch.common.component.AbstractComponent;
40-
import org.elasticsearch.common.settings.SecureString;
4139
import org.elasticsearch.common.settings.Settings;
40+
import org.elasticsearch.common.util.LazyInitializable;
4241

43-
class AwsEc2ServiceImpl extends AbstractComponent implements AwsEc2Service, Closeable {
42+
class AwsEc2ServiceImpl extends AbstractComponent implements AwsEc2Service {
4443

4544
public static final String EC2_METADATA_URL = "http://169.254.169.254/latest/meta-data/";
4645

47-
private AmazonEC2Client client;
46+
private final AtomicReference<LazyInitializable<AmazonEc2Reference, ElasticsearchException>> lazyClientReference =
47+
new AtomicReference<>();
4848

4949
AwsEc2ServiceImpl(Settings settings) {
5050
super(settings);
5151
}
5252

53-
@Override
54-
public synchronized AmazonEC2 client() {
55-
if (client != null) {
56-
return client;
57-
}
58-
59-
this.client = new AmazonEC2Client(buildCredentials(logger, settings), buildConfiguration(logger, settings));
60-
String endpoint = findEndpoint(logger, settings);
61-
if (endpoint != null) {
62-
client.setEndpoint(endpoint);
53+
private AmazonEC2 buildClient(Ec2ClientSettings clientSettings) {
54+
final AWSCredentialsProvider credentials = buildCredentials(logger, clientSettings);
55+
final ClientConfiguration configuration = buildConfiguration(logger, clientSettings);
56+
final AmazonEC2 client = buildClient(credentials, configuration);
57+
if (Strings.hasText(clientSettings.endpoint)) {
58+
logger.debug("using explicit ec2 endpoint [{}]", clientSettings.endpoint);
59+
client.setEndpoint(clientSettings.endpoint);
6360
}
64-
65-
return this.client;
61+
return client;
6662
}
6763

68-
protected static AWSCredentialsProvider buildCredentials(Logger logger, Settings settings) {
69-
AWSCredentialsProvider credentials;
70-
71-
try (SecureString key = ACCESS_KEY_SETTING.get(settings);
72-
SecureString secret = SECRET_KEY_SETTING.get(settings)) {
73-
if (key.length() == 0 && secret.length() == 0) {
74-
logger.debug("Using either environment variables, system properties or instance profile credentials");
75-
credentials = new DefaultAWSCredentialsProviderChain();
76-
} else {
77-
logger.debug("Using basic key/secret credentials");
78-
credentials = new StaticCredentialsProvider(new BasicAWSCredentials(key.toString(), secret.toString()));
79-
}
80-
}
81-
82-
return credentials;
64+
// proxy for testing
65+
AmazonEC2 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration) {
66+
final AmazonEC2 client = new AmazonEC2Client(credentials, configuration);
67+
return client;
8368
}
8469

85-
protected static ClientConfiguration buildConfiguration(Logger logger, Settings settings) {
86-
ClientConfiguration clientConfiguration = new ClientConfiguration();
70+
// pkg private for tests
71+
static ClientConfiguration buildConfiguration(Logger logger, Ec2ClientSettings clientSettings) {
72+
final ClientConfiguration clientConfiguration = new ClientConfiguration();
8773
// the response metadata cache is only there for diagnostics purposes,
8874
// but can force objects from every response to the old generation.
8975
clientConfiguration.setResponseMetadataCacheSize(0);
90-
clientConfiguration.setProtocol(PROTOCOL_SETTING.get(settings));
91-
92-
if (PROXY_HOST_SETTING.exists(settings)) {
93-
String proxyHost = PROXY_HOST_SETTING.get(settings);
94-
Integer proxyPort = PROXY_PORT_SETTING.get(settings);
95-
try (SecureString proxyUsername = PROXY_USERNAME_SETTING.get(settings);
96-
SecureString proxyPassword = PROXY_PASSWORD_SETTING.get(settings)) {
97-
98-
clientConfiguration
99-
.withProxyHost(proxyHost)
100-
.withProxyPort(proxyPort)
101-
.withProxyUsername(proxyUsername.toString())
102-
.withProxyPassword(proxyPassword.toString());
103-
}
76+
clientConfiguration.setProtocol(clientSettings.protocol);
77+
if (Strings.hasText(clientSettings.proxyHost)) {
78+
// TODO: remove this leniency, these settings should exist together and be validated
79+
clientConfiguration.setProxyHost(clientSettings.proxyHost);
80+
clientConfiguration.setProxyPort(clientSettings.proxyPort);
81+
clientConfiguration.setProxyUsername(clientSettings.proxyUsername);
82+
clientConfiguration.setProxyPassword(clientSettings.proxyPassword);
10483
}
105-
10684
// Increase the number of retries in case of 5xx API responses
10785
final Random rand = Randomness.get();
108-
RetryPolicy retryPolicy = new RetryPolicy(
86+
final RetryPolicy retryPolicy = new RetryPolicy(
10987
RetryPolicy.RetryCondition.NO_RETRY_CONDITION,
110-
new RetryPolicy.BackoffStrategy() {
111-
@Override
112-
public long delayBeforeNextRetry(AmazonWebServiceRequest originalRequest,
113-
AmazonClientException exception,
114-
int retriesAttempted) {
115-
// with 10 retries the max delay time is 320s/320000ms (10 * 2^5 * 1 * 1000)
116-
logger.warn("EC2 API request failed, retry again. Reason was:", exception);
117-
return 1000L * (long) (10d * Math.pow(2, retriesAttempted / 2.0d) * (1.0d + rand.nextDouble()));
118-
}
88+
(originalRequest, exception, retriesAttempted) -> {
89+
// with 10 retries the max delay time is 320s/320000ms (10 * 2^5 * 1 * 1000)
90+
logger.warn("EC2 API request failed, retry again. Reason was:", exception);
91+
return 1000L * (long) (10d * Math.pow(2, retriesAttempted / 2.0d) * (1.0d + rand.nextDouble()));
11992
},
12093
10,
12194
false);
12295
clientConfiguration.setRetryPolicy(retryPolicy);
123-
clientConfiguration.setSocketTimeout((int) READ_TIMEOUT_SETTING.get(settings).millis());
124-
96+
clientConfiguration.setSocketTimeout(clientSettings.readTimeoutMillis);
12597
return clientConfiguration;
12698
}
12799

128-
protected static String findEndpoint(Logger logger, Settings settings) {
129-
String endpoint = null;
130-
if (ENDPOINT_SETTING.exists(settings)) {
131-
endpoint = ENDPOINT_SETTING.get(settings);
132-
logger.debug("using explicit ec2 endpoint [{}]", endpoint);
100+
// pkg private for tests
101+
static AWSCredentialsProvider buildCredentials(Logger logger, Ec2ClientSettings clientSettings) {
102+
final BasicAWSCredentials credentials = clientSettings.credentials;
103+
if (credentials == null) {
104+
logger.debug("Using either environment variables, system properties or instance profile credentials");
105+
return new DefaultAWSCredentialsProviderChain();
106+
} else {
107+
logger.debug("Using basic key/secret credentials");
108+
return new StaticCredentialsProvider(credentials);
133109
}
134-
return endpoint;
135110
}
136111

137112
@Override
138-
public void close() throws IOException {
139-
if (client != null) {
140-
client.shutdown();
113+
public AmazonEc2Reference client() {
114+
final LazyInitializable<AmazonEc2Reference, ElasticsearchException> clientReference = this.lazyClientReference.get();
115+
if (clientReference == null) {
116+
throw new IllegalStateException("Missing ec2 client configs");
141117
}
118+
return clientReference.getOrCompute();
119+
}
142120

143-
// Ensure that IdleConnectionReaper is shutdown
121+
/**
122+
* Refreshes the settings for the AmazonEC2 client. The new client will be build
123+
* using these new settings. The old client is usable until released. On release it
124+
* will be destroyed instead of being returned to the cache.
125+
*/
126+
@Override
127+
public void refreshAndClearCache(Ec2ClientSettings clientSettings) {
128+
final LazyInitializable<AmazonEc2Reference, ElasticsearchException> newClient = new LazyInitializable<>(
129+
() -> new AmazonEc2Reference(buildClient(clientSettings)), clientReference -> clientReference.incRef(),
130+
clientReference -> clientReference.decRef());
131+
final LazyInitializable<AmazonEc2Reference, ElasticsearchException> oldClient = this.lazyClientReference.getAndSet(newClient);
132+
if (oldClient != null) {
133+
oldClient.reset();
134+
}
135+
}
136+
137+
@Override
138+
public void close() {
139+
final LazyInitializable<AmazonEc2Reference, ElasticsearchException> clientReference = this.lazyClientReference.getAndSet(null);
140+
if (clientReference != null) {
141+
clientReference.reset();
142+
}
143+
// shutdown IdleConnectionReaper background thread
144+
// it will be restarted on new client usage
144145
IdleConnectionReaper.shutdown();
145146
}
147+
146148
}

0 commit comments

Comments
 (0)