Skip to content

Commit 45ac820

Browse files
DaveCTurnerDaveCTurner
committed
Add support for AWS session tokens (#30414)
AWS supports the creation and use of credentials that are only valid for a fixed period of time. These credentials comprise three parts: the usual access key and secret key, together with a session token. This commit adds support for these three-part credentials to the EC2 discovery plugin and the S3 repository plugin. Note that session tokens are only valid for a limited period of time and yet there is no mechanism for refreshing or rotating them when they expire without restarting Elasticsearch. Nonetheless, this feature is already useful for nodes that need only run for a few days, such as for training, testing or evaluation. #29135 tracks the work towards allowing these credentials to be refreshed at runtime. Resolves #16428
1 parent 7f2fa0d commit 45ac820

File tree

16 files changed

+685
-119
lines changed

16 files changed

+685
-119
lines changed

‎docs/plugins/discovery-ec2.asciidoc‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,15 @@ Those that must be stored in the keystore are marked as `Secure`.
4040

4141
`access_key`::
4242

43-
An s3 access key. The `secret_key` setting must also be specified. (Secure)
43+
An ec2 access key. The `secret_key` setting must also be specified. (Secure)
4444

4545
`secret_key`::
4646

47-
An s3 secret key. The `access_key` setting must also be specified. (Secure)
47+
An ec2 secret key. The `access_key` setting must also be specified. (Secure)
48+
49+
`session_token`::
50+
An ec2 session token. The `access_key` and `secret_key` settings must also
51+
be specified. (Secure)
4852

4953
`endpoint`::
5054

‎docs/plugins/repository-s3.asciidoc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,10 @@ are marked as `Secure`.
7373

7474
An s3 secret key. The `access_key` setting must also be specified. (Secure)
7575

76+
`session_token`::
77+
An s3 session token. The `access_key` and `secret_key` settings must also
78+
be specified. (Secure)
79+
7680
`endpoint`::
7781

7882
The s3 service endpoint to connect to. This will be automatically

‎plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,9 @@
1919

2020
package org.elasticsearch.discovery.ec2;
2121

22-
import java.util.Random;
23-
import java.util.concurrent.atomic.AtomicReference;
24-
2522
import com.amazonaws.ClientConfiguration;
23+
import com.amazonaws.auth.AWSCredentials;
2624
import com.amazonaws.auth.AWSCredentialsProvider;
27-
import com.amazonaws.auth.BasicAWSCredentials;
2825
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
2926
import com.amazonaws.http.IdleConnectionReaper;
3027
import com.amazonaws.internal.StaticCredentialsProvider;
@@ -39,6 +36,9 @@
3936
import org.elasticsearch.common.settings.Settings;
4037
import org.elasticsearch.common.util.LazyInitializable;
4138

39+
import java.util.Random;
40+
import java.util.concurrent.atomic.AtomicReference;
41+
4242
class AwsEc2ServiceImpl extends AbstractComponent implements AwsEc2Service {
4343

4444
public static final String EC2_METADATA_URL = "http://169.254.169.254/latest/meta-data/";
@@ -99,7 +99,7 @@ static ClientConfiguration buildConfiguration(Logger logger, Ec2ClientSettings c
9999

100100
// pkg private for tests
101101
static AWSCredentialsProvider buildCredentials(Logger logger, Ec2ClientSettings clientSettings) {
102-
final BasicAWSCredentials credentials = clientSettings.credentials;
102+
final AWSCredentials credentials = clientSettings.credentials;
103103
if (credentials == null) {
104104
logger.debug("Using either environment variables, system properties or instance profile credentials");
105105
return new DefaultAWSCredentialsProviderChain();

‎plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java‎

Lines changed: 47 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,20 @@
2121

2222
import com.amazonaws.ClientConfiguration;
2323
import com.amazonaws.Protocol;
24+
import com.amazonaws.auth.AWSCredentials;
2425
import com.amazonaws.auth.BasicAWSCredentials;
25-
26+
import com.amazonaws.auth.BasicSessionCredentials;
27+
import org.apache.logging.log4j.Logger;
28+
import org.elasticsearch.common.logging.DeprecationLogger;
29+
import org.elasticsearch.common.logging.Loggers;
2630
import org.elasticsearch.common.settings.SecureSetting;
2731
import org.elasticsearch.common.settings.SecureString;
2832
import org.elasticsearch.common.settings.Setting;
29-
import org.elasticsearch.common.settings.Settings;
3033
import org.elasticsearch.common.settings.Setting.Property;
34+
import org.elasticsearch.common.settings.Settings;
35+
import org.elasticsearch.common.settings.SettingsException;
3136
import org.elasticsearch.common.unit.TimeValue;
37+
3238
import java.util.Locale;
3339

3440
/**
@@ -42,6 +48,9 @@ final class Ec2ClientSettings {
4248
/** The secret key (ie password) for connecting to ec2. */
4349
static final Setting<SecureString> SECRET_KEY_SETTING = SecureSetting.secureString("discovery.ec2.secret_key", null);
4450

51+
/** The session token for connecting to ec2. */
52+
static final Setting<SecureString> SESSION_TOKEN_SETTING = SecureSetting.secureString("discovery.ec2.session_token", null);
53+
4554
/** The host name of a proxy to connect to ec2 through. */
4655
static final Setting<String> PROXY_HOST_SETTING = Setting.simpleString("discovery.ec2.proxy.host", Property.NodeScope);
4756

@@ -66,8 +75,12 @@ final class Ec2ClientSettings {
6675
static final Setting<TimeValue> READ_TIMEOUT_SETTING = Setting.timeSetting("discovery.ec2.read_timeout",
6776
TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope);
6877

78+
private static final Logger logger = Loggers.getLogger(Ec2ClientSettings.class);
79+
80+
private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger);
81+
6982
/** Credentials to authenticate with ec2. */
70-
final BasicAWSCredentials credentials;
83+
final AWSCredentials credentials;
7184

7285
/**
7386
* The ec2 endpoint the client should talk to, or empty string to use the
@@ -96,7 +109,7 @@ final class Ec2ClientSettings {
96109
/** The read timeout for the ec2 client. */
97110
final int readTimeoutMillis;
98111

99-
protected Ec2ClientSettings(BasicAWSCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort,
112+
protected Ec2ClientSettings(AWSCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort,
100113
String proxyUsername, String proxyPassword, int readTimeoutMillis) {
101114
this.credentials = credentials;
102115
this.endpoint = endpoint;
@@ -108,26 +121,45 @@ protected Ec2ClientSettings(BasicAWSCredentials credentials, String endpoint, Pr
108121
this.readTimeoutMillis = readTimeoutMillis;
109122
}
110123

111-
static BasicAWSCredentials loadCredentials(Settings settings) {
112-
try (SecureString accessKey = ACCESS_KEY_SETTING.get(settings);
113-
SecureString secretKey = SECRET_KEY_SETTING.get(settings);) {
114-
if (accessKey.length() != 0) {
115-
if (secretKey.length() != 0) {
116-
return new BasicAWSCredentials(accessKey.toString(), secretKey.toString());
124+
static AWSCredentials loadCredentials(Settings settings) {
125+
try (SecureString key = ACCESS_KEY_SETTING.get(settings);
126+
SecureString secret = SECRET_KEY_SETTING.get(settings);
127+
SecureString sessionToken = SESSION_TOKEN_SETTING.get(settings)) {
128+
if (key.length() == 0 && secret.length() == 0) {
129+
if (sessionToken.length() > 0) {
130+
throw new SettingsException("Setting [{}] is set but [{}] and [{}] are not",
131+
SESSION_TOKEN_SETTING.getKey(), ACCESS_KEY_SETTING.getKey(), SECRET_KEY_SETTING.getKey());
132+
}
133+
134+
logger.debug("Using either environment variables, system properties or instance profile credentials");
135+
return null;
136+
} else {
137+
if (key.length() == 0) {
138+
DEPRECATION_LOGGER.deprecated("Setting [{}] is set but [{}] is not, which will be unsupported in future",
139+
SECRET_KEY_SETTING.getKey(), ACCESS_KEY_SETTING.getKey());
140+
}
141+
if (secret.length() == 0) {
142+
DEPRECATION_LOGGER.deprecated("Setting [{}] is set but [{}] is not, which will be unsupported in future",
143+
ACCESS_KEY_SETTING.getKey(), SECRET_KEY_SETTING.getKey());
144+
}
145+
146+
final AWSCredentials credentials;
147+
if (sessionToken.length() == 0) {
148+
logger.debug("Using basic key/secret credentials");
149+
credentials = new BasicAWSCredentials(key.toString(), secret.toString());
117150
} else {
118-
throw new IllegalArgumentException("Missing secret key for ec2 client.");
151+
logger.debug("Using basic session credentials");
152+
credentials = new BasicSessionCredentials(key.toString(), secret.toString(), sessionToken.toString());
119153
}
120-
} else if (secretKey.length() != 0) {
121-
throw new IllegalArgumentException("Missing access key for ec2 client.");
154+
return credentials;
122155
}
123-
return null;
124156
}
125157
}
126158

127159
// pkg private for tests
128160
/** Parse settings for a single client. */
129161
static Ec2ClientSettings getClientSettings(Settings settings) {
130-
final BasicAWSCredentials credentials = loadCredentials(settings);
162+
final AWSCredentials credentials = loadCredentials(settings);
131163
try (SecureString proxyUsername = PROXY_USERNAME_SETTING.get(settings);
132164
SecureString proxyPassword = PROXY_PASSWORD_SETTING.get(settings)) {
133165
return new Ec2ClientSettings(

‎plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ public List<Setting<?>> getSettings() {
107107
// Register EC2 discovery settings: discovery.ec2
108108
Ec2ClientSettings.ACCESS_KEY_SETTING,
109109
Ec2ClientSettings.SECRET_KEY_SETTING,
110+
Ec2ClientSettings.SESSION_TOKEN_SETTING,
110111
Ec2ClientSettings.ENDPOINT_SETTING,
111112
Ec2ClientSettings.PROTOCOL_SETTING,
112113
Ec2ClientSettings.PROXY_HOST_SETTING,

‎plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java‎

Lines changed: 47 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,11 @@
2323
import com.amazonaws.Protocol;
2424
import com.amazonaws.auth.AWSCredentials;
2525
import com.amazonaws.auth.AWSCredentialsProvider;
26+
import com.amazonaws.auth.BasicSessionCredentials;
2627
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
2728
import org.elasticsearch.common.settings.MockSecureSettings;
2829
import org.elasticsearch.common.settings.Settings;
29-
import org.elasticsearch.discovery.ec2.AwsEc2ServiceImpl;
30+
import org.elasticsearch.common.settings.SettingsException;
3031
import org.elasticsearch.test.ESTestCase;
3132

3233
import static org.hamcrest.Matchers.instanceOf;
@@ -44,15 +45,53 @@ public void testAWSCredentialsWithElasticsearchAwsSettings() {
4445
final MockSecureSettings secureSettings = new MockSecureSettings();
4546
secureSettings.setString("discovery.ec2.access_key", "aws_key");
4647
secureSettings.setString("discovery.ec2.secret_key", "aws_secret");
47-
final Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
48-
launchAWSCredentialsWithElasticsearchSettingsTest(settings, "aws_key", "aws_secret");
48+
final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger,
49+
Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials();
50+
assertThat(credentials.getAWSAccessKeyId(), is("aws_key"));
51+
assertThat(credentials.getAWSSecretKey(), is("aws_secret"));
4952
}
5053

51-
protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings settings, String expectedKey, String expectedSecret) {
52-
final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, Ec2ClientSettings.getClientSettings(settings))
53-
.getCredentials();
54-
assertThat(credentials.getAWSAccessKeyId(), is(expectedKey));
55-
assertThat(credentials.getAWSSecretKey(), is(expectedSecret));
54+
public void testAWSSessionCredentialsWithElasticsearchAwsSettings() {
55+
final MockSecureSettings secureSettings = new MockSecureSettings();
56+
secureSettings.setString("discovery.ec2.access_key", "aws_key");
57+
secureSettings.setString("discovery.ec2.secret_key", "aws_secret");
58+
secureSettings.setString("discovery.ec2.session_token", "aws_session_token");
59+
final BasicSessionCredentials credentials = (BasicSessionCredentials) AwsEc2ServiceImpl.buildCredentials(logger,
60+
Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials();
61+
assertThat(credentials.getAWSAccessKeyId(), is("aws_key"));
62+
assertThat(credentials.getAWSSecretKey(), is("aws_secret"));
63+
assertThat(credentials.getSessionToken(), is("aws_session_token"));
64+
}
65+
66+
public void testDeprecationOfLoneAccessKey() {
67+
final MockSecureSettings secureSettings = new MockSecureSettings();
68+
secureSettings.setString("discovery.ec2.access_key", "aws_key");
69+
final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger,
70+
Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials();
71+
assertThat(credentials.getAWSAccessKeyId(), is("aws_key"));
72+
assertThat(credentials.getAWSSecretKey(), is(""));
73+
assertSettingDeprecationsAndWarnings(new String[]{},
74+
"Setting [discovery.ec2.access_key] is set but [discovery.ec2.secret_key] is not, which will be unsupported in future");
75+
}
76+
77+
public void testDeprecationOfLoneSecretKey() {
78+
final MockSecureSettings secureSettings = new MockSecureSettings();
79+
secureSettings.setString("discovery.ec2.secret_key", "aws_secret");
80+
final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger,
81+
Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials();
82+
assertThat(credentials.getAWSAccessKeyId(), is(""));
83+
assertThat(credentials.getAWSSecretKey(), is("aws_secret"));
84+
assertSettingDeprecationsAndWarnings(new String[]{},
85+
"Setting [discovery.ec2.secret_key] is set but [discovery.ec2.access_key] is not, which will be unsupported in future");
86+
}
87+
88+
public void testRejectionOfLoneSessionToken() {
89+
final MockSecureSettings secureSettings = new MockSecureSettings();
90+
secureSettings.setString("discovery.ec2.session_token", "aws_session_token");
91+
SettingsException e = expectThrows(SettingsException.class, () -> AwsEc2ServiceImpl.buildCredentials(logger,
92+
Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())));
93+
assertThat(e.getMessage(), is(
94+
"Setting [discovery.ec2.session_token] is set but [discovery.ec2.access_key] and [discovery.ec2.secret_key] are not"));
5695
}
5796

5897
public void testAWSDefaultConfiguration() {

0 commit comments

Comments
 (0)