Skip to content

Commit 811ddca

Browse files
virajjasanivirajjasani
committed
HADOOP-18850 Enable dual-layer server-side encryption with AWS KMS keys
1 parent b871805 commit 811ddca

File tree

10 files changed

+296
-18
lines changed

10 files changed

+296
-18
lines changed

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ public enum S3AEncryptionMethods {
3333
SSE_KMS("SSE-KMS", true, false),
3434
SSE_C("SSE-C", true, true),
3535
CSE_KMS("CSE-KMS", false, true),
36-
CSE_CUSTOM("CSE-CUSTOM", false, true);
36+
CSE_CUSTOM("CSE-CUSTOM", false, true),
37+
DSSE_KMS("DSSE-KMS", true, false);
3738

3839
/**
3940
* Error string when {@link #getMethod(String)} fails.

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1421,6 +1421,11 @@ public static EncryptionSecrets buildEncryptionSecrets(String bucket,
14211421
diagnostics);
14221422
break;
14231423

1424+
case DSSE_KMS:
1425+
LOG.debug("Using DSSE-KMS with {}",
1426+
diagnostics);
1427+
break;
1428+
14241429
case NONE:
14251430
default:
14261431
LOG.debug("Data is unencrypted");

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,8 @@ public static Optional<String> getSSECustomerKey(final EncryptionSecrets secrets
5353
* @return an optional key to attach to a request.
5454
*/
5555
public static Optional<String> getSSEAwsKMSKey(final EncryptionSecrets secrets) {
56-
if (secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_KMS
56+
if ((secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_KMS
57+
|| secrets.getEncryptionMethod() == S3AEncryptionMethods.DSSE_KMS)
5758
&& secrets.hasEncryptionKey()) {
5859
return Optional.of(secrets.getEncryptionKey());
5960
} else {

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,10 @@ protected void copyEncryptionParameters(HeadObjectResponse srcom,
279279
// Set the KMS key if present, else S3 uses AWS managed key.
280280
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
281281
.ifPresent(kmsKey -> copyObjectRequestBuilder.ssekmsKeyId(kmsKey));
282+
} else if (S3AEncryptionMethods.DSSE_KMS == algorithm) {
283+
copyObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE);
284+
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
285+
.ifPresent(copyObjectRequestBuilder::ssekmsKeyId);
282286
} else if (S3AEncryptionMethods.SSE_C == algorithm) {
283287
EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets)
284288
.ifPresent(base64customerKey -> {
@@ -354,6 +358,10 @@ private void putEncryptionParameters(PutObjectRequest.Builder putObjectRequestBu
354358
// Set the KMS key if present, else S3 uses AWS managed key.
355359
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
356360
.ifPresent(kmsKey -> putObjectRequestBuilder.ssekmsKeyId(kmsKey));
361+
} else if (S3AEncryptionMethods.DSSE_KMS == algorithm) {
362+
putObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE);
363+
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
364+
.ifPresent(putObjectRequestBuilder::ssekmsKeyId);
357365
} else if (S3AEncryptionMethods.SSE_C == algorithm) {
358366
EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets)
359367
.ifPresent(base64customerKey -> {
@@ -415,6 +423,10 @@ private void multipartUploadEncryptionParameters(
415423
// Set the KMS key if present, else S3 uses AWS managed key.
416424
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
417425
.ifPresent(kmsKey -> mpuRequestBuilder.ssekmsKeyId(kmsKey));
426+
} else if (S3AEncryptionMethods.DSSE_KMS == algorithm) {
427+
mpuRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE);
428+
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
429+
.ifPresent(mpuRequestBuilder::ssekmsKeyId);
418430
} else if (S3AEncryptionMethods.SSE_C == algorithm) {
419431
EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets)
420432
.ifPresent(base64customerKey -> {

‎hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ private EncryptionTestUtils() {
3939

4040
public static final String AWS_KMS_SSE_ALGORITHM = "aws:kms";
4141

42+
public static final String AWS_KMS_DSSE_ALGORITHM = "aws:kms:dsse";
43+
4244
public static final String SSE_C_ALGORITHM = "AES256";
4345

4446
/**
@@ -94,6 +96,13 @@ public static void assertEncrypted(S3AFileSystem fs,
9496
kmsKeyArn,
9597
md.ssekmsKeyId());
9698
break;
99+
case DSSE_KMS:
100+
assertEquals("Wrong algorithm in " + details,
101+
AWS_KMS_DSSE_ALGORITHM, md.serverSideEncryptionAsString());
102+
assertEquals("Wrong KMS key in " + details,
103+
kmsKeyArn,
104+
md.ssekmsKeyId());
105+
break;
97106
default:
98107
assertEquals("AES256", md.serverSideEncryptionAsString());
99108
}

‎hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADSSEEncryptionWithDefaultS3Settings.java‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs.s3a;
20+
21+
import java.io.IOException;
22+
23+
import org.junit.Ignore;
24+
import org.junit.Test;
25+
26+
import org.apache.commons.lang3.StringUtils;
27+
import org.apache.hadoop.conf.Configuration;
28+
import org.apache.hadoop.fs.FileSystem;
29+
import org.apache.hadoop.fs.Path;
30+
import org.apache.hadoop.fs.contract.ContractTestUtils;
31+
import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
32+
33+
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
34+
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
35+
import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
36+
import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM;
37+
import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM;
38+
import static org.apache.hadoop.fs.s3a.EncryptionTestUtils.AWS_KMS_DSSE_ALGORITHM;
39+
import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.DSSE_KMS;
40+
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
41+
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
42+
import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionNotSet;
43+
import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey;
44+
45+
/**
46+
* Concrete class that extends {@link AbstractTestS3AEncryption}
47+
* and tests already configured bucket level DSSE encryption using s3 console.
48+
*/
49+
public class ITestS3ADSSEEncryptionWithDefaultS3Settings extends
50+
AbstractTestS3AEncryption {
51+
52+
@Override
53+
public void setup() throws Exception {
54+
super.setup();
55+
// get the KMS key for this test.
56+
S3AFileSystem fs = getFileSystem();
57+
Configuration c = fs.getConf();
58+
skipIfEncryptionNotSet(c, getSSEAlgorithm());
59+
}
60+
61+
@SuppressWarnings("deprecation")
62+
@Override
63+
protected void patchConfigurationEncryptionSettings(
64+
final Configuration conf) {
65+
removeBaseAndBucketOverrides(conf,
66+
S3_ENCRYPTION_ALGORITHM,
67+
SERVER_SIDE_ENCRYPTION_ALGORITHM);
68+
conf.set(S3_ENCRYPTION_ALGORITHM,
69+
getSSEAlgorithm().getMethod());
70+
}
71+
72+
/**
73+
* Setting this to NONE as we don't want to overwrite
74+
* already configured encryption settings.
75+
* @return the algorithm
76+
*/
77+
@Override
78+
protected S3AEncryptionMethods getSSEAlgorithm() {
79+
return S3AEncryptionMethods.NONE;
80+
}
81+
82+
/**
83+
* The check here is that the object is encrypted
84+
* <i>and</i> that the encryption key is the KMS key
85+
* provided, not any default key.
86+
* @param path path
87+
*/
88+
@Override
89+
protected void assertEncrypted(Path path) throws IOException {
90+
S3AFileSystem fs = getFileSystem();
91+
Configuration c = fs.getConf();
92+
String kmsKey = getS3EncryptionKey(getTestBucketName(c), c);
93+
EncryptionTestUtils.assertEncrypted(fs, path, DSSE_KMS, kmsKey);
94+
}
95+
96+
@Override
97+
@Ignore
98+
@Test
99+
public void testEncryptionSettingPropagation() throws Throwable {
100+
}
101+
102+
@Override
103+
@Ignore
104+
@Test
105+
public void testEncryption() throws Throwable {
106+
}
107+
108+
/**
109+
* Skipping if the test bucket is not configured with
110+
* aws:kms encryption algorithm.
111+
*/
112+
@Override
113+
public void testEncryptionOverRename() throws Throwable {
114+
skipIfBucketNotKmsEncrypted();
115+
super.testEncryptionOverRename();
116+
}
117+
118+
/**
119+
* If the test bucket is not configured with aws:kms encryption algorithm,
120+
* skip the test.
121+
*
122+
* @throws IOException If the object creation/deletion/access fails.
123+
*/
124+
private void skipIfBucketNotKmsEncrypted() throws IOException {
125+
S3AFileSystem fs = getFileSystem();
126+
Path path = path(getMethodName() + "find-encryption-algo");
127+
ContractTestUtils.touch(fs, path);
128+
try {
129+
String sseAlgorithm =
130+
getS3AInternals().getObjectMetadata(path).serverSideEncryptionAsString();
131+
if (StringUtils.isBlank(sseAlgorithm) || !sseAlgorithm.equals(AWS_KMS_DSSE_ALGORITHM)) {
132+
skip("Test bucket is not configured with " + AWS_KMS_DSSE_ALGORITHM);
133+
}
134+
} finally {
135+
ContractTestUtils.assertDeleted(fs, path, false);
136+
}
137+
}
138+
139+
@Test
140+
public void testEncryptionOverRename2() throws Throwable {
141+
skipIfBucketNotKmsEncrypted();
142+
S3AFileSystem fs = getFileSystem();
143+
144+
// write the file with the unencrypted FS.
145+
// this will pick up whatever defaults we have.
146+
Path src = path(createFilename(1024));
147+
byte[] data = dataset(1024, 'a', 'z');
148+
EncryptionSecrets secrets = fs.getEncryptionSecrets();
149+
validateEncryptionSecrets(secrets);
150+
writeDataset(fs, src, data, data.length, 1024 * 1024, true);
151+
ContractTestUtils.verifyFileContents(fs, src, data);
152+
153+
Configuration fs2Conf = new Configuration(fs.getConf());
154+
fs2Conf.set(S3_ENCRYPTION_ALGORITHM,
155+
DSSE_KMS.getMethod());
156+
try (FileSystem kmsFS = FileSystem.newInstance(fs.getUri(), fs2Conf)) {
157+
Path targetDir = path("target");
158+
kmsFS.mkdirs(targetDir);
159+
ContractTestUtils.rename(kmsFS, src, targetDir);
160+
Path renamedFile = new Path(targetDir, src.getName());
161+
ContractTestUtils.verifyFileContents(fs, renamedFile, data);
162+
String kmsKey = getS3EncryptionKey(getTestBucketName(fs2Conf), fs2Conf);
163+
// we assert that the renamed file has picked up the KMS key of our FS
164+
EncryptionTestUtils.assertEncrypted(fs, renamedFile, DSSE_KMS, kmsKey);
165+
}
166+
}
167+
}

‎hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
* <p>
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
* <p>
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs.s3a;
20+
21+
import org.apache.commons.lang.StringUtils;
22+
import org.apache.hadoop.conf.Configuration;
23+
24+
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
25+
import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY;
26+
import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.DSSE_KMS;
27+
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
28+
29+
/**
30+
* Concrete class that extends {@link AbstractTestS3AEncryption}
31+
* and tests DSSE-KMS encryption.
32+
*/
33+
public class ITestS3AEncryptionDSSEKMSUserDefinedKey
34+
extends AbstractTestS3AEncryption {
35+
36+
@Override
37+
protected Configuration createConfiguration() {
38+
// get the KMS key for this test.
39+
Configuration c = new Configuration();
40+
String kmsKey = S3AUtils.getS3EncryptionKey(getTestBucketName(c), c);
41+
// skip the test if DSSE-KMS or KMS key not set.
42+
if (StringUtils.isBlank(kmsKey)) {
43+
skip(S3_ENCRYPTION_KEY + " is not set for " +
44+
DSSE_KMS.getMethod());
45+
}
46+
Configuration conf = super.createConfiguration();
47+
conf.set(S3_ENCRYPTION_KEY, kmsKey);
48+
return conf;
49+
}
50+
51+
@Override
52+
protected S3AEncryptionMethods getSSEAlgorithm() {
53+
return DSSE_KMS;
54+
}
55+
}

‎hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -115,20 +115,34 @@ public void testEncryption() throws Throwable {
115115
*/
116116
@Override
117117
public void testEncryptionOverRename() throws Throwable {
118+
skipIfBucketNotKmsEncrypted();
119+
super.testEncryptionOverRename();
120+
}
121+
122+
/**
123+
* If the test bucket is not configured with aws:kms encryption algorithm,
124+
* skip the test.
125+
*
126+
* @throws IOException If the object creation/deletion/access fails.
127+
*/
128+
private void skipIfBucketNotKmsEncrypted() throws IOException {
118129
S3AFileSystem fs = getFileSystem();
119130
Path path = path(getMethodName() + "find-encryption-algo");
120131
ContractTestUtils.touch(fs, path);
121-
String sseAlgorithm = getS3AInternals().getObjectMetadata(path)
122-
.serverSideEncryptionAsString();
123-
if(StringUtils.isBlank(sseAlgorithm) ||
124-
!sseAlgorithm.equals(AWS_KMS_SSE_ALGORITHM)) {
125-
skip("Test bucket is not configured with " + AWS_KMS_SSE_ALGORITHM);
132+
try {
133+
String sseAlgorithm =
134+
getS3AInternals().getObjectMetadata(path).serverSideEncryptionAsString();
135+
if (StringUtils.isBlank(sseAlgorithm) || !sseAlgorithm.equals(AWS_KMS_SSE_ALGORITHM)) {
136+
skip("Test bucket is not configured with " + AWS_KMS_SSE_ALGORITHM);
137+
}
138+
} finally {
139+
ContractTestUtils.assertDeleted(fs, path, false);
126140
}
127-
super.testEncryptionOverRename();
128141
}
129142

130143
@Test
131144
public void testEncryptionOverRename2() throws Throwable {
145+
skipIfBucketNotKmsEncrypted();
132146
S3AFileSystem fs = getFileSystem();
133147

134148
// write the file with the unencrypted FS.

‎hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@
7777
import java.text.DateFormat;
7878
import java.text.SimpleDateFormat;
7979
import java.util.ArrayList;
80+
import java.util.Arrays;
8081
import java.util.List;
8182
import java.util.Set;
8283
import java.util.TreeSet;
@@ -1455,19 +1456,25 @@ public static S3AFileStatus innerGetFileStatus(
14551456
* Skip a test if encryption algorithm or encryption key is not set.
14561457
*
14571458
* @param configuration configuration to probe.
1459+
* @param s3AEncryptionMethods list of encryption algorithms to probe.
1460+
* @throws IOException if the secret lookup fails.
14581461
*/
14591462
public static void skipIfEncryptionNotSet(Configuration configuration,
1460-
S3AEncryptionMethods s3AEncryptionMethod) throws IOException {
1463+
S3AEncryptionMethods... s3AEncryptionMethods) throws IOException {
1464+
if (s3AEncryptionMethods == null || s3AEncryptionMethods.length == 0) {
1465+
throw new IllegalArgumentException("Specify at least one encryption method");
1466+
}
14611467
// if S3 encryption algorithm is not set to desired method or AWS encryption
14621468
// key is not set, then skip.
14631469
String bucket = getTestBucketName(configuration);
14641470
final EncryptionSecrets secrets = buildEncryptionSecrets(bucket, configuration);
1465-
if (!s3AEncryptionMethod.getMethod().equals(secrets.getEncryptionMethod().getMethod())
1466-
|| StringUtils.isBlank(secrets.getEncryptionKey())) {
1467-
skip(S3_ENCRYPTION_KEY + " is not set for " + s3AEncryptionMethod
1468-
.getMethod() + " or " + S3_ENCRYPTION_ALGORITHM + " is not set to "
1469-
+ s3AEncryptionMethod.getMethod()
1470-
+ " in " + secrets);
1471+
boolean encryptionMethodMatching = Arrays.stream(s3AEncryptionMethods).anyMatch(
1472+
s3AEncryptionMethod -> s3AEncryptionMethod.getMethod()
1473+
.equals(secrets.getEncryptionMethod().getMethod()));
1474+
if (!encryptionMethodMatching || StringUtils.isBlank(secrets.getEncryptionKey())) {
1475+
skip(S3_ENCRYPTION_KEY + " is not set or " + S3_ENCRYPTION_ALGORITHM + " is not set to "
1476+
+ Arrays.stream(s3AEncryptionMethods).map(S3AEncryptionMethods::getMethod)
1477+
.collect(Collectors.toList()) + " in " + secrets);
14711478
}
14721479
}
14731480

0 commit comments

Comments
 (0)