Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
85 commits
Select commit Hold shift + click to select a range
9dc24eb
config to req headers
sumangala17 Sep 15, 2021
bd579f2
cache keys, refactor client encr into 1 fn, outstream
sumangala17 Sep 16, 2021
59d171f
use existing getpathstatus, cache at stream open
sumangala17 Sep 16, 2021
538820e
main code change
sumangala17 Sep 20, 2021
09c0fbb
other tests syntax fix
sumangala17 Sep 20, 2021
10adf42
encryption test
sumangala17 Sep 20, 2021
84df4a2
encryption adapter
sumangala17 Sep 21, 2021
7365d82
move key fetch/compute to client
sumangala17 Sep 21, 2021
ffb4764
enc-cxt - SecretKey, fetch key object from provider
sumangala17 Sep 22, 2021
31f4c0f
refactor, move cpk to adapter, abfsKey cxt
sumangala17 Oct 1, 2021
af6cd1d
partial bug fix
sumangala17 Oct 4, 2021
2ad36d3
fix mockProv map, clean up
sumangala17 Oct 5, 2021
0e19fd4
tests, formatting, fix cpk test path
sumangala17 Oct 11, 2021
4dc1601
cpk test minor change
sumangala17 Oct 11, 2021
8998bc0
fix merge conflicts
sumangala17 Oct 11, 2021
8357830
checkstyle
sumangala17 Oct 11, 2021
8035ed7
javadoc
sumangala17 Oct 11, 2021
3bf75c3
add parameterized test
sumangala17 Oct 14, 2021
fdc3149
complete tests
sumangala17 Oct 14, 2021
a3559a3
address review comments
sumangala17 Nov 10, 2021
99cf2bd
test to use random context and key strings
sumangala17 Nov 19, 2021
f6bb16f
fix merge conflicts, correct import
sumangala17 Nov 19, 2021
ae6fce6
Merge branch 'trunk' into HADOOP-17912
sumangala17 Nov 22, 2021
04a73e0
grp createpath args, pre-computed key configs, x-ms-version
sumangala17 Nov 22, 2021
d96e08d
allow account specific ECP only
sumangala17 Nov 23, 2021
fb75454
encryption context encoding + checkstyle
sumangala17 Nov 30, 2021
ac6ff51
merge conflict
sumangala17 Nov 30, 2021
3caeb7b
generalize key -> String to byte array, add doc for cpk options
sumangala17 Dec 2, 2021
a2fc26e
Merge branch 'trunk' into HADOOP-17912
sumangala17 Dec 31, 2021
b840da3
yetus fix
sumangala17 Jan 2, 2022
72b5f2f
yetus blanks
sumangala17 Jan 3, 2022
66eb217
sync with trunk
sumangala17 May 11, 2022
1721d43
merge
sumangala17 May 11, 2022
15a0ce8
skip for nonHNS account
sumangala17 May 13, 2022
2f2947c
override final variable xMsVersion in mock to fix NPE
sumangala17 May 13, 2022
16973d4
merged pranavsaxena-microsoft:HADOOP-17912-backemerge
saxenapranav Jul 8, 2022
c27c28d
backmerged pranavsaxena:trunk
saxenapranav Jul 8, 2022
0e700ca
backmerged apache/hadoop:trunk
saxenapranav Jul 8, 2022
4164ee2
remove additional space
saxenapranav Jul 8, 2022
6c05dac
Findbug resolution
saxenapranav Jul 8, 2022
cd7d055
take out ABFSKey class out; Ensure that ABFSKey is used by the Encryp…
saxenapranav Jul 12, 2022
926e7d9
refactor
saxenapranav Jul 12, 2022
af5e90b
Added Apache license
saxenapranav Jul 19, 2022
9505ed9
initial commit
mariosmeim-db Sep 1, 2022
f3abbb9
revert unrelated change
mariosmeim-db Sep 1, 2022
a3d31f1
remove unused import
mariosmeim-db Sep 2, 2022
5842b28
minor
mariosmeim-db Sep 2, 2022
1f1d9d1
clean up deps
mariosmeim-db Sep 2, 2022
4f86224
fix test
mariosmeim-db Sep 7, 2022
873b3fe
purge buffers before destroying adapter
mariosmeim-db Sep 7, 2022
e8c1fb5
checkstyle
saxenapranav Sep 15, 2022
ae2d065
checkstyle-2
saxenapranav Sep 15, 2022
4f1e33b
read in test should have etag
saxenapranav Sep 16, 2022
8806ceb
general code refactor
saxenapranav Oct 7, 2022
ef092dd
general refactor
saxenapranav Oct 7, 2022
fec93f4
Made ABFSKey final
saxenapranav Oct 7, 2022
c2d35d8
added javadocs
saxenapranav Oct 7, 2022
1e987a5
refactors
saxenapranav Oct 7, 2022
6364cfb
added javadocs.
saxenapranav Oct 10, 2022
a1b1906
small refactors + javadocs
saxenapranav Oct 10, 2022
d35a276
removed null handling of encryptionAdapter in abfsclient.addEncryptio…
saxenapranav Oct 11, 2022
450324a
refactored how encryptionAdapter was created in case of versionFileSt…
saxenapranav Oct 11, 2022
183f1bc
small refactors
saxenapranav Oct 12, 2022
d0a16bd
fixed tests
saxenapranav Oct 12, 2022
a69ee4a
refactored condition in openFileForRead
saxenapranav Oct 17, 2022
061930b
getIsNamespace in abfsclient
saxenapranav Nov 17, 2022
00bd8c3
Added NamespaceUtil
saxenapranav Nov 17, 2022
a642ef3
ListResultEntrySchema to have value for x-ms-encryption-context: http…
saxenapranav Nov 21, 2022
56f3efa
added test for the listPath
saxenapranav Nov 21, 2022
adc82f6
test for: using result of listStatus to open a file in case the file …
saxenapranav Dec 8, 2022
241fb9b
Remove public documentation for Encryption Options.
saxenapranav Dec 12, 2022
9daea68
remove non required import in abfsOutputStream; itestAbfsCustomEncryp…
saxenapranav Dec 12, 2022
2313790
small refactors
saxenapranav Dec 12, 2022
aee1c7b
refactor mock verification so that parameter can be null
saxenapranav Dec 12, 2022
bb45ae9
assertions changed for abfsclient namespaceEnabled check
saxenapranav Dec 13, 2022
3c6eddb
Revert "assertions changed for abfsclient namespaceEnabled check"
saxenapranav Dec 13, 2022
a231592
AbfsClientUtils to set certain fields for test. AbfsClient to have pa…
saxenapranav Dec 13, 2022
6ab375c
added javadocs; indentation; comments on https://github.com/sumangala…
saxenapranav Dec 22, 2022
aaf1582
minor change
saxenapranav Dec 22, 2022
f9658e0
backmerged trunk
saxenapranav Dec 29, 2022
4002da6
minor refactor
saxenapranav Dec 29, 2022
1990a48
testAbfsInputStream method signature change refactor
saxenapranav Dec 29, 2022
a2472d5
spotbugs + javadocs
saxenapranav Dec 29, 2022
95b0f9d
javadocs refactor
saxenapranav Jan 2, 2023
0cd8c8c
checkstyle
saxenapranav Jan 2, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions hadoop-tools/hadoop-azure/.gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
.checkstyle
bin/
src/test/resources/combinationConfigFiles
src/test/resources/abfs-combination-test-configs.xml
dev-support/testlogs
src/test/resources/accountSettings/*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import org.apache.hadoop.fs.azurebfs.diagnostics.StringConfigurationBasicValidator;
import org.apache.hadoop.fs.azurebfs.enums.Trilean;
import org.apache.hadoop.fs.azurebfs.extensions.CustomTokenProviderAdaptee;
import org.apache.hadoop.fs.azurebfs.extensions.EncryptionContextProvider;
import org.apache.hadoop.fs.azurebfs.extensions.SASTokenProvider;
import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
import org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider;
Expand Down Expand Up @@ -328,6 +329,9 @@ public class AbfsConfiguration{
FS_AZURE_ENABLE_ABFS_LIST_ITERATOR, DefaultValue = DEFAULT_ENABLE_ABFS_LIST_ITERATOR)
private boolean enableAbfsListIterator;

private String clientProvidedEncryptionKey;
private String clientProvidedEncryptionKeySHA;

public AbfsConfiguration(final Configuration rawConfig, String accountName)
throws IllegalAccessException, InvalidConfigurationValueException, IOException {
this.rawConfig = ProviderUtils.excludeIncompatibleCredentialProviders(
Expand Down Expand Up @@ -944,6 +948,32 @@ public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemExceptio
}
}

public EncryptionContextProvider createEncryptionContextProvider() {
try {
String configKey = FS_AZURE_ENCRYPTION_CONTEXT_PROVIDER_TYPE;
if (get(configKey) == null) {
return null;
}
Class<? extends EncryptionContextProvider> encryptionContextClass =
getAccountSpecificClass(configKey, null,
EncryptionContextProvider.class);
Preconditions.checkArgument(encryptionContextClass != null, String.format(
"The configuration value for %s is invalid, or config key is not account-specific",
configKey));

EncryptionContextProvider encryptionContextProvider =
ReflectionUtils.newInstance(encryptionContextClass, rawConfig);
Preconditions.checkArgument(encryptionContextProvider != null,
String.format("Failed to initialize %s", encryptionContextClass));

LOG.trace("{} init complete", encryptionContextClass.getName());
return encryptionContextProvider;
} catch (Exception e) {
throw new IllegalArgumentException(
"Unable to load encryption context provider class: ", e);
}
}

public boolean isReadAheadEnabled() {
return this.enabledReadAhead;
}
Expand Down Expand Up @@ -1055,9 +1085,22 @@ public boolean enableAbfsListIterator() {
return this.enableAbfsListIterator;
}

public String getClientProvidedEncryptionKey() {
String accSpecEncKey = accountConf(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY);
return rawConfig.get(accSpecEncKey, null);
public String getEncodedClientProvidedEncryptionKey() {
if (clientProvidedEncryptionKey == null) {
String accSpecEncKey = accountConf(
FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY);
clientProvidedEncryptionKey = rawConfig.get(accSpecEncKey, null);
}
return clientProvidedEncryptionKey;
}

public String getEncodedClientProvidedEncryptionKeySHA() {
if (clientProvidedEncryptionKeySHA == null) {
String accSpecEncKey = accountConf(
FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY_SHA);
clientProvidedEncryptionKeySHA = rawConfig.get(accSpecEncKey, null);
}
return clientProvidedEncryptionKeySHA;
}

@VisibleForTesting
Expand Down
Loading