Skip to content

Commit feb384f

Browse files
committed
HADOOP-19161. S3A: option "fs.s3a.performance.flags" to take list of performance flags
* naming * filesystem.toString() prints the values Change-Id: I54c5fec1d0cc19513e75a25ec9ef4482614b473c
1 parent 5b2974f commit feb384f

File tree

2 files changed

+35
-34
lines changed

2 files changed

+35
-34
lines changed

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -724,16 +724,16 @@ public void initialize(URI name, Configuration originalConf)
724724
checkNoS3Guard(this.getUri(), getConf());
725725

726726
// read in performance options and parse them to a list of flags.
727-
performanceFlags = S3APerformanceFlags.create(
727+
performanceFlags = S3APerformanceFlags.build(
728728
getConf().getTrimmedStrings(FS_S3A_PERFORMANCE_FLAGS));
729729
// performance creation flag for code which wants performance
730730
// at the risk of overwrites.
731731
// this uses the performance flags as the default and then
732732
// updates the performance flags to match.
733733
// a bit convoluted.
734734
boolean performanceCreation = conf.getBoolean(FS_S3A_CREATE_PERFORMANCE,
735-
performanceFlags.isCreatePerformance());
736-
performanceFlags.setCreatePerformance(performanceCreation);
735+
performanceFlags.isCreate());
736+
performanceFlags.setCreate(performanceCreation);
737737

738738
LOG.debug("{} = {}", FS_S3A_CREATE_PERFORMANCE, performanceCreation);
739739
allowAuthoritativePaths = S3Guard.getAuthoritativePaths(this);
@@ -1977,7 +1977,7 @@ public FSDataOutputStream create(Path f, FsPermission permission,
19771977

19781978
// work out the options to pass down
19791979
CreateFileBuilder.CreateFileOptions options;
1980-
if (getPerformanceFlags().isCreatePerformance()) {
1980+
if (getPerformanceFlags().isCreate()) {
19811981
options = OPTIONS_CREATE_FILE_PERFORMANCE;
19821982
} else {
19831983
options = overwrite
@@ -2150,7 +2150,7 @@ public FSDataOutputStreamBuilder createFile(final Path path) {
21502150
builder
21512151
.create()
21522152
.overwrite(true)
2153-
.must(FS_S3A_CREATE_PERFORMANCE, getPerformanceFlags().isCreatePerformance());
2153+
.must(FS_S3A_CREATE_PERFORMANCE, getPerformanceFlags().isCreate());
21542154
return builder;
21552155
} catch (IOException e) {
21562156
// catch any IOEs raised in span creation and convert to
@@ -2215,7 +2215,7 @@ public FSDataOutputStream createNonRecursive(Path p,
22152215
.withFlags(flags)
22162216
.blockSize(blockSize)
22172217
.bufferSize(bufferSize)
2218-
.must(FS_S3A_CREATE_PERFORMANCE, performanceFlags.isCreatePerformance());
2218+
.must(FS_S3A_CREATE_PERFORMANCE, performanceFlags.isCreate());
22192219
if (progress != null) {
22202220
builder.progress(progress);
22212221
}
@@ -4838,6 +4838,7 @@ public String toString() {
48384838
sb.append(", partSize=").append(partSize);
48394839
sb.append(", enableMultiObjectsDelete=").append(enableMultiObjectsDelete);
48404840
sb.append(", maxKeys=").append(maxKeys);
4841+
sb.append(", ").append(performanceFlags);
48414842
if (cannedACL != null) {
48424843
sb.append(", cannedACL=").append(cannedACL);
48434844
}

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3APerformanceFlags.java

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -54,45 +54,45 @@ public final class S3APerformanceFlags implements StreamCapabilities {
5454
/**
5555
* Higher performance create operations.
5656
*/
57-
private boolean createPerformance;
57+
private boolean create;
5858

5959
/**
6060
* Delete operation to skip parent probe.
6161
*/
62-
private boolean deletePerformance;
62+
private boolean delete;
6363

6464
/**
6565
* Mkdir to skip checking for type of parent paths.
6666
*/
67-
private boolean mkdirPerformance;
67+
private boolean mkdir;
6868

6969
public S3APerformanceFlags() {
7070
}
7171

72-
public boolean isCreatePerformance() {
73-
return createPerformance;
72+
public boolean isCreate() {
73+
return create;
7474
}
7575

76-
public boolean isDeletePerformance() {
77-
return deletePerformance;
76+
public boolean isDelete() {
77+
return delete;
7878
}
7979

80-
public boolean isMkdirPerformance() {
81-
return mkdirPerformance;
80+
public boolean isMkdir() {
81+
return mkdir;
8282
}
8383

84-
public S3APerformanceFlags setCreatePerformance(final boolean createPerformance) {
85-
this.createPerformance = createPerformance;
84+
public S3APerformanceFlags setCreate(final boolean create) {
85+
this.create = create;
8686
return this;
8787
}
8888

89-
public S3APerformanceFlags setDeletePerformance(final boolean deletePerformance) {
90-
this.deletePerformance = deletePerformance;
89+
public S3APerformanceFlags setDelete(final boolean delete) {
90+
this.delete = delete;
9191
return this;
9292
}
9393

94-
public S3APerformanceFlags setMkdirPerformance(final boolean mkdirPerformance) {
95-
this.mkdirPerformance = mkdirPerformance;
94+
public S3APerformanceFlags setMkdir(final boolean mkdir) {
95+
this.mkdir = mkdir;
9696
return this;
9797
}
9898

@@ -102,13 +102,13 @@ public boolean hasCapability(final String capability) {
102102
switch (capability.toLowerCase(Locale.ROOT)) {
103103
case FS_S3A_PERFORMANCE_FLAGS + CREATE:
104104
case FS_S3A_CREATE_PERFORMANCE_ENABLED:
105-
return isCreatePerformance();
105+
return isCreate();
106106

107107
case FS_S3A_PERFORMANCE_FLAGS + MKDIR:
108-
return isMkdirPerformance();
108+
return isMkdir();
109109

110110
case FS_S3A_PERFORMANCE_FLAGS + DELETE:
111-
return isDeletePerformance();
111+
return isDelete();
112112

113113
default:
114114
}
@@ -118,9 +118,9 @@ public boolean hasCapability(final String capability) {
118118
@Override
119119
public String toString() {
120120
return "S3APerformanceFlags{" +
121-
"createPerformance=" + createPerformance +
122-
", deletePerformance=" + deletePerformance +
123-
", mkdirPerformance=" + mkdirPerformance +
121+
"create=" + create +
122+
", delete=" + delete +
123+
", mkdir=" + mkdir +
124124
'}';
125125
}
126126

@@ -129,23 +129,23 @@ public String toString() {
129129
* @param options options from a configuration string.
130130
* @return a set of options
131131
*/
132-
public static S3APerformanceFlags create(String... options) {
132+
public static S3APerformanceFlags build(String... options) {
133133
S3APerformanceFlags flags = new S3APerformanceFlags();
134134
for (String option : options) {
135135
switch (option.trim().toLowerCase(Locale.ROOT)) {
136136
case CREATE:
137-
flags.createPerformance = true;
137+
flags.create = true;
138138
break;
139139
case DELETE:
140-
flags.deletePerformance = true;
140+
flags.delete = true;
141141
break;
142142
case MKDIR:
143-
flags.mkdirPerformance = true;
143+
flags.mkdir = true;
144144
break;
145145
case ALL:
146-
flags.createPerformance = true;
147-
flags.mkdirPerformance = true;
148-
flags.deletePerformance = true;
146+
flags.create = true;
147+
flags.mkdir = true;
148+
flags.delete = true;
149149
break;
150150

151151
/* case "hive":

0 commit comments

Comments
 (0)