-
Notifications
You must be signed in to change notification settings - Fork 9.1k
HADOOP-16790: Add Write Convenience Methods #1792
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
f9a4c68
49848ac
fda16a1
4384386
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -21,23 +21,28 @@ | |
| import java.io.BufferedInputStream; | ||
| import java.io.BufferedOutputStream; | ||
| import java.io.BufferedReader; | ||
| import java.io.BufferedWriter; | ||
| import java.io.File; | ||
| import java.io.FileNotFoundException; | ||
| import java.io.IOException; | ||
| import java.io.InputStream; | ||
| import java.io.InputStreamReader; | ||
| import java.io.OutputStream; | ||
| import java.io.OutputStreamWriter; | ||
| import java.net.InetAddress; | ||
| import java.net.URI; | ||
| import java.net.UnknownHostException; | ||
| import java.nio.charset.Charset; | ||
| import java.nio.charset.CharsetEncoder; | ||
| import java.nio.charset.StandardCharsets; | ||
| import java.nio.file.AccessDeniedException; | ||
| import java.nio.file.FileSystems; | ||
| import java.nio.file.Files; | ||
| import java.util.ArrayList; | ||
| import java.util.Enumeration; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Objects; | ||
| import java.util.concurrent.ExecutionException; | ||
| import java.util.concurrent.ExecutorService; | ||
| import java.util.concurrent.Executors; | ||
|
|
@@ -1633,4 +1638,235 @@ public static boolean compareFs(FileSystem srcFs, FileSystem destFs) { | |
| // check for ports | ||
| return srcUri.getPort()==dstUri.getPort(); | ||
| } | ||
|
|
||
| /** | ||
| * Writes bytes to a file. This utility method opens the file for writing, | ||
| * creating the file if it does not exist, or overwrites an existing file. All | ||
| * bytes in the byte array are written to the file. | ||
| * | ||
| * @param fs the file system with which to create the file | ||
| * @param path the path to the file | ||
| * @param bytes the byte array with the bytes to write | ||
| * | ||
| * @return the file system | ||
| * | ||
| * @throws NullPointerException if any of the arguments are {@code null} | ||
| * @throws IOException if an I/O error occurs creating or writing to the file | ||
| */ | ||
| public static FileSystem write(final FileSystem fs, final Path path, | ||
| final byte[] bytes) throws IOException { | ||
|
|
||
| Objects.requireNonNull(path); | ||
| Objects.requireNonNull(bytes); | ||
|
|
||
| try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build()) { | ||
| out.write(bytes); | ||
| } | ||
|
|
||
| return fs; | ||
| } | ||
|
|
||
| /** | ||
| * Writes bytes to a file. This utility method opens the file for writing, | ||
| * creating the file if it does not exist, or overwrites an existing file. All | ||
| * bytes in the byte array are written to the file. | ||
| * | ||
| * @param fileContext the file context with which to create the file | ||
| * @param path the path to the file | ||
| * @param bytes the byte array with the bytes to write | ||
| * | ||
| * @return the file context | ||
| * | ||
| * @throws NullPointerException if any of the arguments are {@code null} | ||
| * @throws IOException if an I/O error occurs creating or writing to the file | ||
| */ | ||
| public static FileContext write(final FileContext fileContext, | ||
| final Path path, final byte[] bytes) throws IOException { | ||
|
|
||
| Objects.requireNonNull(path); | ||
| Objects.requireNonNull(bytes); | ||
|
|
||
| try (FSDataOutputStream out = | ||
| fileContext.create(path).overwrite(true).build()) { | ||
| out.write(bytes); | ||
| } | ||
|
|
||
| return fileContext; | ||
| } | ||
|
|
||
| /** | ||
| * Write lines of text to a file. Each line is a char sequence and is written | ||
| * to the file in sequence with each line terminated by the platform's line | ||
| * separator, as defined by the system property {@code | ||
| * line.separator}. Characters are encoded into bytes using the specified | ||
| * charset. This utility method opens the file for writing, creating the file | ||
| * if it does not exist, or overwrites an existing file. | ||
| * | ||
| * @param fs the file system with which to create the file | ||
| * @param path the path to the file | ||
| * @param lines a Collection to iterate over the char sequences | ||
| * @param cs the charset to use for encoding | ||
| * | ||
| * @return the file system | ||
| * | ||
| * @throws NullPointerException if any of the arguments are {@code null} | ||
| * @throws IOException if an I/O error occurs creating or writing to the file | ||
| */ | ||
| public static FileSystem write(final FileSystem fs, final Path path, | ||
| final Iterable<? extends CharSequence> lines, final Charset cs) | ||
| throws IOException { | ||
|
|
||
| Objects.requireNonNull(path); | ||
| Objects.requireNonNull(lines); | ||
| Objects.requireNonNull(cs); | ||
|
|
||
| CharsetEncoder encoder = cs.newEncoder(); | ||
| try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build(); | ||
| BufferedWriter writer = | ||
| new BufferedWriter(new OutputStreamWriter(out, encoder))) { | ||
| for (CharSequence line : lines) { | ||
| writer.append(line); | ||
| writer.newLine(); | ||
| } | ||
| } | ||
| return fs; | ||
| } | ||
|
|
||
| /** | ||
| * Write lines of text to a file. Each line is a char sequence and is written | ||
| * to the file in sequence with each line terminated by the platform's line | ||
| * separator, as defined by the system property {@code | ||
| * line.separator}. Characters are encoded into bytes using the specified | ||
| * charset. This utility method opens the file for writing, creating the file | ||
| * if it does not exist, or overwrites an existing file. | ||
| * | ||
| * @param fileContext the file context with which to create the file | ||
| * @param path the path to the file | ||
| * @param lines a Collection to iterate over the char sequences | ||
| * @param cs the charset to use for encoding | ||
| * | ||
| * @return the file context | ||
| * | ||
| * @throws NullPointerException if any of the arguments are {@code null} | ||
| * @throws IOException if an I/O error occurs creating or writing to the file | ||
| */ | ||
| public static FileContext write(final FileContext fileContext, | ||
| final Path path, final Iterable<? extends CharSequence> lines, | ||
| final Charset cs) throws IOException { | ||
|
|
||
| Objects.requireNonNull(path); | ||
| Objects.requireNonNull(lines); | ||
| Objects.requireNonNull(cs); | ||
|
|
||
| CharsetEncoder encoder = cs.newEncoder(); | ||
| try (FSDataOutputStream out = fileContext.create(path).overwrite(true).build(); | ||
| BufferedWriter writer = | ||
| new BufferedWriter(new OutputStreamWriter(out, encoder))) { | ||
| for (CharSequence line : lines) { | ||
| writer.append(line); | ||
| writer.newLine(); | ||
| } | ||
| } | ||
| return fileContext; | ||
| } | ||
|
|
||
| /** | ||
| * Write a line of text to a file. Characters are encoded into bytes using the | ||
| * specified charset. This utility method opens the file for writing, creating | ||
| * the file if it does not exist, or overwrites an existing file. | ||
| * | ||
| * @param fs the file system with which to create the file | ||
| * @param path the path to the file | ||
| * @param charseq the char sequence to write to the file | ||
| * @param cs the charset to use for encoding | ||
| * | ||
| * @return the file system | ||
| * | ||
| * @throws NullPointerException if any of the arguments are {@code null} | ||
| * @throws IOException if an I/O error occurs creating or writing to the file | ||
| */ | ||
| public static FileSystem write(final FileSystem fs, final Path path, | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. add in overwrite options. We've been dealing with 404 caching in S3A, which relies on createFile(overwrite = false). Unless you make the default, it must be something callers can use. |
||
| final CharSequence charseq, final Charset cs) throws IOException { | ||
|
|
||
| Objects.requireNonNull(path); | ||
| Objects.requireNonNull(charseq); | ||
| Objects.requireNonNull(cs); | ||
|
|
||
| CharsetEncoder encoder = cs.newEncoder(); | ||
| try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build(); | ||
| BufferedWriter writer = | ||
| new BufferedWriter(new OutputStreamWriter(out, encoder))) { | ||
| writer.append(charseq); | ||
| } | ||
| return fs; | ||
| } | ||
|
|
||
| /** | ||
| * Write a line of text to a file. Characters are encoded into bytes using the | ||
| * specified charset. This utility method opens the file for writing, creating | ||
| * the file if it does not exist, or overwrites an existing file. | ||
| * | ||
| * @param FileContext the file context with which to create the file | ||
| * @param path the path to the file | ||
| * @param charseq the char sequence to write to the file | ||
| * @param cs the charset to use for encoding | ||
| * | ||
| * @return the file context | ||
| * | ||
| * @throws NullPointerException if any of the arguments are {@code null} | ||
| * @throws IOException if an I/O error occurs creating or writing to the file | ||
| */ | ||
| public static FileContext write(final FileContext fs, final Path path, | ||
| final CharSequence charseq, final Charset cs) throws IOException { | ||
|
|
||
| Objects.requireNonNull(path); | ||
| Objects.requireNonNull(charseq); | ||
| Objects.requireNonNull(cs); | ||
|
|
||
| CharsetEncoder encoder = cs.newEncoder(); | ||
| try (FSDataOutputStream out = fs.create(path).overwrite(true).build(); | ||
| BufferedWriter writer = | ||
| new BufferedWriter(new OutputStreamWriter(out, encoder))) { | ||
| writer.append(charseq); | ||
| } | ||
| return fs; | ||
| } | ||
|
|
||
| /** | ||
| * Write a line of text to a file. Characters are encoded into bytes using | ||
| * UTF-8. This utility method opens the file for writing, creating the file if | ||
| * it does not exist, or overwrites an existing file. | ||
| * | ||
| * @param fs the files system with which to create the file | ||
| * @param path the path to the file | ||
| * @param charseq the char sequence to write to the file | ||
| * | ||
| * @return the file system | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Might as well. Allows for method chaining. For example it's common to write a file into the tmp directory then move it into its final destination to avoid writing garbage into the target directory if the write fails.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. "common" as in not-object-store-optimised There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sure. Gives one the flexibility to take advantage of it or ignore it if it is of no value. |
||
| * | ||
| * @throws NullPointerException if any of the arguments are {@code null} | ||
| * @throws IOException if an I/O error occurs creating or writing to the file | ||
| */ | ||
| public static FileSystem write(final FileSystem fs, final Path path, | ||
| final CharSequence charseq) throws IOException { | ||
| return write(fs, path, charseq, StandardCharsets.UTF_8); | ||
| } | ||
|
|
||
| /** | ||
| * Write a line of text to a file. Characters are encoded into bytes using | ||
| * UTF-8. This utility method opens the file for writing, creating the file if | ||
| * it does not exist, or overwrites an existing file. | ||
| * | ||
| * @param fileContext the files system with which to create the file | ||
| * @param path the path to the file | ||
| * @param charseq the char sequence to write to the file | ||
| * | ||
| * @return the file context | ||
| * | ||
| * @throws NullPointerException if any of the arguments are {@code null} | ||
| * @throws IOException if an I/O error occurs creating or writing to the file | ||
| */ | ||
| public static FileContext write(final FileContext fileContext, | ||
| final Path path, final CharSequence charseq) throws IOException { | ||
| return write(fileContext, path, charseq, StandardCharsets.UTF_8); | ||
| } | ||
| } | ||
Uh oh!
There was an error while loading. Please reload this page.