Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,23 +21,28 @@
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.AccessDeniedException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
Expand Down Expand Up @@ -1633,4 +1638,235 @@ public static boolean compareFs(FileSystem srcFs, FileSystem destFs) {
// check for ports
return srcUri.getPort()==dstUri.getPort();
}

/**
* Writes bytes to a file. This utility method opens the file for writing,
* creating the file if it does not exist, or overwrites an existing file. All
* bytes in the byte array are written to the file.
*
* @param fs the file system with which to create the file
* @param path the path to the file
* @param bytes the byte array with the bytes to write
*
* @return the file system
*
* @throws NullPointerException if any of the arguments are {@code null}
* @throws IOException if an I/O error occurs creating or writing to the file
*/
public static FileSystem write(final FileSystem fs, final Path path,
final byte[] bytes) throws IOException {

Objects.requireNonNull(path);
Objects.requireNonNull(bytes);

try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build()) {
out.write(bytes);
}

return fs;
}

/**
* Writes bytes to a file. This utility method opens the file for writing,
* creating the file if it does not exist, or overwrites an existing file. All
* bytes in the byte array are written to the file.
*
* @param fileContext the file context with which to create the file
* @param path the path to the file
* @param bytes the byte array with the bytes to write
*
* @return the file context
*
* @throws NullPointerException if any of the arguments are {@code null}
* @throws IOException if an I/O error occurs creating or writing to the file
*/
public static FileContext write(final FileContext fileContext,
final Path path, final byte[] bytes) throws IOException {

Objects.requireNonNull(path);
Objects.requireNonNull(bytes);

try (FSDataOutputStream out =
fileContext.create(path).overwrite(true).build()) {
out.write(bytes);
}

return fileContext;
}

/**
* Write lines of text to a file. Each line is a char sequence and is written
* to the file in sequence with each line terminated by the platform's line
* separator, as defined by the system property {@code
* line.separator}. Characters are encoded into bytes using the specified
* charset. This utility method opens the file for writing, creating the file
* if it does not exist, or overwrites an existing file.
*
* @param fs the file system with which to create the file
* @param path the path to the file
* @param lines a Collection to iterate over the char sequences
* @param cs the charset to use for encoding
*
* @return the file system
*
* @throws NullPointerException if any of the arguments are {@code null}
* @throws IOException if an I/O error occurs creating or writing to the file
*/
public static FileSystem write(final FileSystem fs, final Path path,
final Iterable<? extends CharSequence> lines, final Charset cs)
throws IOException {

Objects.requireNonNull(path);
Objects.requireNonNull(lines);
Objects.requireNonNull(cs);

CharsetEncoder encoder = cs.newEncoder();
try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build();
BufferedWriter writer =
new BufferedWriter(new OutputStreamWriter(out, encoder))) {
for (CharSequence line : lines) {
writer.append(line);
writer.newLine();
}
}
return fs;
}

/**
* Write lines of text to a file. Each line is a char sequence and is written
* to the file in sequence with each line terminated by the platform's line
* separator, as defined by the system property {@code
* line.separator}. Characters are encoded into bytes using the specified
* charset. This utility method opens the file for writing, creating the file
* if it does not exist, or overwrites an existing file.
*
* @param fileContext the file context with which to create the file
* @param path the path to the file
* @param lines a Collection to iterate over the char sequences
* @param cs the charset to use for encoding
*
* @return the file context
*
* @throws NullPointerException if any of the arguments are {@code null}
* @throws IOException if an I/O error occurs creating or writing to the file
*/
public static FileContext write(final FileContext fileContext,
final Path path, final Iterable<? extends CharSequence> lines,
final Charset cs) throws IOException {

Objects.requireNonNull(path);
Objects.requireNonNull(lines);
Objects.requireNonNull(cs);

CharsetEncoder encoder = cs.newEncoder();
try (FSDataOutputStream out = fileContext.create(path).overwrite(true).build();
BufferedWriter writer =
new BufferedWriter(new OutputStreamWriter(out, encoder))) {
for (CharSequence line : lines) {
writer.append(line);
writer.newLine();
}
}
return fileContext;
}

/**
* Write a line of text to a file. Characters are encoded into bytes using the
* specified charset. This utility method opens the file for writing, creating
* the file if it does not exist, or overwrites an existing file.
*
* @param fs the file system with which to create the file
* @param path the path to the file
* @param charseq the char sequence to write to the file
* @param cs the charset to use for encoding
*
* @return the file system
*
* @throws NullPointerException if any of the arguments are {@code null}
* @throws IOException if an I/O error occurs creating or writing to the file
*/
public static FileSystem write(final FileSystem fs, final Path path,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add in overwrite options. We've been dealing with 404 caching in S3A, which relies on createFile(overwrite = false). Unless you make the default, it must be something callers can use.

final CharSequence charseq, final Charset cs) throws IOException {

Objects.requireNonNull(path);
Objects.requireNonNull(charseq);
Objects.requireNonNull(cs);

CharsetEncoder encoder = cs.newEncoder();
try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build();
BufferedWriter writer =
new BufferedWriter(new OutputStreamWriter(out, encoder))) {
writer.append(charseq);
}
return fs;
}

/**
* Write a line of text to a file. Characters are encoded into bytes using the
* specified charset. This utility method opens the file for writing, creating
* the file if it does not exist, or overwrites an existing file.
*
* @param FileContext the file context with which to create the file
* @param path the path to the file
* @param charseq the char sequence to write to the file
* @param cs the charset to use for encoding
*
* @return the file context
*
* @throws NullPointerException if any of the arguments are {@code null}
* @throws IOException if an I/O error occurs creating or writing to the file
*/
public static FileContext write(final FileContext fs, final Path path,
final CharSequence charseq, final Charset cs) throws IOException {

Objects.requireNonNull(path);
Objects.requireNonNull(charseq);
Objects.requireNonNull(cs);

CharsetEncoder encoder = cs.newEncoder();
try (FSDataOutputStream out = fs.create(path).overwrite(true).build();
BufferedWriter writer =
new BufferedWriter(new OutputStreamWriter(out, encoder))) {
writer.append(charseq);
}
return fs;
}

/**
* Write a line of text to a file. Characters are encoded into bytes using
* UTF-8. This utility method opens the file for writing, creating the file if
* it does not exist, or overwrites an existing file.
*
* @param fs the files system with which to create the file
* @param path the path to the file
* @param charseq the char sequence to write to the file
*
* @return the file system
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might as well. Allows for method chaining. For example it's common to write a file into the tmp directory then move it into its final destination to avoid writing garbage into the target directory if the write fails.

File.write(fs, tmpPath, byte[]).rename(tmpPath, path);

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"common" as in not-object-store-optimised

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure. Gives one the flexibility to take advantage of it or ignore it if it is of no value.

*
* @throws NullPointerException if any of the arguments are {@code null}
* @throws IOException if an I/O error occurs creating or writing to the file
*/
public static FileSystem write(final FileSystem fs, final Path path,
final CharSequence charseq) throws IOException {
return write(fs, path, charseq, StandardCharsets.UTF_8);
}

/**
* Write a line of text to a file. Characters are encoded into bytes using
* UTF-8. This utility method opens the file for writing, creating the file if
* it does not exist, or overwrites an existing file.
*
* @param fileContext the files system with which to create the file
* @param path the path to the file
* @param charseq the char sequence to write to the file
*
* @return the file context
*
* @throws NullPointerException if any of the arguments are {@code null}
* @throws IOException if an I/O error occurs creating or writing to the file
*/
public static FileContext write(final FileContext fileContext,
final Path path, final CharSequence charseq) throws IOException {
return write(fileContext, path, charseq, StandardCharsets.UTF_8);
}
}
Loading