HADOOP-13327 add RawLocalFileystem Syncable

Doesn't actually get through as there's a BufferedOutputStream
in the way... will need to do something there :)

Change-Id: Ib2e4517e266168f3ad106e55ceac987ed443aeec

Author: steveloughran

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java

@@ -46,6 +46,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.impl.StoreImplementationUtils;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.nativeio.NativeIO;
@@ -233,7 +234,8 @@ public FSDataInputStream open(PathHandle fd, int bufferSize)
   /*********************************************************
    * For create()'s FSOutputStream.
    *********************************************************/
-  class LocalFSFileOutputStream extends OutputStream {
+  class LocalFSFileOutputStream extends OutputStream
+      implements Syncable, StreamCapabilities {
     private FileOutputStream fos;
 
     private LocalFSFileOutputStream(Path f, boolean append,
@@ -288,6 +290,22 @@ public void write(int b) throws IOException {
         throw new FSError(e);                // assume native fs error
       }
     }
+
+    @Override
+    public void hflush() throws IOException {
+      flush();
+    }
+
+    @Override
+    public void hsync() throws IOException {
+      flush();
+      fos.getFD().sync();
+    }
+
+    @Override
+    public boolean hasCapability(String capability) {
+      return StoreImplementationUtils.supportsSyncable(capability);
+    }
   }
 
   @Override

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/outputstream.md

@@ -782,11 +782,8 @@ local data as can be written to full checksummed blocks of data.
 That is, the flush operations are not guaranteed to write all the pending
 data until the file is finally closed.
 
-That is, `sync()`, `hsync()` and `hflush()` may not persist all data written
-to the stream.
-
-For anyone thinking "this is a violation of this specification" —they are correct.
-The local filesystem was intended for testing, rather than production use.
+For this reason, the local fileystem accessed via `file://` URLs
+does not support syncable.
 
 ### <a name="checksummed-fs-issues"></a> Checksummed output streams
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.fs.contract;
 
+import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.FileStatus;
@@ -31,8 +32,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertCapabilities;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.getFileStatusEventually;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
@@ -454,15 +457,22 @@ public void testHSync() throws Throwable {
 
     try (FSDataOutputStream out = fs.create(path, true)) {
 
-      boolean doesHFlush = out.hasCapability(
-          StreamCapabilities.StreamCapability.HFLUSH.getValue());
+      boolean doesHFlush = out.hasCapability(StreamCapabilities.HFLUSH);
       if (doesHFlush) {
         out.hflush();
       }
-      assertEquals("hflush support", supportsFlush, doesHFlush);
-      boolean doesHSync = out.hasCapability(
-          StreamCapabilities.StreamCapability.HSYNC.getValue());
-      assertEquals("hsync support", supportsSync, doesHSync);
+      String[] hflushCapabilities = {
+          StreamCapabilities.HFLUSH,
+      };
+      String[] hsyncCapabilities = {
+          StreamCapabilities.HSYNC,
+      };
+      assertCapabilities(out,
+          supportsFlush ? hflushCapabilities : null,
+          supportsFlush ? null : hflushCapabilities);
+      assertCapabilities(out,
+          supportsSync ? hsyncCapabilities : null,
+          supportsSync ? null : hsyncCapabilities);
 
       try {
         out.hflush();

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractFSContractTestBase.java

@@ -295,7 +295,7 @@ protected void handleRelaxedException(String action,
     if (getContract().isSupported(SUPPORTS_STRICT_EXCEPTIONS, false)) {
       throw e;
     }
-    LOG.warn("The expected exception {} was not the exception class" +
+    LOG.warn("The expected exception {}  was not the exception class" +
              " raised on {}: {}", action , e.getClass(), expectedException, e);
   }
 
@@ -398,13 +398,4 @@ protected String generateAndLogErrorListing(Path src, Path dst) throws
     }
     return destDirLS;
   }
-
-  /**
-   * Get a path from the name of the current test case.
-   * @return a path based on the test method being executed.
-   * @throws IOException IO problem
-   */
-  protected Path methodPath() throws IOException {
-    return path(methodName.getMethodName());
-  }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java

@@ -1496,14 +1496,16 @@ public static void assertCapabilities(
     StreamCapabilities source = (StreamCapabilities) stream;
     if (shouldHaveCapabilities != null) {
       for (String shouldHaveCapability : shouldHaveCapabilities) {
-        assertTrue("Should have capability: " + shouldHaveCapability,
+        assertTrue("Should have capability: " + shouldHaveCapability
+                + " in " + source,
             source.hasCapability(shouldHaveCapability));
       }
     }
 
     if (shouldNotHaveCapabilities != null) {
       for (String shouldNotHaveCapability : shouldNotHaveCapabilities) {
-        assertFalse("Should not have capability: " + shouldNotHaveCapability,
+        assertFalse("Should not have capability: " + shouldNotHaveCapability
+                + " in " + source,
             source.hasCapability(shouldNotHaveCapability));
       }
     }
@@ -1523,7 +1525,8 @@ public static void assertHasPathCapabilities(
 
     for (String shouldHaveCapability: capabilities) {
       assertTrue("Should have capability: " + shouldHaveCapability
-              + " under " + path,
+              + " under " + path
+              + " in " + source,
           source.hasPathCapability(path, shouldHaveCapability));
     }
   }

hadoop-common-project/hadoop-common/src/test/resources/contract/localfs.xml

@@ -121,4 +121,14 @@ case sensitivity and permission options are determined at run time from OS type
     <value>true</value>
   </property>
 
+  <property>
+    <name>fs.contract.supports-settimes</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-getfilestatus</name>
+    <value>true</value>
+  </property>
+
 </configuration>

hadoop-common-project/hadoop-common/src/test/resources/contract/rawlocal.xml

@@ -127,4 +127,14 @@
     <value>true</value>
   </property>
 
+  <property>
+    <name>fs.contract.supports-hflush</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-hsync</name>
+    <value>true</value>
+  </property>
+
 </configuration>