[WASMFS] Implement Open File Table, dup and dup2 syscalls (#15181)

Relevant Issue: #15041

- Implement Open File Table for WASMFS
- Implement dup and dup2 syscalls
- File Handle locking mechanism
- Add test to verify that dup and dup2 work and that printing to stdout is still functional using the new open file table data structure

Author: ethanalee

system/lib/wasmfs/file.h

@@ -0,0 +1,44 @@
+// Copyright 2021 The Emscripten Authors.  All rights reserved.
+// Emscripten is available under two separate licenses, the MIT license and the
+// University of Illinois/NCSA Open Source License.  Both these licenses can be
+// found in the LICENSE file.
+// This file defines the file object of the new file system.
+// Current Status: Work in Progress.
+// See https://github.com/emscripten-core/emscripten/issues/15041.
+
+#pragma once
+
+#include <emscripten/html5.h>
+#include <mutex>
+#include <vector>
+#include <wasi/api.h>
+
+namespace wasmfs {
+
+class File {
+  // TODO: Add other File properties later.
+
+  // A mutex is needed for multiple accesses to the same file.
+  std::mutex mutex;
+
+  virtual __wasi_errno_t read(const uint8_t* buf, __wasi_size_t len) = 0;
+  virtual __wasi_errno_t write(const uint8_t* buf, __wasi_size_t len) = 0;
+
+public:
+  virtual ~File() = default;
+  class Handle {
+    File& file;
+    std::unique_lock<std::mutex> lock;
+
+  public:
+    Handle(File& file) : file(file), lock(file.mutex) {}
+    Handle(Handle&&) = default;
+
+    __wasi_errno_t read(const uint8_t* buf, __wasi_size_t len) { return file.read(buf, len); }
+    __wasi_errno_t write(const uint8_t* buf, __wasi_size_t len) { return file.write(buf, len); }
+  };
+
+  Handle get() { return Handle(*this); }
+};
+
+} // namespace wasmfs

system/lib/wasmfs/file_table.cpp

@@ -0,0 +1,112 @@
+// Copyright 2021 The Emscripten Authors.  All rights reserved.
+// Emscripten is available under two separate licenses, the MIT license and the
+// University of Illinois/NCSA Open Source License.  Both these licenses can be
+// found in the LICENSE file.
+// This file defines the open file table of the new file system.
+// Current Status: Work in Progress.
+// See https://github.com/emscripten-core/emscripten/issues/15041.
+
+#include "file_table.h"
+
+namespace wasmfs {
+
+std::vector<std::shared_ptr<OpenFileState>> FileTable::entries;
+
+static __wasi_errno_t writeStdBuffer(const uint8_t* buf, __wasi_size_t len,
+  void (*console_write)(const char*), std::vector<char>& fd_write_buffer) {
+  for (__wasi_size_t j = 0; j < len; j++) {
+    uint8_t current = buf[j];
+    if (current == '\0' || current == '\n') {
+      fd_write_buffer.push_back('\0'); // for null-terminated C strings
+      console_write(&fd_write_buffer[0]);
+      fd_write_buffer.clear();
+    } else {
+      fd_write_buffer.push_back(current);
+    }
+  }
+  return __WASI_ERRNO_SUCCESS;
+}
+
+class StdinFile : public File {
+
+  __wasi_errno_t write(const uint8_t* buf, __wasi_size_t len) override {
+    return __WASI_ERRNO_INVAL;
+  }
+
+  __wasi_errno_t read(const uint8_t* buf, __wasi_size_t len) override {
+    return __WASI_ERRNO_INVAL;
+  };
+};
+
+class StdoutFile : public File {
+  std::vector<char> writeBuffer;
+
+  __wasi_errno_t write(const uint8_t* buf, __wasi_size_t len) override {
+    return writeStdBuffer(buf, len, &emscripten_console_log, writeBuffer);
+  }
+
+  __wasi_errno_t read(const uint8_t* buf, __wasi_size_t len) override {
+    return __WASI_ERRNO_INVAL;
+  };
+};
+
+class StderrFile : public File {
+  std::vector<char> writeBuffer;
+
+  // TODO: May not want to proxy stderr (fd == 2) to the main thread.
+  // This will not show in HTML - a console.warn in a worker is sufficient.
+  // This would be a change from the current FS.
+  __wasi_errno_t write(const uint8_t* buf, __wasi_size_t len) override {
+    return writeStdBuffer(buf, len, &emscripten_console_error, writeBuffer);
+  }
+
+  __wasi_errno_t read(const uint8_t* buf, __wasi_size_t len) override {
+    return __WASI_ERRNO_INVAL;
+  };
+};
+
+FileTable::FileTable() {
+  entries.push_back(std::make_shared<OpenFileState>(0, std::make_shared<StdinFile>()));
+  entries.push_back(std::make_shared<OpenFileState>(0, std::make_shared<StdoutFile>()));
+  entries.push_back(std::make_shared<OpenFileState>(0, std::make_shared<StderrFile>()));
+}
+
+FileTable::Handle FileTable::get() {
+  static FileTable fileTable;
+  return FileTable::Handle(fileTable);
+}
+
+// Operator Overloading for FileTable::Handle::Entry
+FileTable::Handle::Entry::operator std::shared_ptr<OpenFileState>() const {
+  if (fd >= fileTableHandle.fileTable.entries.size() || fd < 0) {
+    return nullptr;
+  }
+
+  return fileTableHandle.fileTable.entries[fd];
+}
+
+FileTable::Handle::Entry& FileTable::Handle::Entry::operator=(std::shared_ptr<OpenFileState> ptr) {
+  assert(fd >= 0);
+
+  if (fd >= fileTableHandle.fileTable.entries.size()) {
+    fileTableHandle.fileTable.entries.resize(fd + 1);
+  }
+  fileTableHandle.fileTable.entries[fd] = ptr;
+
+  return *this;
+}
+
+std::shared_ptr<OpenFileState>& FileTable::Handle::Entry::operator->() {
+  assert(fd < fileTableHandle.fileTable.entries.size() && fd >= 0);
+
+  return fileTableHandle.fileTable.entries[fd];
+}
+
+FileTable::Handle::Entry::operator bool() const {
+  if (fd >= fileTableHandle.fileTable.entries.size() || fd < 0) {
+    return false;
+  }
+
+  return fileTableHandle.fileTable.entries[fd] != nullptr;
+}
+} // namespace wasmfs

system/lib/wasmfs/file_table.h

@@ -0,0 +1,90 @@
+// Copyright 2021 The Emscripten Authors.  All rights reserved.
+// Emscripten is available under two separate licenses, the MIT license and the
+// University of Illinois/NCSA Open Source License.  Both these licenses can be
+// found in the LICENSE file.
+// This file defines the open file table of the new file system.
+// Current Status: Work in Progress.
+// See https://github.com/emscripten-core/emscripten/issues/15041.
+
+#pragma once
+
+#include "file.h"
+#include <assert.h>
+#include <mutex>
+#include <utility>
+#include <vector>
+#include <wasi/api.h>
+
+namespace wasmfs {
+
+class OpenFileState {
+  std::shared_ptr<File> file;
+  __wasi_filedelta_t offset;
+  std::mutex mutex;
+
+public:
+  OpenFileState(uint32_t offset, std::shared_ptr<File> file) : offset(offset), file(file) {}
+
+  class Handle {
+    OpenFileState& openFileState;
+    std::unique_lock<std::mutex> lock;
+
+  public:
+    Handle(OpenFileState& openFileState)
+      : openFileState(openFileState), lock(openFileState.mutex) {}
+
+    std::shared_ptr<File>& getFile() { return openFileState.file; };
+  };
+
+  Handle get() { return Handle(*this); }
+};
+
+class FileTable {
+  static std::vector<std::shared_ptr<OpenFileState>> entries;
+  std::mutex mutex;
+
+  FileTable();
+
+public:
+  // Handle represents an RAII wrapper object. Access to the global FileTable must go through a
+  // Handle. A Handle holds the single global FileTable's lock for the duration of its lifetime.
+  // This is necessary because a FileTable may have atomic operations where the lock must be held
+  // across multiple methods. By providing access through the handle, callers of file table methods
+  // do not need to remember to take a lock for every access.
+  class Handle {
+    FileTable& fileTable;
+    std::unique_lock<std::mutex> lock;
+
+  public:
+    Handle(FileTable& fileTable) : fileTable(fileTable), lock(fileTable.mutex) {}
+
+    // The Entry class abstracts over the list of entries, providing a simple and safe interface
+    // that looks much like accessing a std::map, in that table[x] = y will allocate a new entry if
+    // one is not already present there. One minor difference from std::map is that table[x] does
+    // not return a reference, and can be used to check for the lack of an item there without
+    // allocation (similar to how table[x] works on a JS object), which keeps syntax concise.
+    struct Entry {
+      // Need to store a reference to the single global filetable, which is a local static variable.
+      Handle& fileTableHandle;
+      __wasi_fd_t fd;
+
+      operator std::shared_ptr<OpenFileState>() const;
+
+      Entry& operator=(std::shared_ptr<OpenFileState> ptr);
+
+      Entry& operator=(Entry& entry) { return *this = std::shared_ptr<OpenFileState>(entry); }
+
+      std::shared_ptr<OpenFileState>& operator->();
+
+      // Check whether the entry exists (i.e. contains an OpenFileState).
+      operator bool() const;
+    };
+
+    Entry operator[](__wasi_fd_t fd) { return Entry{*this, fd}; };
+  };
+
+  // This get method is responsible for lazily initializing the FileTable.
+  // There is only ever one FileTable in the system.
+  static Handle get();
+};
+} // namespace wasmfs

system/lib/wasmfs/wasmfs.cpp

@@ -6,47 +6,87 @@
 // Current Status: Work in Progress.
 // See https://github.com/emscripten-core/emscripten/issues/15041.
 
+#include "file.h"
+#include "file_table.h"
 #include <emscripten/emscripten.h>
 #include <emscripten/html5.h>
+#include <errno.h>
+#include <mutex>
 #include <stdlib.h>
+#include <utility>
 #include <vector>
 #include <wasi/api.h>
 
 extern "C" {
 
-static std::vector<char> fd_write_stdstream_buffer;
+using namespace wasmfs;
+
+long __syscall_dup2(long oldfd, long newfd) {
+  FileTable::Handle fileTable = FileTable::get();
+
+  auto oldOpenFile = fileTable[oldfd];
+  // If oldfd is not a valid file descriptor, then the call fails,
+  // and newfd is not closed.
+  if (!oldOpenFile) {
+    return -(EBADF);
+  }
+
+  if (newfd < 0) {
+    return -(EBADF);
+  }
+
+  if (oldfd == newfd) {
+    return oldfd;
+  }
+
+  // If the file descriptor newfd was previously open, it will just be overwritten silently.
+  fileTable[newfd] = oldOpenFile;
+  return newfd;
+}
+
+long __syscall_dup(long fd) {
+
+  FileTable::Handle fileTable = FileTable::get();
+
+  // Check that an open file exists corresponding to the given fd.
+  auto currentOpenFile = fileTable[fd];
+  if (!currentOpenFile) {
+    return -(EBADF);
+  }
+
+  // Adds given OpenFileState to FileTable entries. Returns fd (insertion index in entries).
+  // If no free space is found, currentOpenFile will be appended to the back of the entries.
+  for (__wasi_fd_t i = 0;; i++) {
+    if (!fileTable[i]) {
+      // Free open file entry.
+      fileTable[i] = currentOpenFile;
+      return i;
+    }
+  }
+
+  return -(EBADF);
+}
 
 __wasi_errno_t __wasi_fd_write(
   __wasi_fd_t fd, const __wasi_ciovec_t* iovs, size_t iovs_len, __wasi_size_t* nwritten) {
-  // FD 1 = STDOUT and FD 2 = STDERR.
-  // Temporary hardcoding of filedescriptor values.
-  // TODO: May not want to proxy stderr (fd == 2) to the main thread.
-  // This will not show in HTML - a console.warn in a worker is suffficient.
-  // This would be a change from the current FS.
-  if (fd == 1 || fd == 2) {
-    __wasi_size_t num = 0;
-    for (size_t i = 0; i < iovs_len; i++) {
-      const uint8_t* buf = iovs[i].buf;
-      __wasi_size_t len = iovs[i].buf_len;
-      for (__wasi_size_t j = 0; j < len; j++) {
-        uint8_t current = buf[j];
-        if (current == 0 || current == 10) {
-          fd_write_stdstream_buffer.push_back('\0'); // for null-terminated C strings
-          if (fd == 1) {
-            emscripten_console_log(&fd_write_stdstream_buffer[0]);
-          } else if (fd == 2) {
-            emscripten_console_error(&fd_write_stdstream_buffer[0]);
-          }
-          fd_write_stdstream_buffer.clear();
-        } else {
-          fd_write_stdstream_buffer.push_back(current);
-        }
-      }
-      num += len;
-    }
-    *nwritten = num;
+  FileTable::Handle fileTable = FileTable::get();
+  if (!fileTable[fd]) {
+    return __WASI_ERRNO_BADF;
+  }
+
+  auto file = fileTable[fd]->get().getFile()->get();
+
+  __wasi_size_t num = 0;
+  for (size_t i = 0; i < iovs_len; i++) {
+    const uint8_t* buf = iovs[i].buf;
+    __wasi_size_t len = iovs[i].buf_len;
+
+    file.write(buf, len);
+    num += len;
   }
-  return 0;
+  *nwritten = num;
+
+  return __WASI_ERRNO_SUCCESS;
 }
 
 __wasi_errno_t __wasi_fd_seek(
@@ -62,7 +102,21 @@ __wasi_errno_t __wasi_fd_close(__wasi_fd_t fd) {
 
 __wasi_errno_t __wasi_fd_read(
   __wasi_fd_t fd, const __wasi_iovec_t* iovs, size_t iovs_len, __wasi_size_t* nread) {
-  emscripten_console_log("__wasi_fd_read has been temporarily stubbed and is inert");
-  abort();
+  FileTable::Handle fileTable = FileTable::get();
+  if (!fileTable[fd]) {
+    return __WASI_ERRNO_BADF;
+  }
+
+  auto file = fileTable[fd]->get().getFile()->get();
+  __wasi_size_t num = 0;
+  for (size_t i = 0; i < iovs_len; i++) {
+    const uint8_t* buf = iovs[i].buf;
+    __wasi_size_t len = iovs[i].buf_len;
+
+    file.read(buf, len);
+    num += len;
+  }
+  *nread = num;
+  return __WASI_ERRNO_INVAL;
 }
 }

tests/common.py

@@ -556,8 +556,6 @@ def build(self, filename, libraries=[], includes=[], force_c=False, js_outfile=T
     suffix = '.js' if js_outfile else '.wasm'
     compiler = [compiler_for(filename, force_c)]
     if compiler[0] == EMCC:
-      # TODO change test behaviour in the future when WASMFS becomes default file system
-      # WASMFS is excluded here since it currently requires stdlib++ functions
       # TODO(https://github.com/emscripten-core/emscripten/issues/11121)
       # For historical reasons emcc compiles and links as C++ by default.
       # However we want to run our tests in a more strict manner.  We can