Add OSS support (#199)

Author: zou000

WORKSPACE

@@ -300,3 +300,65 @@ http_archive(
         "http://github.com/libexpat/libexpat/archive/R_2_2_6.tar.gz",
     ],
 )
+
+http_archive(
+    name = "libapr1",
+    build_file = "//third_party:libapr1.BUILD",
+    patch_args = ["-p1"],
+    patches = [
+        "//third_party:libapr1.patch",
+    ],
+    sha256 = "1a0909a1146a214a6ab9de28902045461901baab4e0ee43797539ec05b6dbae0",
+    strip_prefix = "apr-1.6.5",
+    urls = [
+        "https://github.com/apache/apr/archive/1.6.5.tar.gz",
+    ],
+)
+
+http_archive(
+    name = "libaprutil1",
+    build_file = "//third_party:libaprutil1.BUILD",
+    patch_args = ["-p1"],
+    patches = [
+        "//third_party:libaprutil1.patch",
+    ],
+    sha256 = "4c9ae319cedc16890fc2776920e7d529672dda9c3a9a9abd53bd80c2071b39af",
+    strip_prefix = "apr-util-1.6.1",
+    urls = [
+        "https://github.com/apache/apr-util/archive/1.6.1.tar.gz",
+    ],
+)
+
+http_archive(
+    name = "mxml",
+    build_file = "//third_party:mxml.BUILD",
+    patch_args = ["-p1"],
+    patches = [
+        "//third_party:mxml.patch",
+    ],
+    sha256 = "4d850d15cdd4fdb9e82817eb069050d7575059a9a2729c82b23440e4445da199",
+    strip_prefix = "mxml-2.12",
+    urls = [
+        "https://github.com/michaelrsweet/mxml/archive/v2.12.tar.gz",
+    ],
+)
+
+http_archive(
+    name = "minini",
+    build_file = "//third_party:minini.BUILD",
+    sha256 = "a97dd5ac6811af95c8f2aeaa6894b3113377e78ffd585363c6848745760d0152",
+    strip_prefix = "minIni-1.0",
+    urls = [
+        "https://github.com/ElasticDL/minIni/archive/v1.0.tar.gz",
+    ],
+)
+
+http_archive(
+    name = "aliyun_oss_c_sdk",
+    build_file = "//third_party:oss_c_sdk.BUILD",
+    sha256 = "6450d3970578c794b23e9e1645440c6f42f63be3f82383097660db5cf2fba685",
+    strip_prefix = "aliyun-oss-c-sdk-3.7.0",
+    urls = [
+        "https://github.com/aliyun/aliyun-oss-c-sdk/archive/3.7.0.tar.gz",
+    ],
+)

tensorflow_io/oss/BUILD

@@ -0,0 +1,24 @@
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+cc_binary(
+    name = "python/ops/_oss_ops.so",
+    srcs = [
+        "kernels/ossfs/oss_file_system.cc",
+        "kernels/ossfs/oss_file_system.h",
+        "ops/ossfs_ops.cc",
+    ],
+    copts = [
+        "-D_GLIBCXX_USE_CXX11_ABI=0",
+    ],
+    linkshared = 1,
+    deps = [
+        "@aliyun_oss_c_sdk",
+        "@local_config_tf//:libtensorflow_framework",
+        "@local_config_tf//:tf_header_lib",
+        "@minini",
+    ],
+)

tensorflow_io/oss/README.md

@@ -0,0 +1,35 @@
+# TensorFlow OSS Filesystem Extension
+
+OSS is an object storage service provided by Alibaba Cloud, see [here](https://www.alibabacloud.com/product/oss) for more information about the service.
+
+This module provides an extension that emulates a filesystem using the object storage service. The directory structures are encoded in object keys and file contents are stored in objects. The extension is implemented using [OSS C SDK](https://github.com/aliyun/aliyun-oss-c-sdk).
+
+To use the extension, first save your OSS credential in a file,  in `INI` format:
+
+```
+[OSSCredentials]
+host = cn-hangzhou.oss.aliyun-inc.com
+accessid = your_oss_access_id
+accesskey = you_oss_access_key
+```
+
+Then set environment variable `OSS_CREDENTIALS` to the path of the file.
+
+In Python code, import the extension `ossfs_op` module to use the extension with `gfile`. The files and directory URI should have `oss://` prefix, followed by a bucket name, then the directory hierarchy.
+
+```python
+import tensorflow_io.oss.python.ops.ossfs_ops
+from tensorflow.python.platform import gfile
+
+gfile.MkDir('oss://your_bucket_name/test_dir')
+```
+
+With the extension installed, OSS files can be use with Dataset Ops, etc., in the same fashion as other files.
+
+```python
+dataset = tf.data.TextLineDataset(["oss://bucket_name/data_dir/file1"])
+```
+
+## Test
+
+File `tests/test_oss.py` contains basic filesystem functionality tests. See `README.md` in root directory for more information about running tests. Besides `OSS_CREDENTIALS`, the tests also require an `OSS_FS_TEST_BUCKET` environment variable containing an accessible bucket name. Make sure they are set before running `pytest tests`. You can also just run the OSS test using `pytest tests/test_oss.py`

tensorflow_io/oss/__init__.py

@@ -0,0 +1,32 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Alibaba OSS File System.
+
+@@ossfs_ops
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow_io.oss.python.ops import ossfs_ops  # pylint: disable=unused-import
+
+from tensorflow.python.util.all_util import remove_undocumented
+
+_allowed_symbols = [
+    "ossfs_ops",
+]
+
+remove_undocumented(__name__, allowed_exception_list=_allowed_symbols)