Skip to content

Convert AZFS to use C modular file system API #1143

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Nov 5, 2020
Merged

Convert AZFS to use C modular file system API #1143

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
81f853a
Convert AZFS to use C modular file system API
yongtang Sep 24, 2020
c746317
Avoid C++ dependency to libtensorflow_framework.so
yongtang Oct 25, 2020
a1e651e
Update to address CentOS 7 issues
yongtang Nov 2, 2020
07c8e9d
Bump version to TF 2.4.0rc0, and 0.17.0
yongtang Sep 24, 2020
e919dc8
Additional fix for CentOS 7
yongtang Nov 3, 2020
1d32414
Fix a small bug in return with EOF
yongtang Nov 3, 2020
a5bf115
Address review comments
yongtang Nov 5, 2020
6c203fc
Add missing `"` typo
yongtang Nov 5, 2020
37dfa10
Make Sleep portable on Windows C++ (Visual Studio)
yongtang Nov 5, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ jobs:
python3 setup.py --package-version | xargs python3 -m pip install
python3 tools/build/configure.py
cat .bazelrc
bazel build -s --verbose_failures //tensorflow_io/core:python/ops/libtensorflow_io.so
bazel build -s --verbose_failures //tensorflow_io/core:python/ops/libtensorflow_io.so //tensorflow_io/core:python/ops/libtensorflow_io_plugins.so
- uses: actions/upload-artifact@v1
with:
name: ${{ runner.os }}-bazel-bin
Expand Down
8 changes: 6 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,10 @@ libraries (.so). The gcc provided by Developer Toolset and rh-python36 should be
Also, the libstdc++ has to be linked statically to avoid discrepancy of libstdc++ installed on
CentOS vs. newer gcc version by devtoolset.

Furthermore, a special flag `--//tensorflow_io/core:static_build` has to be passed to Bazel
in order to avoid duplication of symbols in statically linked libraries for file system
plugins.

The following will install bazel, devtoolset-9, rh-python36, and build the shared libraries:
```sh
#!/usr/bin/env bash
Expand All @@ -331,10 +335,10 @@ scl enable rh-python36 devtoolset-9 \
scl enable rh-python36 devtoolset-9 \
'./configure.sh'

# Build shared libraries
# Build shared libraries, notice the passing of --//tensorflow_io/core:static_build
BAZEL_LINKOPTS="-static-libstdc++ -static-libgcc" BAZEL_LINKLIBS="-lm -l%:libstdc++.a" \
scl enable rh-python36 devtoolset-9 \
'bazel build -s --verbose_failures //tensorflow_io/...'
'bazel build -s --verbose_failures --//tensorflow_io/core:static_build //tensorflow_io/...'

# Once build is complete, shared libraries will be available in
# `bazel-bin/tensorflow_io/core/python/ops/` and it is possible
Expand Down
52 changes: 34 additions & 18 deletions tensorflow_io/core/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,19 @@ load(
"tf_io_copts",
)
load("@io_bazel_rules_go//go:def.bzl", "go_binary")
load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")

bool_flag(
name = "static_build",
build_setting_default = False,
)

config_setting(
name = "static_build_on",
flag_values = {
":static_build": "True",
},
)

cc_library(
name = "cpuinfo",
Expand Down Expand Up @@ -562,36 +575,23 @@ cc_library(
)

cc_library(
name = "azfs_ops",
name = "file_system_plugins",
srcs = [
"kernels/azfs_kernels.cc",
"kernels/azfs_kernels.h",
"kernels/file_system_plugins.cc",
"kernels/file_system_plugins.h",
],
copts = tf_io_copts(),
linkstatic = True,
deps = [
"@com_github_azure_azure_storage_cpplite//:azure",
"@com_google_absl//absl/strings",
"@local_config_tf//:libtensorflow_framework",
"@local_config_tf//:tf_header_lib",
],
alwayslink = 1,
)

# bazel test \
# --action_env=TF_AZURE_USE_DEV_STORAGE=1 \
# //tensorflow_io/azure:azfs_ops_test
cc_test(
name = "azfs_ops_test",
srcs = [
"kernels/azfs_kernels_test.cc",
],
deps = [
":azfs_ops",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
],
)

cc_library(
name = "sql_ops",
srcs = [
Expand Down Expand Up @@ -674,7 +674,6 @@ cc_binary(
"//tensorflow_io/bigquery:bigquery_ops",
"//tensorflow_io/core:audio_video_ops",
"//tensorflow_io/core:avro_ops",
"//tensorflow_io/core:azfs_ops",
"//tensorflow_io/core:cpuinfo",
"//tensorflow_io/core:file_ops",
"//tensorflow_io/core:grpc_ops",
Expand Down Expand Up @@ -705,6 +704,23 @@ cc_binary(
"//tensorflow_io/core:oss_ops",
"//tensorflow_io/core/kernels/gsmemcachedfs:gs_memcached_file_system",
],
}) + select({
"//tensorflow_io/core:static_build_on": [
"//tensorflow_io/core:file_system_plugins",
],
"//conditions:default": [],
}),
)

cc_binary(
name = "python/ops/libtensorflow_io_plugins.so",
copts = tf_io_copts(),
linkshared = 1,
deps = select({
"//tensorflow_io/core:static_build_on": [],
"//conditions:default": [
"//tensorflow_io/core:file_system_plugins",
],
}),
)

Expand Down
Loading