BLD: Update environment to ubuntu16.04 (#389)

* BLD: Update environment to ubuntu16.04

* Drop support for py34

* Remove install from loop

* Update working manylinux2010 build

* Update test cases for dtype autocasting

* Add issue number

Lint

Add missing files

Create crosstool_wrapper_driver_is_not_gcc.tpl

Create msvc_wrapper_for_nvcc.py.tpl

Update permission

* Add comments to requirements

* Move cuda clang tpl

* Install buildifier for CI sanity

* Fix sanity check

* Correct cuda header path

* Fix bazel dict join

* Fix last gpu build and update install comment

Author: seanpmorgan

.travis.yml

@@ -20,21 +20,21 @@ jobs:
   include:
     # Linux Tests
     - stage: test
-      name: "Test on Ubuntu 14.04"
+      name: "Test on Ubuntu 16.04"
       script:
-      - docker run --rm -v ${PWD}:/addons -w /addons tensorflow/tensorflow:nightly-custom-op make unit-test
+      - docker run --rm -v ${PWD}:/addons -w /addons tensorflow/tensorflow:custom-op-ubuntu16 make unit-test
 
     # Linux Builds
     - stage: build
-      name: "Build on Ubuntu 14.04 for Python 2.7 3.4 3.5 3.6"
+      name: "Build on Ubuntu 16.04 for Python 2.7 3.5 3.6 3.7"
       script:
-      - docker run --rm -v $PWD:/addons -w /addons tensorflow/tensorflow:nightly-custom-op tools/ci_build/builds/release_linux.sh
+      - docker run --rm -v $PWD:/addons -w /addons tensorflow/tensorflow:custom-op-ubuntu16 tools/ci_build/builds/release_linux.sh
       after_success:
         - twine upload -u $PYPI_USER -p $PYPI_PW wheelhouse/*.whl
 
     # MacOS Builds
     - stage: build
-      name: "Build on MacOS for Python 2.7 3.4 3.5 3.6"
+      name: "Build on MacOS for Python 2.7 3.5 3.6 3.7"
       os: osx
       osx_image: xcode10
       script:

CONTRIBUTING.md

@@ -76,7 +76,7 @@ bash tools/run_docker.sh -c 'make unit-test'
 or run manually:
 
 ```bash
-docker run --rm -it -v ${PWD}:/addons -w /addons tensorflow/tensorflow:nightly-custom-op /bin/bash
+docker run --rm -it -v ${PWD}:/addons -w /addons tensorflow/tensorflow:custom-op-ubuntu16 /bin/bash
 
 ./configure.sh  # Links project with TensorFlow dependency
 
@@ -94,7 +94,7 @@ bash tools/run_docker.sh -d gpu -c 'make gpu-unit-test'
 or run manually:
 
 ```bash
-docker run --runtime=nvidia --rm -it -v ${PWD}:/addons -w /addons tensorflow/tensorflow:custom-op-gpu /bin/bash
+docker run --runtime=nvidia --rm -it -v ${PWD}:/addons -w /addons tensorflow/tensorflow:custom-op-gpu-ubuntu16 /bin/bash
 ./configure.sh  # Links project with TensorFlow dependency
 
 bazel test -c opt -k \

build_deps/gpu/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc.tpl

@@ -174,7 +174,7 @@ def _get_win_cuda_defines(repository_ctx):
     # If we are not on Windows, return empty vaules for Windows specific fields.
     # This ensures the CROSSTOOL file parser is happy.
     if not _is_windows(repository_ctx):
-        return {
+        return dict({
             "%{msvc_env_tmp}": "",
             "%{msvc_env_path}": "",
             "%{msvc_env_include}": "",
@@ -184,7 +184,7 @@ def _get_win_cuda_defines(repository_ctx):
             "%{msvc_link_path}": "",
             "%{msvc_lib_path}": "",
             "%{cxx_builtin_include_directory}": "",
-        }
+        })
 
     vc_path = find_vc_path(repository_ctx)
     if not vc_path:
@@ -957,6 +957,8 @@ def _get_cuda_config(repository_ctx):
     )
 
 def _tpl(repository_ctx, tpl, substitutions = {}, out = None):
+    if substitutions == None:
+        substitutions = {}
     if not out:
         out = tpl.replace(":", "/")
     repository_ctx.template(
@@ -1301,7 +1303,7 @@ def _create_local_cuda_repository(repository_ctx):
     _tpl(
         repository_ctx,
         "crosstool:CROSSTOOL",
-        cuda_defines + _get_win_cuda_defines(repository_ctx),
+        cuda_defines.update(_get_win_cuda_defines(repository_ctx)),
         out = "crosstool/CROSSTOOL",
     )
 

build_deps/gpu/cuda_configure.bzl

@@ -1 +1,2 @@
-tf-nightly-2.0-preview==2.0.0.dev20190731
+# TensorFlow greater than this date is manylinux2010 compliant
+tf-nightly-2.0-preview>=2.0.0.dev20190802

build_deps/requirements.txt

@@ -1 +1,2 @@
-tf-nightly-gpu-2.0-preview==2.0.0.dev20190731
+# TensorFlow greater than this date is manylinux2010 compliant
+tf-nightly-gpu-2.0-preview>=2.0.0.dev20190802

build_deps/requirements_gpu.txt

@@ -168,7 +168,7 @@ def _symlink_genrule_for_dir(
 
             # Copy the headers to create a sandboxable setup.
             cmd = "cp -f"
-            command.append(cmd + ' "%s" "%s"' % (src_files[i], dest))
+            command.append(cmd + ' "%s" "%s" | true' % (src_files[i], dest))
             outs.append('        "' + dest_dir + dest_files[i] + '",')
     genrule = _genrule(
         genrule_name,

build_deps/tf_dependency/tf_configure.bzl

@@ -0,0 +1,167 @@
+# This file is expanded from a template by cuda_configure.bzl
+# Update cuda_configure.bzl#verify_build_defines when adding new variables.
+
+load(":cc_toolchain_config.bzl", "cc_toolchain_config")
+
+licenses(["restricted"])
+
+package(default_visibility = ["//visibility:public"])
+
+toolchain(
+    name = "toolchain-linux-x86_64",
+    exec_compatible_with = [
+        "@bazel_tools//platforms:linux",
+        "@bazel_tools//platforms:x86_64",
+    ],
+    target_compatible_with = [
+        "@bazel_tools//platforms:linux",
+        "@bazel_tools//platforms:x86_64",
+    ],
+    toolchain = ":cc-compiler-local",
+    toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
+)
+
+cc_toolchain_suite(
+    name = "toolchain",
+    toolchains = {
+        "local|compiler": ":cc-compiler-local",
+        "darwin|compiler": ":cc-compiler-darwin",
+        "x64_windows|msvc-cl": ":cc-compiler-windows",
+        "x64_windows": ":cc-compiler-windows",
+        "arm": ":cc-compiler-local",
+        "aarch64": ":cc-compiler-local",
+        "k8": ":cc-compiler-local",
+        "piii": ":cc-compiler-local",
+        "ppc": ":cc-compiler-local",
+        "darwin": ":cc-compiler-darwin",
+    },
+)
+
+cc_toolchain(
+    name = "cc-compiler-local",
+    all_files = ":crosstool_wrapper_driver_is_not_gcc",
+    compiler_files = ":empty",
+    dwp_files = ":empty",
+    linker_files = ":crosstool_wrapper_driver_is_not_gcc",
+    objcopy_files = ":empty",
+    strip_files = ":empty",
+    # To support linker flags that need to go to the start of command line
+    # we need the toolchain to support parameter files. Parameter files are
+    # last on the command line and contain all shared libraries to link, so all
+    # regular options will be left of them.
+    supports_param_files = 1,
+    toolchain_config = ":cc-compiler-local-config",
+    toolchain_identifier = "local_linux",
+)
+
+cc_toolchain_config(
+    name = "cc-compiler-local-config",
+    builtin_include_directories = [
+        "/dt7/usr/include/c++/7",
+        "/dt7/usr/include/c++/7/x86_64-pc-linux-gnu",
+        "/dt7/usr/include/c++/7/backward",
+        "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include",
+        "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include-fixed",
+        "/dt7/usr/include",
+        "/usr/local/cuda-10.0/targets/x86_64-linux/include",
+        "/usr/local/cuda-10.0/include",
+        "/usr/local/cuda-10.0/extras/CUPTI/include",
+        "/usr/include",
+    ],
+    cpu = "local",
+    extra_no_canonical_prefixes_flags = ["-fno-canonical-system-headers"],
+    host_compiler_path = "clang/bin/crosstool_wrapper_driver_is_not_gcc",
+    host_compiler_prefix = "/usr/bin",
+    host_compiler_warnings = [],
+    host_unfiltered_compile_flags = [],
+    linker_bin_path = "/usr/bin",
+)
+
+cc_toolchain(
+    name = "cc-compiler-darwin",
+    all_files = ":crosstool_wrapper_driver_is_not_gcc",
+    compiler_files = ":empty",
+    dwp_files = ":empty",
+    linker_files = ":crosstool_wrapper_driver_is_not_gcc",
+    objcopy_files = ":empty",
+    strip_files = ":empty",
+    supports_param_files = 0,
+    toolchain_config = ":cc-compiler-local-darwin",
+    toolchain_identifier = "local_darwin",
+)
+
+cc_toolchain_config(
+    name = "cc-compiler-local-darwin",
+    builtin_include_directories = [
+        "/dt7/usr/include/c++/7",
+        "/dt7/usr/include/c++/7/x86_64-pc-linux-gnu",
+        "/dt7/usr/include/c++/7/backward",
+        "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include",
+        "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include-fixed",
+        "/dt7/usr/include",
+        "/usr/local/cuda-10.0/targets/x86_64-linux/include",
+        "/usr/local/cuda-10.0/include",
+        "/usr/local/cuda-10.0/extras/CUPTI/include",
+        "/usr/include",
+    ],
+    cpu = "darwin",
+    extra_no_canonical_prefixes_flags = ["-fno-canonical-system-headers"],
+    host_compiler_path = "clang/bin/crosstool_wrapper_driver_is_not_gcc",
+    host_compiler_prefix = "/usr/bin",
+    host_compiler_warnings = [],
+    host_unfiltered_compile_flags = [],
+    linker_bin_path = "/usr/bin",
+)
+
+cc_toolchain(
+    name = "cc-compiler-windows",
+    all_files = ":windows_msvc_wrapper_files",
+    compiler_files = ":empty",
+    dwp_files = ":empty",
+    linker_files = ":windows_msvc_wrapper_files",
+    objcopy_files = ":empty",
+    strip_files = ":empty",
+    supports_param_files = 1,
+    toolchain_config = ":cc-compiler-windows-config",
+    toolchain_identifier = "local_windows",
+)
+
+cc_toolchain_config(
+    name = "cc-compiler-windows-config",
+    builtin_include_directories = [
+        "/dt7/usr/include/c++/7",
+        "/dt7/usr/include/c++/7/x86_64-pc-linux-gnu",
+        "/dt7/usr/include/c++/7/backward",
+        "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include",
+        "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include-fixed",
+        "/dt7/usr/include",
+        "/usr/local/cuda-10.0/targets/x86_64-linux/include",
+        "/usr/local/cuda-10.0/include",
+        "/usr/local/cuda-10.0/extras/CUPTI/include",
+        "/usr/include",
+    ],
+    cpu = "x64_windows",
+    msvc_cl_path = "msvc_not_used",
+    msvc_env_include = "msvc_not_used",
+    msvc_env_lib = "msvc_not_used",
+    msvc_env_path = "msvc_not_used",
+    msvc_env_tmp = "msvc_not_used",
+    msvc_lib_path = "msvc_not_used",
+    msvc_link_path = "msvc_not_used",
+    msvc_ml_path = "msvc_not_used",
+)
+
+filegroup(
+    name = "empty",
+    srcs = [],
+)
+
+filegroup(
+    name = "crosstool_wrapper_driver_is_not_gcc",
+    srcs = ["clang/bin/crosstool_wrapper_driver_is_not_gcc"],
+)
+
+filegroup(
+    name = "windows_msvc_wrapper_files",
+    srcs = glob(["windows/msvc_*"]),
+)