HDFS-15385 Upgrade boost library to 1.72

BUILDING.txt

@@ -6,6 +6,7 @@ Requirements:
 * Unix System
 * JDK 1.8
 * Maven 3.3 or later
+* Boost 1.72 (if compiling native code)
 * Protocol Buffers 3.7.1 (if compiling native code)
 * CMake 3.1 or newer (if compiling native code)
 * Zlib devel (if compiling native code)
@@ -72,6 +73,12 @@ Installing required packages for clean install of Ubuntu 14.04 LTS Desktop:
         && ./configure\
         && make install \
         && rm -rf /opt/protobuf-3.7-src
+* Boost
+  $ curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
+    && tar --bzip2 -xf boost_1_72_0.tar.bz2 \
+    && cd boost_1_72_0 \
+    && ./bootstrap.sh --prefix=/usr/ \
+    && ./b2 --without-python install
 
 Optional packages:
 
@@ -468,6 +475,7 @@ Requirements:
 * Windows System
 * JDK 1.8
 * Maven 3.0 or later
+* Boost 1.72
 * Protocol Buffers 3.7.1
 * CMake 3.1 or newer
 * Visual Studio 2010 Professional or Higher

dev-support/docker/Dockerfile

@@ -92,6 +92,21 @@ ENV MAVEN_HOME /usr
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
 ENV FINDBUGS_HOME /usr
 
+#######
+# Install Boost 1.72 (1.65 ships with Bionic)
+#######
+# hadolint ignore=DL3003
+RUN mkdir -p /opt/boost-library \
+    && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
+    && mv boost_1_72_0.tar.bz2 /opt/boost-library \
+    && cd /opt/boost-library \
+    && tar --bzip2 -xf boost_1_72_0.tar.bz2 \
+    && cd /opt/boost-library/boost_1_72_0 \
+    && ./bootstrap.sh --prefix=/usr/ \
+    && ./b2 --without-python install \
+    && cd /root \
+    && rm -rf /opt/boost-library
+
 ######
 # Install Google Protobuf 3.7.1 (3.0.0 ships with Bionic)
 ######

dev-support/docker/Dockerfile_aarch64

@@ -95,6 +95,21 @@ ENV MAVEN_HOME /usr
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64
 ENV FINDBUGS_HOME /usr
 
+#######
+# Install Boost 1.72 (1.65 ships with Bionic)
+#######
+# hadolint ignore=DL3003
+RUN mkdir -p /opt/boost-library \
+    && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
+    && mv boost_1_72_0.tar.bz2 /opt/boost-library \
+    && cd /opt/boost-library \
+    && tar --bzip2 -xf boost_1_72_0.tar.bz2 \
+    && cd /opt/boost-library/boost_1_72_0 \
+    && ./bootstrap.sh --prefix=/usr/ \
+    && ./b2 --without-python install \
+    && cd /root \
+    && rm -rf /opt/boost-library
+
 ######
 # Install Google Protobuf 3.7.1 (3.0.0 ships with Bionic)
 ######

hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt

@@ -18,6 +18,8 @@
 
 cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
 
+project(hadoop_hdfs_native_client)
+
 enable_testing()
 
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt

@@ -28,6 +28,8 @@ project (libhdfspp)
 
 cmake_minimum_required(VERSION 2.8)
 
+find_package (Boost 1.72.0 REQUIRED)
+
 enable_testing()
 include (CTest)
 
@@ -220,7 +222,7 @@ include_directories(
 
 include_directories( SYSTEM
   ${PROJECT_BINARY_DIR}/lib/proto
-  third_party/asio-1.10.2/include
+  ${Boost_INCLUDE_DIRS}
   third_party/rapidxml-1.13
   third_party/gmock-1.7.0
   third_party/tr2

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/ioservice.h

@@ -61,10 +61,7 @@
 #include <functional>
 #include <memory>
 
-// forward decl
-namespace asio {
-  class io_service;
-}
+#include <boost/asio/io_service.hpp>
 
 namespace hdfs {
 
@@ -133,7 +130,7 @@ class IoService : public std::enable_shared_from_this<IoService>
    * Access underlying io_service object.  Only to be used in asio library calls.
    * After HDFS-11884 is complete only tests should need direct access to the asio::io_service.
    **/
-  virtual asio::io_service& GetRaw() = 0;
+  virtual boost::asio::io_service& GetRaw() = 0;
 };
 
 

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt

@@ -19,6 +19,7 @@ if(NEED_LINK_DL)
    set(LIB_DL dl)
 endif()
 
+include_directories(${Boost_INCLUDE_DIRS} ../../include)
 add_library(common_obj OBJECT status.cc sasl_digest_md5.cc ioservice_impl.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc uri.cc util.cc retry_policy.cc cancel_tracker.cc logging.cc libhdfs_events_impl.cc auth_info.cc namenode_info.cc statinfo.cc fsinfo.cc content_summary.cc locks.cc config_parser.cc)
 add_library(common $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
 target_link_libraries(common ${LIB_DL})

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/async_stream.h

@@ -19,15 +19,17 @@
 #ifndef LIB_COMMON_ASYNC_STREAM_H_
 #define LIB_COMMON_ASYNC_STREAM_H_
 
-#include <asio/buffer.hpp>
-#include <asio/error_code.hpp>
+#include <boost/asio/buffer.hpp>
+#include <boost/system/error_code.hpp>
+#include <boost/asio/system_executor.hpp>
+
 #include <functional>
 
 namespace hdfs {
 
 // Contiguous buffer types
-typedef asio::mutable_buffers_1 MutableBuffer;
-typedef asio::const_buffers_1   ConstBuffer;
+typedef boost::asio::mutable_buffers_1 MutableBuffer;
+typedef boost::asio::const_buffers_1   ConstBuffer;
 
 /*
  * asio-compatible stream implementation.
@@ -38,13 +40,20 @@ typedef asio::const_buffers_1   ConstBuffer;
  */
 class AsyncStream  {
 public:
+  using executor_type = boost::asio::system_executor;
+  executor_type executor_;
+
   virtual void async_read_some(const MutableBuffer &buf,
-          std::function<void (const asio::error_code & error,
+          std::function<void (const boost::system::error_code & error,
                                  std::size_t bytes_transferred) > handler) = 0;
 
   virtual void async_write_some(const ConstBuffer &buf,
-            std::function<void (const asio::error_code & error,
+            std::function<void (const boost::system::error_code & error,
                                  std::size_t bytes_transferred) > handler) = 0;
+
+  executor_type get_executor() {
+      return executor_;
+  }
 };
 
 }

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/asio.h

@@ -21,7 +21,10 @@
 #include "continuation.h"
 #include "common/util.h"
 #include "hdfspp/status.h"
-#include <asio/write.hpp>
+
+#include <boost/asio/write.hpp>
+#include <boost/system/error_code.hpp>
+
 #include <memory>
 
 namespace hdfs {
@@ -37,8 +40,8 @@ class WriteContinuation : public Continuation {
 
   virtual void Run(const Next &next) override {
     auto handler =
-        [next](const asio::error_code &ec, size_t) { next(ToStatus(ec)); };
-    asio::async_write(*stream_, buffer_, handler);
+        [next](const boost::system::error_code &ec, size_t) { next(ToStatus(ec)); };
+    boost::asio::async_write(*stream_, buffer_, handler);
   }
 
 private:

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/continuation/protobuf.h

@@ -20,8 +20,10 @@
 
 #include "common/util.h"
 
-#include <asio/read.hpp>
-
+#include <boost/asio/read.hpp>
+#include <boost/asio/write.hpp>
+#include <boost/asio/buffer.hpp>
+#include <boost/system/error_code.hpp>
 #include <google/protobuf/message_lite.h>
 #include <google/protobuf/io/coded_stream.h>
 #include <google/protobuf/io/zero_copy_stream_impl_lite.h>
@@ -39,7 +41,7 @@ struct ReadDelimitedPBMessageContinuation : public Continuation {
 
   virtual void Run(const Next &next) override {
     namespace pbio = google::protobuf::io;
-    auto handler = [this, next](const asio::error_code &ec, size_t) {
+    auto handler = [this, next](const boost::system::error_code &ec, size_t) {
       Status status;
       if (ec) {
         status = ToStatus(ec);
@@ -57,15 +59,15 @@ struct ReadDelimitedPBMessageContinuation : public Continuation {
       }
       next(status);
     };
-    asio::async_read(*stream_,
-        asio::buffer(buf_),
+    boost::asio::async_read(*stream_,
+        boost::asio::buffer(buf_),
         std::bind(&ReadDelimitedPBMessageContinuation::CompletionHandler, this,
                   std::placeholders::_1, std::placeholders::_2),
         handler);
   }
 
 private:
-  size_t CompletionHandler(const asio::error_code &ec, size_t transferred) {
+  size_t CompletionHandler(const boost::system::error_code &ec, size_t transferred) {
     if (ec) {
       return 0;
     }
@@ -103,7 +105,7 @@ struct WriteDelimitedPBMessageContinuation : Continuation {
       return;
     }
 
-    asio::async_write(*stream_, asio::buffer(buf_), [next](const asio::error_code &ec, size_t) { next(ToStatus(ec)); } );
+    boost::asio::async_write(*stream_, boost::asio::buffer(buf_), [next](const boost::system::error_code &ec, size_t) { next(ToStatus(ec)); } );
   }
 
 private:

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/fsinfo.cc

@@ -17,6 +17,8 @@
  */
 
 #include <hdfspp/fsinfo.h>
+
+#include <algorithm>
 #include <sstream>
 #include <iomanip>
 

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/ioservice_impl.cc

@@ -125,7 +125,7 @@ void IoServiceImpl::Run() {
   // from escaping this library and crashing the process.
 
   // As recommended in http://www.boost.org/doc/libs/1_39_0/doc/html/boost_asio/reference/io_service.html#boost_asio.reference.io_service.effect_of_exceptions_thrown_from_handlers
-  asio::io_service::work work(io_service_);
+  boost::asio::io_service::work work(io_service_);
   while(true)
   {
     try
@@ -145,7 +145,7 @@ void IoServiceImpl::Stop() {
   io_service_.stop();
 }
 
-asio::io_service& IoServiceImpl::GetRaw() {
+boost::asio::io_service& IoServiceImpl::GetRaw() {
   return io_service_;
 }
 

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/ioservice_impl.h

@@ -21,7 +21,7 @@
 
 #include "hdfspp/ioservice.h"
 
-#include <asio/io_service.hpp>
+#include <boost/asio/io_service.hpp>
 #include "common/new_delete.h"
 
 #include <mutex>
@@ -45,7 +45,7 @@ class IoServiceImpl : public IoService {
   void PostTask(std::function<void(void)> asyncTask) override;
   void Run() override;
   void Stop() override;
-  asio::io_service& GetRaw() override;
+  boost::asio::io_service& GetRaw() override;
 
   // Add a single worker thread, in the common case try to avoid this in favor
   // of Init[Default]Workers. Public for use by tests and rare cases where a
@@ -57,7 +57,7 @@ class IoServiceImpl : public IoService {
 
  private:
   std::mutex state_lock_;
-  ::asio::io_service io_service_;
+  boost::asio::io_service io_service_;
 
   // For doing logging + resource manager updates on thread start/exit
   void ThreadStartHook();

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.cc

@@ -136,7 +136,7 @@ LogMessage& LogMessage::operator<<(const std::string& str) {
   return *this;
 }
 
-LogMessage& LogMessage::operator<<(const ::asio::ip::tcp::endpoint& endpoint) {
+LogMessage& LogMessage::operator<<(const boost::asio::ip::tcp::endpoint& endpoint) {
   msg_buffer_ << endpoint;
   return *this;
 }

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h

@@ -19,7 +19,7 @@
 #ifndef LIB_COMMON_LOGGING_H_
 #define LIB_COMMON_LOGGING_H_
 
-#include <asio/ip/tcp.hpp>
+#include <boost/asio/ip/tcp.hpp>
 
 #include "hdfspp/log.h"
 
@@ -193,7 +193,7 @@ class LogMessage {
   LogMessage& operator<<(void *);
 
   //asio types
-  LogMessage& operator<<(const ::asio::ip::tcp::endpoint& endpoint);
+  LogMessage& operator<<(const boost::asio::ip::tcp::endpoint& endpoint);
 
   //thread and mutex types
   LogMessage& operator<<(const std::thread::id& tid);

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc

@@ -70,16 +70,16 @@ bool ResolveInPlace(std::shared_ptr<IoService> ioservice, ResolvedNamenodeInfo &
   return true;
 }
 
-typedef std::vector<asio::ip::tcp::endpoint> endpoint_vector;
+typedef std::vector<boost::asio::ip::tcp::endpoint> endpoint_vector;
 
 // RAII wrapper
 class ScopedResolver {
  private:
   std::shared_ptr<IoService> io_service_;
   std::string host_;
   std::string port_;
-  ::asio::ip::tcp::resolver::query query_;
-  ::asio::ip::tcp::resolver resolver_;
+  boost::asio::ip::tcp::resolver::query query_;
+  boost::asio::ip::tcp::resolver resolver_;
   endpoint_vector endpoints_;
 
   // Caller blocks on access if resolution isn't finished
@@ -111,9 +111,9 @@ class ScopedResolver {
     std::shared_ptr<std::promise<Status>> shared_result = result_status_;
 
     // Callback to pull a copy of endpoints out of resolver and set promise
-    auto callback = [this, shared_result](const asio::error_code &ec, ::asio::ip::tcp::resolver::iterator out) {
+    auto callback = [this, shared_result](const boost::system::error_code &ec, boost::asio::ip::tcp::resolver::iterator out) {
       if(!ec) {
-        std::copy(out, ::asio::ip::tcp::resolver::iterator(), std::back_inserter(endpoints_));
+        std::copy(out, boost::asio::ip::tcp::resolver::iterator(), std::back_inserter(endpoints_));
       }
       shared_result->set_value( ToStatus(ec) );
     };

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.h

@@ -19,7 +19,7 @@
 #ifndef COMMON_HDFS_NAMENODE_INFO_H_
 #define COMMON_HDFS_NAMENODE_INFO_H_
 
-#include <asio.hpp>
+#include <boost/asio/ip/tcp.hpp>
 
 #include <hdfspp/options.h>
 
@@ -37,7 +37,7 @@ struct ResolvedNamenodeInfo : public NamenodeInfo {
   ResolvedNamenodeInfo& operator=(const NamenodeInfo &info);
   std::string str() const;
 
-  std::vector<::asio::ip::tcp::endpoint> endpoints;
+  std::vector<boost::asio::ip::tcp::endpoint> endpoints;
 };
 
 // Clear endpoints if set and resolve all of them in parallel.

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/retry_policy.cc

@@ -19,6 +19,8 @@
 #include "common/retry_policy.h"
 #include "common/logging.h"
 
+#include <boost/asio/error.hpp>
+
 #include <sstream>
 
 namespace hdfs {
@@ -57,7 +59,7 @@ RetryAction FixedDelayWithFailover::ShouldRetry(const Status &s, uint64_t retrie
   (void)max_failover_conn_retries_;
   LOG_TRACE(kRPC, << "FixedDelayWithFailover::ShouldRetry(retries=" << retries << ", failovers=" << failovers << ")");
 
-  if(failovers < max_failover_retries_ && (s.code() == ::asio::error::timed_out || s.get_server_exception_type() == Status::kStandbyException) )
+  if(failovers < max_failover_retries_ && (s.code() == boost::asio::error::timed_out || s.get_server_exception_type() == Status::kStandbyException) )
   {
     // Try connecting to another NN in case this one keeps timing out
     // Can add the backoff wait specified by dfs.client.failover.sleep.base.millis here