diff --git a/WORKSPACE b/WORKSPACE index 7dfc3a606..4cde66689 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -300,3 +300,65 @@ http_archive( "http://github.com/libexpat/libexpat/archive/R_2_2_6.tar.gz", ], ) + +http_archive( + name = "libapr1", + build_file = "//third_party:libapr1.BUILD", + patch_args = ["-p1"], + patches = [ + "//third_party:libapr1.patch", + ], + sha256 = "1a0909a1146a214a6ab9de28902045461901baab4e0ee43797539ec05b6dbae0", + strip_prefix = "apr-1.6.5", + urls = [ + "https://github.com/apache/apr/archive/1.6.5.tar.gz", + ], +) + +http_archive( + name = "libaprutil1", + build_file = "//third_party:libaprutil1.BUILD", + patch_args = ["-p1"], + patches = [ + "//third_party:libaprutil1.patch", + ], + sha256 = "4c9ae319cedc16890fc2776920e7d529672dda9c3a9a9abd53bd80c2071b39af", + strip_prefix = "apr-util-1.6.1", + urls = [ + "https://github.com/apache/apr-util/archive/1.6.1.tar.gz", + ], +) + +http_archive( + name = "mxml", + build_file = "//third_party:mxml.BUILD", + patch_args = ["-p1"], + patches = [ + "//third_party:mxml.patch", + ], + sha256 = "4d850d15cdd4fdb9e82817eb069050d7575059a9a2729c82b23440e4445da199", + strip_prefix = "mxml-2.12", + urls = [ + "https://github.com/michaelrsweet/mxml/archive/v2.12.tar.gz", + ], +) + +http_archive( + name = "minini", + build_file = "//third_party:minini.BUILD", + sha256 = "a97dd5ac6811af95c8f2aeaa6894b3113377e78ffd585363c6848745760d0152", + strip_prefix = "minIni-1.0", + urls = [ + "https://github.com/ElasticDL/minIni/archive/v1.0.tar.gz", + ], +) + +http_archive( + name = "aliyun_oss_c_sdk", + build_file = "//third_party:oss_c_sdk.BUILD", + sha256 = "6450d3970578c794b23e9e1645440c6f42f63be3f82383097660db5cf2fba685", + strip_prefix = "aliyun-oss-c-sdk-3.7.0", + urls = [ + "https://github.com/aliyun/aliyun-oss-c-sdk/archive/3.7.0.tar.gz", + ], +) diff --git a/tensorflow_io/oss/BUILD b/tensorflow_io/oss/BUILD new file mode 100644 index 000000000..80ceafc32 --- /dev/null +++ b/tensorflow_io/oss/BUILD @@ -0,0 +1,24 @@ +package( + default_visibility = ["//visibility:public"], +) + +licenses(["notice"]) # Apache 2.0 + +cc_binary( + name = "python/ops/_oss_ops.so", + srcs = [ + "kernels/ossfs/oss_file_system.cc", + "kernels/ossfs/oss_file_system.h", + "ops/ossfs_ops.cc", + ], + copts = [ + "-D_GLIBCXX_USE_CXX11_ABI=0", + ], + linkshared = 1, + deps = [ + "@aliyun_oss_c_sdk", + "@local_config_tf//:libtensorflow_framework", + "@local_config_tf//:tf_header_lib", + "@minini", + ], +) diff --git a/tensorflow_io/oss/README.md b/tensorflow_io/oss/README.md new file mode 100644 index 000000000..0e9065bf1 --- /dev/null +++ b/tensorflow_io/oss/README.md @@ -0,0 +1,35 @@ +# TensorFlow OSS Filesystem Extension + +OSS is an object storage service provided by Alibaba Cloud, see [here](https://www.alibabacloud.com/product/oss) for more information about the service. + +This module provides an extension that emulates a filesystem using the object storage service. The directory structures are encoded in object keys and file contents are stored in objects. The extension is implemented using [OSS C SDK](https://github.com/aliyun/aliyun-oss-c-sdk). + +To use the extension, first save your OSS credential in a file, in `INI` format: + +``` +[OSSCredentials] +host = cn-hangzhou.oss.aliyun-inc.com +accessid = your_oss_access_id +accesskey = you_oss_access_key +``` + +Then set environment variable `OSS_CREDENTIALS` to the path of the file. + +In Python code, import the extension `ossfs_op` module to use the extension with `gfile`. The files and directory URI should have `oss://` prefix, followed by a bucket name, then the directory hierarchy. + +```python +import tensorflow_io.oss.python.ops.ossfs_ops +from tensorflow.python.platform import gfile + +gfile.MkDir('oss://your_bucket_name/test_dir') +``` + +With the extension installed, OSS files can be use with Dataset Ops, etc., in the same fashion as other files. + +```python +dataset = tf.data.TextLineDataset(["oss://bucket_name/data_dir/file1"]) +``` + +## Test + +File `tests/test_oss.py` contains basic filesystem functionality tests. See `README.md` in root directory for more information about running tests. Besides `OSS_CREDENTIALS`, the tests also require an `OSS_FS_TEST_BUCKET` environment variable containing an accessible bucket name. Make sure they are set before running `pytest tests`. You can also just run the OSS test using `pytest tests/test_oss.py` diff --git a/tensorflow_io/oss/__init__.py b/tensorflow_io/oss/__init__.py new file mode 100644 index 000000000..9f4f68603 --- /dev/null +++ b/tensorflow_io/oss/__init__.py @@ -0,0 +1,32 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Alibaba OSS File System. + +@@ossfs_ops +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow_io.oss.python.ops import ossfs_ops # pylint: disable=unused-import + +from tensorflow.python.util.all_util import remove_undocumented + +_allowed_symbols = [ + "ossfs_ops", +] + +remove_undocumented(__name__, allowed_exception_list=_allowed_symbols) diff --git a/tensorflow_io/oss/kernels/ossfs/oss_file_system.cc b/tensorflow_io/oss/kernels/ossfs/oss_file_system.cc new file mode 100644 index 000000000..146b1cc03 --- /dev/null +++ b/tensorflow_io/oss/kernels/ossfs/oss_file_system.cc @@ -0,0 +1,1159 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include + +#include "aos_string.h" +#include "minIni.h" +#include "oss_define.h" +#include "oss_file_system.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/thread_annotations.h" + +namespace tensorflow { + +namespace { + +constexpr char kOSSCredentialsDefaultFile[] = ".osscredentials"; +constexpr char kOSSCredentialsFileEnvKey[] = "OSS_CREDENTIALS"; +constexpr char kOSSCredentialsSection[] = "OSSCredentials"; +constexpr char kOSSCredentialsHostKey[] = "host"; +constexpr char kOSSCredentialsAccessIdKey[] = "accessid"; +constexpr char kOSSCredentialsAccesskeyKey[] = "accesskey"; +constexpr char kOSSAccessIdKey[] = "id"; +constexpr char kOSSAccessKeyKey[] = "key"; +constexpr char kOSSHostKey[] = "host"; +constexpr char kDelim[] = "/"; + +bool checkFile(const string& filename) { + std::ifstream fstream(filename.c_str()); + return fstream.good(); +} + +Status GetCredentialsFileFromEnv(string* filename) { + if (!filename) { + return errors::FailedPrecondition("'filename' cannot be nullptr."); + } + const char* result = std::getenv(kOSSCredentialsFileEnvKey); + if (!result || !checkFile(result)) { + return errors::NotFound(strings::StrCat("$", kOSSCredentialsFileEnvKey, + " is not set or corrupt.")); + } + *filename = result; + return Status::OK(); +} + +void oss_initialize_with_throwable() { + if (aos_http_io_initialize(NULL, 0) != AOSE_OK) { + throw std::exception(); + } +} + +Status oss_initialize() { + static std::once_flag initFlag; + try { + std::call_once(initFlag, [] { oss_initialize_with_throwable(); }); + } catch (...) { + LOG(FATAL) << "can not init OSS connection"; + return errors::Internal("can not init OSS connection"); + } + + return Status::OK(); +} + +void oss_error_message(aos_status_s* status, std::string* msg) { + *msg = status->req_id; + if (aos_status_is_ok(status)) { + return; + } + + msg->append(" "); + msg->append(std::to_string(status->code)); + + if (status->code == 404) { + msg->append(" object not exists!"); + return; + } + + if (status->error_msg) { + msg->append(" "); + msg->append(status->error_msg); + return; + } +} + +class OSSConnection { + public: + OSSConnection(const std::string& endPoint, const std::string& accessKey, + const std::string& accessKeySecret) { + aos_pool_create(&_pool, NULL); + _options = oss_request_options_create(_pool); + _options->config = oss_config_create(_options->pool); + aos_str_set(&_options->config->endpoint, endPoint.c_str()); + aos_str_set(&_options->config->access_key_id, accessKey.c_str()); + aos_str_set(&_options->config->access_key_secret, accessKeySecret.c_str()); + _options->config->is_cname = 0; + _options->ctl = aos_http_controller_create(_options->pool, 0); + } + + ~OSSConnection() { + if (NULL != _pool) { + aos_pool_destroy(_pool); + } + } + + oss_request_options_t* getRequestOptions() { return _options; } + + aos_pool_t* getPool() { return _pool; } + + private: + aos_pool_t* _pool = NULL; + oss_request_options_t* _options = NULL; +}; + +class OSSRandomAccessFile : public RandomAccessFile { + public: + OSSRandomAccessFile(const std::string& endPoint, const std::string& accessKey, + const std::string& accessKeySecret, + const std::string& bucket, const std::string& object, + size_t read_ahead_bytes, size_t file_length) + : shost(endPoint), + sak(accessKey), + ssk(accessKeySecret), + sbucket(bucket), + sobject(object), + total_file_length_(file_length) { + read_ahead_bytes_ = std::min(read_ahead_bytes, file_length); + } + + Status Read(uint64 offset, size_t n, StringPiece* result, + char* scratch) const override { + if (offset > total_file_length_) { + return errors::OutOfRange("EOF reached, ", offset, + " is read out of file length ", + total_file_length_); + } + + if (offset + n > total_file_length_) { + n = total_file_length_ - offset; + } + + VLOG(1) << "read " << sobject << " from " << offset << " to " << offset + n; + + mutex_lock lock(mu_); + const bool range_start_included = offset >= buffer_start_offset_; + const bool range_end_included = + offset + n <= buffer_start_offset_ + buffer_size_; + if (range_start_included && range_end_included) { + // The requested range can be filled from the buffer. + const size_t offset_in_buffer = + std::min(offset - buffer_start_offset_, buffer_size_); + const auto copy_size = std::min(n, buffer_size_ - offset_in_buffer); + VLOG(1) << "read from buffer " << offset_in_buffer << " to " + << offset_in_buffer + copy_size << " total " << buffer_size_; + std::copy(buffer_.begin() + offset_in_buffer, + buffer_.begin() + offset_in_buffer + copy_size, scratch); + *result = StringPiece(scratch, copy_size); + } else { + // Update the buffer content based on the new requested range. + const size_t desired_buffer_size = + std::min(n + read_ahead_bytes_, total_file_length_); + if (n > buffer_.capacity() || + desired_buffer_size > 2 * buffer_.capacity()) { + // Re-allocate only if buffer capacity increased significantly. + VLOG(1) << "reserve buffer to " << desired_buffer_size; + buffer_.reserve(desired_buffer_size); + } + + buffer_start_offset_ = offset; + VLOG(1) << "load buffer" << buffer_start_offset_; + TF_RETURN_IF_ERROR(LoadBufferFromOSS(desired_buffer_size)); + + // Set the results. + memcpy(scratch, buffer_.data(), std::min(buffer_size_, n)); + *result = StringPiece(scratch, std::min(buffer_size_, n)); + } + + if (result->size() < n) { + // This is not an error per se. The RandomAccessFile interface expects + // that Read returns OutOfRange if fewer bytes were read than requested. + return errors::OutOfRange("EOF reached, ", result->size(), + " bytes were read out of ", n, + " bytes requested."); + } + return Status::OK(); + } + + private: + /// A helper function to actually read the data from OSS. This function loads + /// buffer_ from OSS based on its current capacity. + Status LoadBufferFromOSS(size_t desired_buffer_size) const + EXCLUSIVE_LOCKS_REQUIRED(mu_) { + size_t range_start = buffer_start_offset_; + size_t range_end = buffer_start_offset_ + std::min(buffer_.capacity() - 1, + desired_buffer_size - 1); + range_end = std::min(range_end, total_file_length_ - 1); + + OSSConnection conn(shost, sak, ssk); + aos_pool_t* _pool = conn.getPool(); + oss_request_options_t* _options = conn.getRequestOptions(); + aos_string_t bucket_; + aos_string_t object_; + aos_table_t* headers_; + aos_list_t tmp_buffer; + aos_table_t* resp_headers; + + aos_list_init(&tmp_buffer); + aos_str_set(&_options->config->endpoint, shost.c_str()); + aos_str_set(&_options->config->access_key_id, sak.c_str()); + aos_str_set(&_options->config->access_key_secret, ssk.c_str()); + _options->config->is_cname = 0; + _options->ctl = aos_http_controller_create(_options->pool, 0); + aos_str_set(&bucket_, sbucket.c_str()); + aos_str_set(&object_, sobject.c_str()); + headers_ = aos_table_make(_pool, 1); + + std::string range("bytes="); + range.append(std::to_string(range_start)) + .append("-") + .append(std::to_string(range_end)); + apr_table_set(headers_, "Range", range.c_str()); + VLOG(1) << "read from OSS with " << range.c_str(); + + aos_status_t* s = + oss_get_object_to_buffer(_options, &bucket_, &object_, headers_, NULL, + &tmp_buffer, &resp_headers); + if (aos_status_is_ok(s)) { + aos_buf_t* content = NULL; + int64_t size = 0; + int64_t pos = 0; + buffer_.clear(); + buffer_size_ = 0; + + // copy data to local buffer + aos_list_for_each_entry(aos_buf_t, content, &tmp_buffer, node) { + size = aos_buf_size(content); + std::copy(content->pos, content->pos + size, buffer_.begin() + pos); + pos += size; + } + buffer_size_ = pos; + return Status::OK(); + } else { + string msg; + oss_error_message(s, &msg); + VLOG(1) << "read " << sobject << " failed, errMsg: " << msg; + return errors::Internal("read failed: ", sobject, " errMsg: ", msg); + } + } + + std::string shost; + std::string sak; + std::string ssk; + std::string sbucket; + std::string sobject; + const size_t total_file_length_; + size_t read_ahead_bytes_; + + mutable mutex mu_; + mutable std::vector buffer_ GUARDED_BY(mu_); + // The original file offset of the first byte in the buffer. + mutable size_t buffer_start_offset_ GUARDED_BY(mu_) = 0; + mutable size_t buffer_size_ GUARDED_BY(mu_) = 0; +}; + +class OSSReadOnlyMemoryRegion : public ReadOnlyMemoryRegion { + public: + OSSReadOnlyMemoryRegion(std::unique_ptr data, uint64 length) + : data_(std::move(data)), length_(length) {} + const void* data() override { return reinterpret_cast(data_.get()); } + uint64 length() override { return length_; } + + private: + std::unique_ptr data_; + uint64 length_; +}; + +class OSSWritableFile : public WritableFile { + public: + OSSWritableFile(const std::string& endPoint, const std::string& accessKey, + const std::string& accessKeySecret, const std::string& bucket, + const std::string& object, size_t part_size) + : shost(endPoint), + sak(accessKey), + ssk(accessKeySecret), + sbucket(bucket), + sobject(object), + part_size_(part_size), + is_closed_(false), + part_number_(1) { + InitAprPool(); + } + + ~OSSWritableFile() { ReleaseAprPool(); } + + Status Append(StringPiece data) override { + mutex_lock lock(mu_); + TF_RETURN_IF_ERROR(_CheckClosed()); + InitAprPool(); + if (CurrentBufferLength() >= part_size_) { + TF_RETURN_IF_ERROR(_FlushInternal()); + } + + aos_buf_t* tmp_buf = aos_create_buf(pool_, data.size() + 1); + aos_buf_append_string(pool_, tmp_buf, data.data(), data.size()); + aos_list_add_tail(&tmp_buf->node, &buffer_); + return Status::OK(); + } + + Status Close() override { + mutex_lock lock(mu_); + TF_RETURN_IF_ERROR(_CheckClosed()); + InitAprPool(); + TF_RETURN_IF_ERROR(_FlushInternal()); + aos_table_t* complete_headers = NULL; + aos_table_t* resp_headers = NULL; + aos_status_t* status = NULL; + oss_list_upload_part_params_t* params = NULL; + aos_list_t complete_part_list; + oss_list_part_content_t* part_content = NULL; + oss_complete_part_content_t* complete_part_content = NULL; + aos_string_t upload_id; + aos_str_set(&upload_id, upload_id_.c_str()); + + params = oss_create_list_upload_part_params(pool_); + aos_list_init(&complete_part_list); + status = oss_list_upload_part(options_, &bucket_, &object_, &upload_id, + params, &resp_headers); + + if (!aos_status_is_ok(status)) { + string msg; + oss_error_message(status, &msg); + VLOG(1) << "List multipart " << sobject << " failed, errMsg: " << msg; + return errors::Internal("List multipart failed: ", sobject, + " errMsg: ", msg); + } + + aos_list_for_each_entry(oss_list_part_content_t, part_content, + ¶ms->part_list, node) { + complete_part_content = oss_create_complete_part_content(pool_); + aos_str_set(&complete_part_content->part_number, + part_content->part_number.data); + aos_str_set(&complete_part_content->etag, part_content->etag.data); + aos_list_add_tail(&complete_part_content->node, &complete_part_list); + } + + status = oss_complete_multipart_upload(options_, &bucket_, &object_, + &upload_id, &complete_part_list, + complete_headers, &resp_headers); + + if (!aos_status_is_ok(status)) { + string msg; + oss_error_message(status, &msg); + VLOG(1) << "Complete multipart " << sobject << " failed, errMsg: " << msg; + return errors::Internal("Complete multipart failed: ", sobject, + " errMsg: ", msg); + } + + is_closed_ = true; + return Status::OK(); + } + + Status Flush() override { + mutex_lock lock(mu_); + TF_RETURN_IF_ERROR(_CheckClosed()); + if (CurrentBufferLength() >= part_size_) { + InitAprPool(); + TF_RETURN_IF_ERROR(_FlushInternal()); + } + + return Status::OK(); + } + + Status Sync() override { return Flush(); } + + private: + void InitAprPool() { + if (NULL == pool_) { + aos_pool_create(&pool_, NULL); + options_ = oss_request_options_create(pool_); + options_->config = oss_config_create(options_->pool); + aos_str_set(&options_->config->endpoint, shost.c_str()); + aos_str_set(&options_->config->access_key_id, sak.c_str()); + aos_str_set(&options_->config->access_key_secret, ssk.c_str()); + options_->config->is_cname = 0; + options_->ctl = aos_http_controller_create(options_->pool, 0); + + aos_str_set(&bucket_, sbucket.c_str()); + aos_str_set(&object_, sobject.c_str()); + + headers_ = aos_table_make(pool_, 1); + aos_list_init(&buffer_); + } + } + + void ReleaseAprPool() { + if (NULL != pool_) { + aos_pool_destroy(pool_); + pool_ = NULL; + } + } + + Status _InitMultiUpload() { + if (upload_id_.empty()) { + aos_string_t uploadId; + aos_status_t* status = NULL; + aos_table_t* resp_headers = NULL; + + InitAprPool(); + status = oss_init_multipart_upload(options_, &bucket_, &object_, + &uploadId, headers_, &resp_headers); + + if (!aos_status_is_ok(status)) { + string msg; + oss_error_message(status, &msg); + VLOG(1) << "Init multipart upload " << sobject + << " failed, errMsg: " << msg; + return errors::Unavailable("Init multipart upload failed: ", sobject, + " errMsg: ", msg); + } + + upload_id_ = uploadId.data; + } + + return Status::OK(); + } + + Status _FlushInternal() { + aos_table_t* resp_headers = NULL; + aos_status_s* status = NULL; + aos_string_t uploadId; + if (CurrentBufferLength() > 0) { + _InitMultiUpload(); + + aos_str_set(&uploadId, upload_id_.c_str()); + status = + oss_upload_part_from_buffer(options_, &bucket_, &object_, &uploadId, + part_number_, &buffer_, &resp_headers); + + if (!aos_status_is_ok(status)) { + string msg; + oss_error_message(status, &msg); + VLOG(1) << "Upload multipart " << sobject << " failed, errMsg: " << msg; + return errors::Internal("Upload multipart failed: ", sobject, + " errMsg: ", msg); + } + + VLOG(0) << " upload " << sobject << " with part" << part_number_ + << " succ"; + part_number_++; + ReleaseAprPool(); + InitAprPool(); + } + return Status::OK(); + } + + const size_t CurrentBufferLength() { return aos_buf_list_len(&buffer_); } + + Status _CheckClosed() { + if (is_closed_) { + return errors::Internal("Already closed."); + } + + return Status::OK(); + } + + std::string shost; + std::string sak; + std::string ssk; + std::string sbucket; + std::string sobject; + size_t part_size_; + + aos_pool_t* pool_ = NULL; + oss_request_options_t* options_ = NULL; + aos_string_t bucket_; + aos_string_t object_; + aos_table_t* headers_ = NULL; + aos_list_t buffer_; + std::string upload_id_; + + bool is_closed_; + mutex mu_; + int64_t part_number_; +}; +} // namespace + +OSSFileSystem::OSSFileSystem() : init_(false) {} + +Status OSSFileSystem::_InitOSSCredentials() { + string filename; + if (!GetCredentialsFileFromEnv(&filename).ok()) { + filename = getpwuid(getuid())->pw_dir; + filename.append(kDelim).append(kOSSCredentialsDefaultFile); + if (!checkFile(filename)) { + return errors::NotFound("can not find any OSS credentials file"); + } + } + + VLOG(0) << "read OSS credentials from " << filename; + + minIni config(filename); + mutex_lock lock(mu_); + host_.assign(config.gets(kOSSCredentialsSection, kOSSCredentialsHostKey)); + access_id_.assign( + config.gets(kOSSCredentialsSection, kOSSCredentialsAccessIdKey)); + access_key_.assign( + config.gets(kOSSCredentialsSection, kOSSCredentialsAccesskeyKey)); + + if (host_.empty() || access_id_.empty() || access_key_.empty()) { + VLOG(1) << filename << " does not contains full OSS credentials"; + return errors::Internal(filename, "does not contains full OSS credentials"); + } + + init_ = true; + return Status::OK(); +} +// Splits an OSS path to endpoint bucket object and token +// For example +// "oss://bucket-name?id=accessid&key=accesskey&host=endpoint/path/to/file.txt" +Status OSSFileSystem::_ParseOSSURIPath(StringPiece fname, std::string* bucket, + std::string* object) { + if (!bucket || !object) { + return errors::Internal("host bucket token and object cannot be null"); + } + + StringPiece scheme, bucketp, remaining; + io::ParseURI(fname, &scheme, &bucketp, &remaining); + if (scheme != "oss") { + return errors::InvalidArgument("OSS path does not start with 'oss://':", + fname); + } + + str_util::ConsumePrefix(&remaining, kDelim); + *object = string(remaining); + + if (bucketp.find('?') != StringPiece::npos) { + // contains id, key, host information + size_t pos = bucketp.find('?'); + *bucket = string(bucketp.substr(0, pos)); + StringPiece access_info = bucketp.substr(pos + 1); + std::vector access_infos = str_util::Split(access_info, "&"); + for (auto& key_value : access_infos) { + StringPiece data(key_value); + size_t pos = data.find('='); + if (pos == StringPiece::npos) { + return errors::InvalidArgument("OSS path access info faied: ", fname, + " info:", key_value); + } + StringPiece key = data.substr(0, pos); + StringPiece value = data.substr(pos + 1); + if (str_util::StartsWith(key, kOSSAccessIdKey)) { + mutex_lock lock(mu_); + access_id_.assign(value.begin(), value.end()); + } else if (str_util::StartsWith(key, kOSSAccessKeyKey)) { + mutex_lock lock(mu_); + access_key_.assign(value.begin(), value.end()); + } else if (str_util::StartsWith(key, kOSSHostKey)) { + mutex_lock lock(mu_); + host_.assign(value.begin(), value.end()); + } else { + return errors::InvalidArgument("OSS path access info faied: ", fname, + " unkown info:", key_value); + } + } + + if (bucket->empty() || *bucket == ".") { + return errors::InvalidArgument("OSS path does not contain a bucket name:", + fname); + } + + if (access_id_.empty() || access_key_.empty() || host_.empty()) { + return errors::InvalidArgument( + "OSS path does not contain valid access info:", fname); + } + if (init_) { + init_ = false; + } + } else { + str_util::ConsumePrefix(&remaining, kDelim); + *object = string(remaining); + *bucket = string(bucketp); + if (bucket->empty() || *bucket == ".") { + return errors::InvalidArgument("OSS path does not contain a bucket name:", + fname); + } + TF_RETURN_IF_ERROR(_InitOSSCredentials()); + } + + return Status::OK(); +} + +Status OSSFileSystem::NewRandomAccessFile( + const std::string& filename, std::unique_ptr* result) { + TF_RETURN_IF_ERROR(oss_initialize()); + std::string object, bucket; + TF_RETURN_IF_ERROR(_ParseOSSURIPath(filename, &bucket, &object)); + FileStatistics stat; + OSSConnection conn(host_, access_id_, access_key_); + TF_RETURN_IF_ERROR(_RetrieveObjectMetadata( + conn.getPool(), conn.getRequestOptions(), bucket, object, &stat)); + result->reset(new OSSRandomAccessFile(host_, access_id_, access_key_, bucket, + object, read_ahead_bytes_, + stat.length)); + return Status::OK(); +} + +Status OSSFileSystem::NewWritableFile(const std::string& fname, + std::unique_ptr* result) { + TF_RETURN_IF_ERROR(oss_initialize()); + std::string object, bucket; + TF_RETURN_IF_ERROR(_ParseOSSURIPath(fname, &bucket, &object)); + + result->reset(new OSSWritableFile(host_, access_id_, access_key_, bucket, + object, upload_part_bytes_)); + return Status::OK(); +} + +Status OSSFileSystem::NewAppendableFile(const std::string& fname, + std::unique_ptr* result) { + return errors::Unimplemented( + "Does not support appendable file in OSSFileSystem"); +} + +Status OSSFileSystem::NewReadOnlyMemoryRegionFromFile( + const std::string& filename, + std::unique_ptr* result) { + uint64 size; + TF_RETURN_IF_ERROR(GetFileSize(filename, &size)); + std::unique_ptr data(new char[size]); + + std::unique_ptr file; + TF_RETURN_IF_ERROR(NewRandomAccessFile(filename, &file)); + + StringPiece piece; + TF_RETURN_IF_ERROR(file->Read(0, size, &piece, data.get())); + + result->reset(new OSSReadOnlyMemoryRegion(std::move(data), size)); + return Status::OK(); +} + +Status OSSFileSystem::FileExists(const std::string& fname) { + FileStatistics stat; + if (Stat(fname, &stat).ok()) { + return Status::OK(); + } else { + return errors::NotFound(fname, " does not exists"); + } +} + +// For GetChildren , we should not return prefix +Status OSSFileSystem::_ListObjects( + aos_pool_t* pool, oss_request_options_t* options, const std::string& bucket, + const std::string& key, std::vector* result, bool return_all, + bool return_full_path, bool should_remove_suffix, + int max_ret_per_iterator) { + aos_string_t bucket_; + aos_status_t* s = NULL; + oss_list_object_params_t* params = NULL; + oss_list_object_content_t* content = NULL; + const char* next_marker = ""; + + aos_str_set(&bucket_, bucket.c_str()); + params = oss_create_list_object_params(pool); + params->max_ret = max_ret_per_iterator; + aos_str_set(¶ms->prefix, key.c_str()); + aos_str_set(¶ms->marker, next_marker); + + do { + s = oss_list_object(options, &bucket_, params, NULL); + if (!aos_status_is_ok(s)) { + string msg; + oss_error_message(s, &msg); + VLOG(1) << "can not list object " << key << " errMsg: " << msg; + return errors::NotFound("can not list object:", key, " errMsg: ", msg); + } + + aos_list_for_each_entry(oss_list_object_content_t, content, + ¶ms->object_list, node) { + int path_length = content->key.len; + if (should_remove_suffix && path_length > 0 && + content->key.data[content->key.len - 1] == '/') { + path_length = content->key.len - 1; + } + if (return_full_path) { + string child(content->key.data, 0, path_length); + result->push_back(child); + } else { + int prefix_len = (key.length() > 0 && key.at(key.length() - 1) != '/') + ? key.length() + 1 + : key.length(); + // remove prefix for GetChildren + if (content->key.len > prefix_len) { + string child(content->key.data + prefix_len, 0, + path_length - prefix_len); + result->push_back(child); + } + } + } + + next_marker = apr_psprintf(pool, "%.*s", params->next_marker.len, + params->next_marker.data); + + aos_str_set(¶ms->marker, next_marker); + aos_list_init(¶ms->object_list); + aos_list_init(¶ms->common_prefix_list); + } while (params->truncated == AOS_TRUE && return_all); + + return Status::OK(); +} + +Status OSSFileSystem::_StatInternal(aos_pool_t* pool, + oss_request_options_t* options, + const std::string& bucket, + const std::string& object, + FileStatistics* stat) { + Status s = _RetrieveObjectMetadata(pool, options, bucket, object, stat); + if (s.ok()) { + return s; + } + + // add suffix + std::string objectName = object + kDelim; + s = _RetrieveObjectMetadata(pool, options, bucket, objectName, stat); + if (s.ok()) { + stat->is_directory = true; + return s; + } + + // check list if it has children + std::vector listing; + s = _ListObjects(pool, options, bucket, object, &listing, false, true, 10); + if (s == Status::OK() && !listing.empty()) { + stat->is_directory = true; + stat->length = 0; + return s; + } + + return errors::NotFound("can not find ", object); +} + +Status OSSFileSystem::_RetrieveObjectMetadata(aos_pool_t* pool, + oss_request_options_t* options, + const std::string& bucket, + const std::string& object, + FileStatistics* stat) { + aos_string_t oss_bucket; + aos_string_t oss_object; + aos_table_t* headers = NULL; + aos_table_t* resp_headers = NULL; + aos_status_t* status = NULL; + char* content_length_str = NULL; + char* object_date_str = NULL; + + if (object.empty()) { // root always exists + stat->is_directory = true; + stat->length = 0; + return Status::OK(); + } + + aos_str_set(&oss_bucket, bucket.c_str()); + aos_str_set(&oss_object, object.c_str()); + headers = aos_table_make(pool, 0); + + status = oss_head_object(options, &oss_bucket, &oss_object, headers, + &resp_headers); + if (aos_status_is_ok(status)) { + content_length_str = (char*)apr_table_get(resp_headers, OSS_CONTENT_LENGTH); + if (content_length_str != NULL) { + stat->length = static_cast(atoll(content_length_str)); + } + + object_date_str = (char*)apr_table_get(resp_headers, OSS_DATE); + if (object_date_str != NULL) { + // the time is GMT Date, format like below + // Date: Fri, 24 Feb 2012 07:32:52 GMT + std::tm tm = {}; + strptime(object_date_str, "%a, %d %b %Y %H:%M:%S", &tm); + stat->mtime_nsec = static_cast(mktime(&tm) * 1000) * 1e9; + } + + if (object[object.length() - 1] == '/') { + stat->is_directory = true; + } else { + stat->is_directory = false; + } + + return Status::OK(); + } else { + string msg; + VLOG(1) << "can not find " << object << " errMsg: " << msg; + oss_error_message(status, &msg); + return errors::NotFound("can not find ", object, " errMsg: ", msg); + } +} + +Status OSSFileSystem::Stat(const std::string& fname, FileStatistics* stat) { + TF_RETURN_IF_ERROR(oss_initialize()); + std::string object, bucket; + TF_RETURN_IF_ERROR(_ParseOSSURIPath(fname, &bucket, &object)); + OSSConnection oss(host_, access_id_, access_key_); + oss_request_options_t* ossOptions = oss.getRequestOptions(); + aos_pool_t* pool = oss.getPool(); + + return _StatInternal(pool, ossOptions, bucket, object, stat); +} + +Status OSSFileSystem::GetChildren(const std::string& dir, + std::vector* result) { + result->clear(); + TF_RETURN_IF_ERROR(oss_initialize()); + std::string object, bucket; + TF_RETURN_IF_ERROR(_ParseOSSURIPath(dir, &bucket, &object)); + OSSConnection oss(host_, access_id_, access_key_); + oss_request_options_t* oss_options = oss.getRequestOptions(); + aos_pool_t* pool = oss.getPool(); + return _ListObjects(pool, oss_options, bucket, object, result, true, false, + 1000); +} + +Status OSSFileSystem::GetMatchingPaths(const std::string& pattern, + std::vector* results) { + results->clear(); + // Find the fixed prefix by looking for the first wildcard. + const string& fixed_prefix = + pattern.substr(0, pattern.find_first_of("*?[\\")); + const string& dir = string(io::Dirname(fixed_prefix)); + const string& base_prefix = string(io::Basename(fixed_prefix)); + if (dir.empty()) { + return errors::InvalidArgument("An OSS pattern doesn't have a bucket name: ", + pattern); + } + + std::string new_uri = dir; + size_t params_pos = pattern.find_last_of('?'); + if (params_pos != std::string::npos) { + new_uri.append(pattern.substr(params_pos)); + } + + TF_RETURN_IF_ERROR(oss_initialize()); + std::string object, bucket; + TF_RETURN_IF_ERROR(_ParseOSSURIPath(new_uri, &bucket, &object)); + + OSSConnection oss(host_, access_id_, access_key_); + oss_request_options_t* oss_options = oss.getRequestOptions(); + aos_pool_t* pool = oss.getPool(); + + std::vector listing; + TF_RETURN_IF_ERROR(_ListObjects(pool, oss_options, bucket, object, &listing, + true, false, 1000)); + + // Match all obtained paths to the input pattern. + for (auto path : listing) { + if (str_util::StartsWith(StringPiece(path), base_prefix)) { + const string file_path = io::JoinPath(dir, path); + if (Env::Default()->MatchPath(file_path, pattern)) { + results->push_back(dir + kDelim + path); + } + } + } + + return Status::OK(); +} + +Status OSSFileSystem::_DeleteObjectInternal(oss_request_options_t* options, + const std::string& bucket, + const std::string& object) { + aos_string_t bucket_; + aos_string_t object_; + aos_table_t* resp_headers = NULL; + aos_status_t* s = NULL; + + aos_str_set(&bucket_, bucket.c_str()); + aos_str_set(&object_, object.c_str()); + + s = oss_delete_object(options, &bucket_, &object_, &resp_headers); + if (!aos_status_is_ok(s)) { + string msg; + oss_error_message(s, &msg); + VLOG(1) << "delete " << object << " failed, errMsg: " << msg; + return errors::Internal("delete failed: ", object, " errMsg: ", msg); + } + + return Status::OK(); +} + +Status OSSFileSystem::DeleteFile(const std::string& fname) { + TF_RETURN_IF_ERROR(oss_initialize()); + std::string object, bucket; + TF_RETURN_IF_ERROR(_ParseOSSURIPath(fname, &bucket, &object)); + OSSConnection oss(host_, access_id_, access_key_); + oss_request_options_t* oss_options = oss.getRequestOptions(); + + return _DeleteObjectInternal(oss_options, bucket, object); +} + +Status OSSFileSystem::CreateDir(const std::string& dirname) { + TF_RETURN_IF_ERROR(oss_initialize()); + std::string object, bucket; + TF_RETURN_IF_ERROR(_ParseOSSURIPath(dirname, &bucket, &object)); + OSSConnection oss(host_, access_id_, access_key_); + oss_request_options_t* ossOptions = oss.getRequestOptions(); + aos_pool_t* pool = oss.getPool(); + StringPiece dirs(object); + + std::vector splitPaths = + str_util::Split(dirs, '/', str_util::SkipEmpty()); + if (splitPaths.size() < 2) { + return _CreateDirInternal(pool, ossOptions, bucket, object); + } + + FileStatistics stat; + StringPiece parent = io::Dirname(dirs); + if (!_StatInternal(pool, ossOptions, bucket, string(parent), &stat).ok()) { + return errors::Internal("parent does not exists: ", parent); + } + + if (!stat.is_directory) { + return errors::Internal("can not mkdir because parent is a file: ", parent); + } + + TF_RETURN_IF_ERROR(_CreateDirInternal(pool, ossOptions, bucket, object)); + return Status::OK(); +} + +Status OSSFileSystem::RecursivelyCreateDir(const string& dirname) { + TF_RETURN_IF_ERROR(oss_initialize()); + std::string object, bucket; + TF_RETURN_IF_ERROR(_ParseOSSURIPath(dirname, &bucket, &object)); + OSSConnection oss(host_, access_id_, access_key_); + oss_request_options_t* ossOptions = oss.getRequestOptions(); + aos_pool_t* pool = oss.getPool(); + StringPiece dirs(object); + + std::vector splitPaths = + str_util::Split(dirs, '/', str_util::SkipEmpty()); + if (splitPaths.size() < 2) { + return _CreateDirInternal(pool, ossOptions, bucket, object); + } + + std::string dir = ""; + for (auto path : splitPaths) { + dir.append(path + kDelim); + if (!_CreateDirInternal(pool, ossOptions, bucket, dir).ok()) { + return errors::Internal("create dir failed: ", dir); + } + } + + return Status::OK(); +} + +Status OSSFileSystem::_CreateDirInternal(aos_pool_t* pool, + oss_request_options_t* options, + const std::string& bucket, + const std::string& dirname) { + FileStatistics stat; + if (_RetrieveObjectMetadata(pool, options, bucket, dirname, &stat).ok()) { + if (!stat.is_directory) { + return errors::AlreadyExists("object already exists as a file: ", + dirname); + } else { + return Status::OK(); + } + } + std::string object = dirname; + if (dirname.at(dirname.length() - 1) != '/') { + object += '/'; + } + + aos_status_t* s; + aos_table_t* headers; + aos_table_t* resp_headers; + aos_string_t bucket_; + aos_string_t object_; + const char* data = ""; + aos_list_t buffer; + aos_buf_t* content; + + aos_str_set(&bucket_, bucket.c_str()); + aos_str_set(&object_, object.c_str()); + headers = aos_table_make(pool, 0); + + aos_list_init(&buffer); + content = aos_buf_pack(options->pool, data, strlen(data)); + aos_list_add_tail(&content->node, &buffer); + s = oss_put_object_from_buffer(options, &bucket_, &object_, &buffer, headers, + &resp_headers); + + if (aos_status_is_ok(s)) { + return Status::OK(); + } else { + string msg; + oss_error_message(s, &msg); + VLOG(1) << "mkdir " << dirname << " failed, errMsg: " << msg; + return errors::Internal("mkdir failed: ", dirname, " errMsg: ", msg); + } +} + +Status OSSFileSystem::DeleteDir(const std::string& dirname) { + TF_RETURN_IF_ERROR(oss_initialize()); + std::string object, bucket; + TF_RETURN_IF_ERROR(_ParseOSSURIPath(dirname, &bucket, &object)); + OSSConnection oss(host_, access_id_, access_key_); + oss_request_options_t* oss_options = oss.getRequestOptions(); + aos_pool_t* pool = oss.getPool(); + std::vector children; + Status s; + + s = _ListObjects(pool, oss_options, bucket, object, &children, false, true, + 10); + if (s.ok() && !children.empty()) { + return errors::FailedPrecondition("Cannot delete a non-empty directory."); + } + + s = _DeleteObjectInternal(oss_options, bucket, object); + + if (s.ok()) { + return s; + } + + // Maybe should add slash + return _DeleteObjectInternal(oss_options, bucket, object.append(kDelim)); +} + +Status OSSFileSystem::GetFileSize(const std::string& fname, uint64* file_size) { + FileStatistics stat; + TF_RETURN_IF_ERROR(Stat(fname, &stat)); + *file_size = stat.length; + return Status::OK(); +} + +Status OSSFileSystem::RenameFile(const std::string& src, + const std::string& target) { + TF_RETURN_IF_ERROR(oss_initialize()); + std::string sobject, sbucket; + TF_RETURN_IF_ERROR(_ParseOSSURIPath(src, &sbucket, &sobject)); + std::string dobject, dbucket; + TF_RETURN_IF_ERROR(_ParseOSSURIPath(target, &dbucket, &dobject)); + + OSSConnection oss(host_, access_id_, access_key_); + oss_request_options_t* oss_options = oss.getRequestOptions(); + aos_pool_t* pool = oss.getPool(); + + aos_status_t* s; + aos_table_t* headers; + aos_table_t* resp_headers; + aos_string_t source_bucket; + aos_string_t source_object; + aos_string_t dest_bucket; + aos_string_t dest_object; + + aos_str_set(&source_bucket, sbucket.c_str()); + aos_str_set(&source_object, sobject.c_str()); + aos_str_set(&dest_bucket, dbucket.c_str()); + aos_str_set(&dest_object, dobject.c_str()); + headers = aos_table_make(pool, 0); + + s = oss_copy_object(oss_options, &source_bucket, &source_object, &dest_bucket, + &dest_object, headers, &resp_headers); + + if (!aos_status_is_ok(s)) { + string msg; + oss_error_message(s, &msg); + VLOG(1) << "rename " << src << " to " << target + << " failed, errMsg: " << msg; + return errors::Internal("rename ", src, " to ", target, + " failed, errMsg: ", msg); + } + + return _DeleteObjectInternal(oss_options, sbucket, sobject); +} + +Status OSSFileSystem::IsDirectory(const std::string& fname) { + FileStatistics stat; + TF_RETURN_IF_ERROR(Stat(fname, &stat)); + + return stat.is_directory + ? Status::OK() + : errors::FailedPrecondition(fname + " is not a directory"); +} + +Status OSSFileSystem::DeleteRecursively(const std::string& dirname, + int64* undeleted_files, + int64* undeleted_dirs) { + if (!undeleted_files || !undeleted_dirs) { + return errors::Internal( + "'undeleted_files' and 'undeleted_dirs' cannot be nullptr."); + } + *undeleted_files = 0; + *undeleted_dirs = 0; + + TF_RETURN_IF_ERROR(oss_initialize()); + std::string object, bucket; + TF_RETURN_IF_ERROR(_ParseOSSURIPath(dirname, &bucket, &object)); + OSSConnection oss(host_, access_id_, access_key_); + oss_request_options_t* oss_options = oss.getRequestOptions(); + aos_pool_t* pool = oss.getPool(); + std::vector children; + + FileStatistics stat; + Status s; + s = _StatInternal(pool, oss_options, bucket, object, &stat); + if (!s.ok() || !stat.is_directory) { + *undeleted_dirs = 1; + return errors::NotFound(dirname, " doesn't exist or not a directory."); + } + + s = _ListObjects(pool, oss_options, bucket, object, &children, true, true, + false, 1000); + if (!s.ok()) { + // empty dir, just delete it + return _DeleteObjectInternal(oss_options, bucket, object); + } + + for (auto child : children) { + s = _DeleteObjectInternal(oss_options, bucket, child); + if (!s.ok()) { + s = _StatInternal(pool, oss_options, bucket, child, &stat); + if (s.ok()) { + if (stat.is_directory) { + ++*undeleted_dirs; + } else { + ++*undeleted_files; + } + } + } + } + + if (*undeleted_dirs == 0 && *undeleted_files == 0) { + // delete directory itself. + if (object.at(object.length() - 1) == '/') { + return _DeleteObjectInternal(oss_options, bucket, object); + } else { + return _DeleteObjectInternal(oss_options, bucket, object.append(kDelim)); + } + } + return Status::OK(); +} +} // end namespace tensorflow diff --git a/tensorflow_io/oss/kernels/ossfs/oss_file_system.h b/tensorflow_io/oss/kernels/ossfs/oss_file_system.h new file mode 100644 index 000000000..368320248 --- /dev/null +++ b/tensorflow_io/oss/kernels/ossfs/oss_file_system.h @@ -0,0 +1,132 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_IO_OSS_KERNELS_OSSFS_OSS_FILE_SYSTEM_H_ +#define TENSORFLOW_IO_OSS_KERNELS_OSSFS_OSS_FILE_SYSTEM_H_ + +#include +#include +#include +#include + +#include "aos_log.h" +#include "aos_status.h" +#include "aos_string.h" +#include "aos_util.h" +#include "oss_api.h" +#include "oss_auth.h" +#include "oss_util.h" + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/file_system.h" +#include "tensorflow/core/platform/mutex.h" + +namespace tensorflow { + +/// Aliyun oss implementation of a file system. +class OSSFileSystem : public FileSystem { + public: + OSSFileSystem(); + + Status NewRandomAccessFile( + const string& filename, + std::unique_ptr* result) override; + + Status NewWritableFile(const string& fname, + std::unique_ptr* result) override; + + Status NewAppendableFile(const string& fname, + std::unique_ptr* result) override; + + Status NewReadOnlyMemoryRegionFromFile( + const string& filename, + std::unique_ptr* result) override; + + Status FileExists(const string& fname) override; + + Status Stat(const string& fname, FileStatistics* stat) override; + + Status GetChildren(const string& dir, std::vector* result) override; + + Status GetMatchingPaths(const string& pattern, + std::vector* results) override; + + Status DeleteFile(const string& fname) override; + + Status CreateDir(const string& dirname) override; + + Status RecursivelyCreateDir(const string& dirname) override; + + Status DeleteDir(const string& dirname) override; + + Status GetFileSize(const string& fname, uint64* file_size) override; + + Status RenameFile(const string& src, const string& target) override; + + Status IsDirectory(const string& fname) override; + + Status DeleteRecursively(const string& dirname, int64* undeleted_files, + int64* undeleted_dirs) override; + + private: + Status _CreateDirInternal(aos_pool_t* pool, oss_request_options_t* options, + const string& bucket, const string& dirname); + + Status _StatInternal(aos_pool_t* pool, oss_request_options_t* options, + const string& bucket, const string& object, + FileStatistics* stat); + + Status _DeleteObjectInternal(oss_request_options_t* options, + const string& bucket, const string& object); + + Status _RetrieveObjectMetadata(aos_pool_t* pool, + oss_request_options_t* options, + const string& bucket, const string& object, + FileStatistics* stat); + + Status _ListObjects(aos_pool_t* pool, oss_request_options_t* options, + const string& bucket, const string& key, + std::vector* result, bool return_all, + bool return_full_path, bool should_remove_suffix = true, + int max_ret_per_iterator = 1000); + + Status _InitOSSCredentials(); + + Status _ParseOSSURIPath(StringPiece fname, std::string* bucket, + std::string* object); + + // The number of bytes to read ahead for buffering purposes + // in the RandomAccessFile implementation. Defaults to 5Mb. + const size_t read_ahead_bytes_ = 5 * 1024 * 1024; + + // The number of bytes for each upload part. Defaults to 64MB + const size_t upload_part_bytes_ = 64 * 1024 * 1024; + + // The max number of attempts to upload a file to OSS using the resumable + // upload API. + const int32 max_upload_attempts_ = 5; + + mutex mu_; + string host_; + string access_id_; + string access_key_; + std::atomic_bool init_; + + TF_DISALLOW_COPY_AND_ASSIGN(OSSFileSystem); +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_IO_OSS_KERNELS_OSSFS_OSS_FILE_SYSTEM_H_ diff --git a/tensorflow_io/oss/ops/ossfs_ops.cc b/tensorflow_io/oss/ops/ossfs_ops.cc new file mode 100644 index 000000000..646317c92 --- /dev/null +++ b/tensorflow_io/oss/ops/ossfs_ops.cc @@ -0,0 +1,24 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/platform/env.h" + +#include "tensorflow_io/oss/kernels/ossfs/oss_file_system.h" + +namespace tensorflow { + +REGISTER_FILE_SYSTEM("oss", OSSFileSystem); + +} // namespace tensorflow diff --git a/tensorflow_io/oss/python/__init__.py b/tensorflow_io/oss/python/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tensorflow_io/oss/python/ops/__init__.py b/tensorflow_io/oss/python/ops/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tensorflow_io/oss/python/ops/ossfs_ops.py b/tensorflow_io/oss/python/ops/ossfs_ops.py new file mode 100644 index 000000000..9d8b8c49a --- /dev/null +++ b/tensorflow_io/oss/python/ops/ossfs_ops.py @@ -0,0 +1,27 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""OSS File System Support + +OSS is an Object Storage Service provided by Alibaba Cloud. This module +implements a filesystem on top of it. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow_io import _load_library + +_load_library("_oss_ops.so") diff --git a/tests/test_ossfs.py b/tests/test_ossfs.py new file mode 100644 index 000000000..1dc3cae50 --- /dev/null +++ b/tests/test_ossfs.py @@ -0,0 +1,109 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for OSS filesystem""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import unittest + +from tensorflow.python.platform import test +from tensorflow.python.platform import gfile +from tensorflow_io.oss import ossfs_ops # pylint: disable=unused-import + +bucket = None +get_oss_path = None + +def _have_required_env(): + return os.getenv("OSS_CREDENTIALS") and os.getenv("OSS_FS_TEST_BUCKET") + +_msg = ("OSS tests skipped. To enable them, set OSS_CREDENTIALS env variable " + "to the path of your oss credential file and OSS_FS_TEST_BUCKET env " + "variable to your oss test bucket name.") + +@unittest.skipIf(not _have_required_env(), _msg) +class OSSFSTest(test.TestCase): + """OSS Filesystem Tests""" + + @classmethod + def setUpClass(cls): # pylint: disable=invalid-name + global bucket, get_oss_path + bucket = os.getenv("OSS_FS_TEST_BUCKET") + get_oss_path = lambda p: os.path.join("oss://" + bucket, "oss_fs_test", p) + gfile.MkDir(get_oss_path("")) + + @classmethod + def tearDownClass(cls): # pylint: disable=invalid-name + gfile.DeleteRecursively(get_oss_path("")) + + def test_file_operations(self): + """ Test file operations""" + + f = get_oss_path("test_file_operations") + self.assertFalse(gfile.Exists(f)) + + fh = gfile.Open(f, mode="w") + content = "file content" + fh.write(content) + fh.close() + self.assertTrue(gfile.Exists(f)) + + fh = gfile.Open(f) + self.assertEqual(fh.read(), content) + + self.assertEqual(gfile.Stat(f).length, len(content)) + + f2 = get_oss_path("test_file_2") + gfile.Rename(f, f2) + self.assertFalse(gfile.Exists(f)) + self.assertTrue(gfile.Exists(f2)) + + def test_dir_operations(self): + """ Test directory operations""" + + d = get_oss_path("d1/d2") + gfile.MakeDirs(d) + self.assertTrue(gfile.Stat(d).is_directory) + + # Test listing bucket directory with and without trailing '/' + content = gfile.ListDirectory("oss://" + bucket) + content_s = gfile.ListDirectory("oss://" + bucket + "/") + self.assertEqual(content, content_s) + self.assertIn("oss_fs_test", content) + self.assertIn("oss_fs_test/d1", content) + self.assertIn("oss_fs_test/d1/d2", content) + + # Test listing test directory with and without trailing '/' + content = gfile.ListDirectory("oss://" + bucket + "/oss_fs_test") + content_s = gfile.ListDirectory("oss://" + bucket + "/oss_fs_test/") + self.assertEqual(content, content_s) + self.assertIn("d1", content) + self.assertIn("d1/d2", content) + + # Test listing sub directories. + content = gfile.ListDirectory(get_oss_path("d1")) + content_s = gfile.ListDirectory(get_oss_path("d1/")) + self.assertEqual(content, content_s) + self.assertIn("d2", content) + + content = gfile.ListDirectory(get_oss_path("d1/d2")) + content_s = gfile.ListDirectory(get_oss_path("d1/d2/")) + self.assertEqual(content, content_s) + self.assertEqual([], content) + + +if __name__ == "__main__": + test.main() diff --git a/third_party/libapr1.BUILD b/third_party/libapr1.BUILD new file mode 100644 index 000000000..655509524 --- /dev/null +++ b/third_party/libapr1.BUILD @@ -0,0 +1,172 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache license (for libapr1) + +cc_library( + name = "libapr1", + srcs = [ + "atomic/unix/builtins.c", + "atomic/unix/ia32.c", + "atomic/unix/mutex.c", + "atomic/unix/ppc.c", + "atomic/unix/s390.c", + "atomic/unix/solaris.c", + "dso/unix/dso.c", + "encoding/apr_escape.c", + "file_io/unix/buffer.c", + "file_io/unix/copy.c", + "file_io/unix/dir.c", + "file_io/unix/fileacc.c", + "file_io/unix/filedup.c", + "file_io/unix/filepath.c", + "file_io/unix/filepath_util.c", + "file_io/unix/filestat.c", + "file_io/unix/flock.c", + "file_io/unix/fullrw.c", + "file_io/unix/mktemp.c", + "file_io/unix/open.c", + "file_io/unix/pipe.c", + "file_io/unix/readwrite.c", + "file_io/unix/seek.c", + "file_io/unix/tempdir.c", + "include/arch/apr_private_common.h", + "include/arch/unix/apr_arch_atomic.h", + "include/arch/unix/apr_arch_dso.h", + "include/arch/unix/apr_arch_file_io.h", + "include/arch/unix/apr_arch_global_mutex.h", + "include/arch/unix/apr_arch_inherit.h", + "include/arch/unix/apr_arch_internal_time.h", + "include/arch/unix/apr_arch_misc.h", + "include/arch/unix/apr_arch_networkio.h", + "include/arch/unix/apr_arch_poll_private.h", + "include/arch/unix/apr_arch_proc_mutex.h", + "include/arch/unix/apr_arch_shm.h", + "include/arch/unix/apr_arch_thread_cond.h", + "include/arch/unix/apr_arch_thread_mutex.h", + "include/arch/unix/apr_arch_thread_rwlock.h", + "include/arch/unix/apr_arch_threadproc.h", + "include/arch/unix/apr_private.h", + "include/private/apr_escape_test_char.h", + "locks/unix/global_mutex.c", + "locks/unix/proc_mutex.c", + "locks/unix/thread_cond.c", + "locks/unix/thread_mutex.c", + "locks/unix/thread_rwlock.c", + "memory/unix/apr_pools.c", + "misc/unix/charset.c", + "misc/unix/env.c", + "misc/unix/errorcodes.c", + "misc/unix/getopt.c", + "misc/unix/otherchild.c", + "misc/unix/rand.c", + "misc/unix/start.c", + "misc/unix/version.c", + "mmap/unix/common.c", + "mmap/unix/mmap.c", + "network_io/unix/inet_ntop.c", + "network_io/unix/inet_pton.c", + "network_io/unix/multicast.c", + "network_io/unix/sendrecv.c", + "network_io/unix/sockaddr.c", + "network_io/unix/socket_util.c", + "network_io/unix/sockets.c", + "network_io/unix/sockopt.c", + "passwd/apr_getpass.c", + "poll/unix/epoll.c", + "poll/unix/kqueue.c", + "poll/unix/poll.c", + "poll/unix/pollcb.c", + "poll/unix/pollset.c", + "poll/unix/port.c", + "poll/unix/select.c", + "poll/unix/wakeup.c", + "poll/unix/z_asio.c", + "random/unix/apr_random.c", + "random/unix/sha2.c", + "random/unix/sha2.h", + "random/unix/sha2_glue.c", + "shmem/unix/shm.c", + "strings/apr_cpystrn.c", + "strings/apr_cstr.c", + "strings/apr_fnmatch.c", + "strings/apr_snprintf.c", + "strings/apr_strings.c", + "strings/apr_strnatcmp.c", + "strings/apr_strtok.c", + "support/unix/waitio.c", + "tables/apr_hash.c", + "tables/apr_skiplist.c", + "tables/apr_tables.c", + "threadproc/unix/proc.c", + "threadproc/unix/procsup.c", + "threadproc/unix/signals.c", + "threadproc/unix/thread.c", + "threadproc/unix/threadpriv.c", + "time/unix/time.c", + "time/unix/timestr.c", + "tools/gen_test_char.c", + "user/unix/groupinfo.c", + "user/unix/userinfo.c", + ], + hdrs = [ + "include/apr.h", + "include/apr_allocator.h", + "include/apr_atomic.h", + "include/apr_cstr.h", + "include/apr_dso.h", + "include/apr_env.h", + "include/apr_errno.h", + "include/apr_escape.h", + "include/apr_file_info.h", + "include/apr_file_io.h", + "include/apr_fnmatch.h", + "include/apr_general.h", + "include/apr_getopt.h", + "include/apr_global_mutex.h", + "include/apr_hash.h", + "include/apr_inherit.h", + "include/apr_lib.h", + "include/apr_mmap.h", + "include/apr_network_io.h", + "include/apr_perms_set.h", + "include/apr_poll.h", + "include/apr_pools.h", + "include/apr_portable.h", + "include/apr_proc_mutex.h", + "include/apr_random.h", + "include/apr_ring.h", + "include/apr_shm.h", + "include/apr_signal.h", + "include/apr_skiplist.h", + "include/apr_strings.h", + "include/apr_support.h", + "include/apr_tables.h", + "include/apr_thread_cond.h", + "include/apr_thread_mutex.h", + "include/apr_thread_proc.h", + "include/apr_thread_rwlock.h", + "include/apr_time.h", + "include/apr_user.h", + "include/apr_version.h", + "include/apr_want.h", + ], + copts = [ + "-pthread", + ], + defines = [ + "_GNU_SOURCE", + "HAVE_CONFIG_H", + "LINUX", + "REENTRANT", + ], + includes = [ + "include", + "include/arch/unix", + "include/private", + ], + linkopts = [ + "-lpthread", + "-ldl", + ], + strip_include_prefix = "include", +) diff --git a/third_party/libapr1.patch b/third_party/libapr1.patch new file mode 100644 index 000000000..79e575af0 --- /dev/null +++ b/third_party/libapr1.patch @@ -0,0 +1,1874 @@ +diff -urN libapr1/include/apr.h libapr/include/apr.h +--- libapr1/include/apr.h 1970-01-01 00:00:00.000000000 +0000 ++++ libapr/include/apr.h 2019-02-09 23:44:46.000000000 +0000 +@@ -0,0 +1,679 @@ ++/* Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++ ++#ifndef APR_H ++#define APR_H ++ ++/* GENERATED FILE WARNING! DO NOT EDIT apr.h ++ * ++ * You must modify apr.h.in instead. ++ * ++ * And please, make an effort to stub apr.hw and apr.hnw in the process. ++ */ ++ ++/** ++ * @file apr.h ++ * @brief APR Platform Definitions ++ * @remark This is a generated header generated from include/apr.h.in by ++ * ./configure, or copied from include/apr.hw or include/apr.hnw ++ * for Win32 or Netware by those build environments, respectively. ++ */ ++ ++/** ++ * @defgroup APR Apache Portability Runtime library ++ * @{ ++ */ ++/** ++ * @defgroup apr_platform Platform Definitions ++ * @{ ++ * @warning ++ * The actual values of macros and typedefs on this page
++ * are platform specific and should NOT be relied upon! ++ */ ++ ++/* So that we can use inline on some critical functions, and use ++ * GNUC attributes (such as to get -Wall warnings for printf-like ++ * functions). Only do this in gcc 2.7 or later ... it may work ++ * on earlier stuff, but why chance it. ++ * ++ * We've since discovered that the gcc shipped with NeXT systems ++ * as "cc" is completely broken. It claims to be __GNUC__ and so ++ * on, but it doesn't implement half of the things that __GNUC__ ++ * means. In particular it's missing inline and the __attribute__ ++ * stuff. So we hack around it. PR#1613. -djg ++ */ ++#if !defined(__GNUC__) || __GNUC__ < 2 || \ ++ (__GNUC__ == 2 && __GNUC_MINOR__ < 7) ||\ ++ defined(NEXT) ++#ifndef __attribute__ ++#define __attribute__(__x) ++#endif ++#define APR_INLINE ++#define APR_HAS_INLINE 0 ++#else ++#define APR_INLINE __inline__ ++#define APR_HAS_INLINE 1 ++#endif ++ ++#define APR_HAVE_ARPA_INET_H 1 ++#define APR_HAVE_CONIO_H 0 ++#ifdef __APPLE__ ++#define APR_HAVE_CRYPT_H 0 ++#else ++#define APR_HAVE_CRYPT_H 1 ++#endif ++#define APR_HAVE_CTYPE_H 1 ++#define APR_HAVE_DIRENT_H 1 ++#define APR_HAVE_ERRNO_H 1 ++#define APR_HAVE_FCNTL_H 1 ++#define APR_HAVE_IO_H 0 ++#define APR_HAVE_LIMITS_H 1 ++#define APR_HAVE_NETDB_H 1 ++#define APR_HAVE_NETINET_IN_H 1 ++#define APR_HAVE_NETINET_SCTP_H 0 ++#define APR_HAVE_NETINET_SCTP_UIO_H 0 ++#define APR_HAVE_NETINET_TCP_H 1 ++#define APR_HAVE_PROCESS_H 0 ++#define APR_HAVE_PTHREAD_H 1 ++#define APR_HAVE_SEMAPHORE_H 1 ++#define APR_HAVE_SIGNAL_H 1 ++#define APR_HAVE_STDARG_H 1 ++#define APR_HAVE_STDINT_H 1 ++#define APR_HAVE_STDIO_H 1 ++#define APR_HAVE_STDLIB_H 1 ++#define APR_HAVE_STRING_H 1 ++#define APR_HAVE_STRINGS_H 1 ++#define APR_HAVE_SYS_IOCTL_H 1 ++#ifdef __APPLE__ ++#define APR_HAVE_SYS_SENDFILE_H 0 ++#else ++#define APR_HAVE_SYS_SENDFILE_H 1 ++#endif ++#define APR_HAVE_SYS_SIGNAL_H 1 ++#define APR_HAVE_SYS_SOCKET_H 1 ++#ifdef __APPLE__ ++#define APR_HAVE_SYS_SOCKIO_H 1 ++#define APR_HAVE_SYS_SYSLIMITS_H 1 ++#else ++#define APR_HAVE_SYS_SOCKIO_H 0 ++#define APR_HAVE_SYS_SYSLIMITS_H 0 ++#endif ++#define APR_HAVE_SYS_TIME_H 1 ++#define APR_HAVE_SYS_TYPES_H 1 ++#define APR_HAVE_SYS_UIO_H 1 ++#define APR_HAVE_SYS_UN_H 1 ++#define APR_HAVE_SYS_WAIT_H 1 ++#define APR_HAVE_TIME_H 1 ++#define APR_HAVE_UNISTD_H 1 ++#define APR_HAVE_WINDOWS_H 0 ++#define APR_HAVE_WINSOCK2_H 0 ++ ++/** @} */ ++/** @} */ ++ ++/* We don't include our conditional headers within the doxyblocks ++ * or the extern "C" namespace ++ */ ++ ++#if APR_HAVE_WINDOWS_H && defined(WIN32) ++/* If windows.h was already included, our preferences don't matter. ++ * If not, include a restricted set of windows headers to our tastes. ++ */ ++#ifndef _WINDOWS_ ++ ++#ifndef WIN32_LEAN_AND_MEAN ++#define WIN32_LEAN_AND_MEAN ++#endif ++ ++#ifndef _WIN32_WINNT ++/* Restrict the server to a subset of Windows XP header files by default ++ */ ++#define _WIN32_WINNT 0x0501 ++#endif ++ ++#ifndef NOUSER ++#define NOUSER ++#endif ++#ifndef NOMCX ++#define NOMCX ++#endif ++#ifndef NOIME ++#define NOIME ++#endif ++ ++#include ++/* ++ * Add a _very_few_ declarations missing from the restricted set of headers ++ * (If this list becomes extensive, re-enable the required headers above!) ++ * winsock headers were excluded by WIN32_LEAN_AND_MEAN, so include them now ++ */ ++#define SW_HIDE 0 ++#ifndef _WIN32_WCE ++#include ++#include ++#include ++#else ++#include ++#endif ++ ++#endif /* ndef _WINDOWS_ */ ++#endif /* APR_HAVE_WINDOWS_H */ ++ ++#if APR_HAVE_SYS_TYPES_H ++#include ++#endif ++ ++#if APR_HAVE_SYS_SOCKET_H ++#include ++#endif ++ ++#if defined(__cplusplus) && !defined(__STDC_CONSTANT_MACROS) ++/* C99 7.18.4 requires that stdint.h only exposes INT64_C ++ * and UINT64_C for C++ implementations if this is defined: */ ++#define __STDC_CONSTANT_MACROS ++#endif ++ ++#if APR_HAVE_STDINT_H ++#include ++#endif ++ ++#if APR_HAVE_SYS_WAIT_H ++#include ++#endif ++ ++#ifdef OS2 ++#define INCL_DOS ++#define INCL_DOSERRORS ++#include ++#endif ++ ++/* header files for PATH_MAX, _POSIX_PATH_MAX */ ++#if APR_HAVE_LIMITS_H ++#include ++#else ++#if APR_HAVE_SYS_SYSLIMITS_H ++#include ++#endif ++#endif ++ ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/** ++ * @addtogroup apr_platform ++ * @ingroup APR ++ * @{ ++ */ ++ ++#define APR_HAVE_SHMEM_MMAP_TMP 1 ++#define APR_HAVE_SHMEM_MMAP_SHM 1 ++#ifdef __APPLE__ ++#define APR_HAVE_SHMEM_MMAP_ZERO 0 ++#else ++#define APR_HAVE_SHMEM_MMAP_ZERO 1 ++#endif ++#define APR_HAVE_SHMEM_SHMGET_ANON 1 ++#define APR_HAVE_SHMEM_SHMGET 1 ++#define APR_HAVE_SHMEM_MMAP_ANON 1 ++#define APR_HAVE_SHMEM_BEOS 0 ++ ++#define APR_USE_SHMEM_MMAP_TMP 0 ++#define APR_USE_SHMEM_MMAP_SHM 0 ++#define APR_USE_SHMEM_MMAP_ZERO 0 ++#define APR_USE_SHMEM_SHMGET_ANON 0 ++#define APR_USE_SHMEM_SHMGET 1 ++#define APR_USE_SHMEM_MMAP_ANON 1 ++#define APR_USE_SHMEM_BEOS 0 ++ ++#define APR_USE_FLOCK_SERIALIZE 0 ++#define APR_USE_SYSVSEM_SERIALIZE 1 ++#define APR_USE_POSIXSEM_SERIALIZE 0 ++#define APR_USE_FCNTL_SERIALIZE 0 ++#define APR_USE_PROC_PTHREAD_SERIALIZE 0 ++#define APR_USE_PTHREAD_SERIALIZE 1 ++ ++#define APR_HAS_FLOCK_SERIALIZE 1 ++#define APR_HAS_SYSVSEM_SERIALIZE 1 ++#define APR_HAS_POSIXSEM_SERIALIZE 1 ++#define APR_HAS_FCNTL_SERIALIZE 1 ++#ifdef __APPLE__ ++#define APR_HAS_PROC_PTHREAD_SERIALIZE 0 ++#else ++#define APR_HAS_PROC_PTHREAD_SERIALIZE 1 ++#endif ++ ++#define APR_PROCESS_LOCK_IS_GLOBAL 0 ++ ++#define APR_HAVE_CORKABLE_TCP 1 ++#define APR_HAVE_GETRLIMIT 1 ++#define APR_HAVE_IN_ADDR 1 ++#define APR_HAVE_INET_ADDR 1 ++#define APR_HAVE_INET_NETWORK 1 ++#define APR_HAVE_IPV6 1 ++#define APR_HAVE_SOCKADDR_UN 1 ++#define APR_HAVE_MEMMOVE 1 ++#define APR_HAVE_SETRLIMIT 1 ++#define APR_HAVE_SIGACTION 1 ++#define APR_HAVE_SIGSUSPEND 1 ++#define APR_HAVE_SIGWAIT 1 ++#define APR_HAVE_SA_STORAGE 1 ++#define APR_HAVE_STRCASECMP 1 ++#define APR_HAVE_STRDUP 1 ++#define APR_HAVE_STRICMP 0 ++#define APR_HAVE_STRNCASECMP 1 ++#define APR_HAVE_STRNICMP 0 ++#define APR_HAVE_STRSTR 1 ++#define APR_HAVE_MEMCHR 1 ++#define APR_HAVE_STRUCT_RLIMIT 1 ++#ifdef __APPLE__ ++#define APR_HAVE_UNION_SEMUN 1 ++#else ++#define APR_HAVE_UNION_SEMUN 0 ++#endif ++#define APR_HAVE_SCTP 0 ++#define APR_HAVE_IOVEC 1 ++ ++/* APR Feature Macros */ ++#define APR_HAS_SHARED_MEMORY 1 ++#define APR_HAS_THREADS 1 ++#define APR_HAS_SENDFILE 0 ++#define APR_HAS_MMAP 1 ++#define APR_HAS_FORK 1 ++#define APR_HAS_RANDOM 1 ++#define APR_HAS_OTHER_CHILD 1 ++#define APR_HAS_DSO 0 ++#define APR_HAS_SO_ACCEPTFILTER 0 ++#define APR_HAS_UNICODE_FS 0 ++#define APR_HAS_PROC_INVOKED 0 ++#define APR_HAS_USER 1 ++#define APR_HAS_LARGE_FILES 0 ++#define APR_HAS_XTHREAD_FILES 0 ++#ifdef __APPLE__ ++#define APR_HAS_OS_UUID 1 ++#else ++#define APR_HAS_OS_UUID 0 ++#endif ++ ++#define APR_PROCATTR_USER_SET_REQUIRES_PASSWORD 0 ++ ++/* APR sets APR_FILES_AS_SOCKETS to 1 on systems where it is possible ++ * to poll on files/pipes. ++ */ ++#define APR_FILES_AS_SOCKETS 1 ++ ++/* This macro indicates whether or not EBCDIC is the native character set. ++ */ ++#define APR_CHARSET_EBCDIC 0 ++ ++/* If we have a TCP implementation that can be "corked", what flag ++ * do we use? ++ */ ++#ifdef __APPLE__ ++#define APR_TCP_NOPUSH_FLAG TCP_NOPUSH ++#else ++#define APR_TCP_NOPUSH_FLAG TCP_CORK ++#endif ++ ++/* Is the TCP_NODELAY socket option inherited from listening sockets? ++*/ ++#define APR_TCP_NODELAY_INHERITED 1 ++ ++/* Is the O_NONBLOCK flag inherited from listening sockets? ++*/ ++#ifdef __APPLE__ ++#define APR_O_NONBLOCK_INHERITED 1 ++#else ++#define APR_O_NONBLOCK_INHERITED 0 ++#endif ++ ++/* Typedefs that APR needs. */ ++ ++typedef unsigned char apr_byte_t; ++ ++typedef short apr_int16_t; ++typedef unsigned short apr_uint16_t; ++ ++typedef int apr_int32_t; ++typedef unsigned int apr_uint32_t; ++ ++#define APR_SIZEOF_VOIDP 8 ++ ++/* ++ * Darwin 10's default compiler (gcc42) builds for both 64 and ++ * 32 bit architectures unless specifically told not to. ++ * In those cases, we need to override types depending on how ++ * we're being built at compile time. ++ * NOTE: This is an ugly work-around for Darwin's ++ * concept of universal binaries, a single package ++ * (executable, lib, etc...) which contains both 32 ++ * and 64 bit versions. The issue is that if APR is ++ * built universally, if something else is compiled ++ * against it, some bit sizes will depend on whether ++ * it is 32 or 64 bit. This is determined by the __LP64__ ++ * flag. Since we need to support both, we have to ++ * handle OS X unqiuely. ++ */ ++#ifdef DARWIN_10 ++#undef APR_SIZEOF_VOIDP ++#undef INT64_C ++#undef UINT64_C ++#ifdef __LP64__ ++ typedef long apr_int64_t; ++ typedef unsigned long apr_uint64_t; ++ #define APR_SIZEOF_VOIDP 8 ++ #define INT64_C(v) (v ## L) ++ #define UINT64_C(v) (v ## UL) ++#else ++ typedef long long apr_int64_t; ++ typedef unsigned long long apr_uint64_t; ++ #define APR_SIZEOF_VOIDP 4 ++ #define INT64_C(v) (v ## LL) ++ #define UINT64_C(v) (v ## ULL) ++#endif ++#else ++ typedef long apr_int64_t; ++ typedef unsigned long apr_uint64_t; ++#endif ++ ++typedef size_t apr_size_t; ++typedef ssize_t apr_ssize_t; ++typedef off_t apr_off_t; ++typedef socklen_t apr_socklen_t; ++typedef ino_t apr_ino_t; ++ ++#if APR_SIZEOF_VOIDP == 8 ++typedef apr_uint64_t apr_uintptr_t; ++#else ++typedef apr_uint32_t apr_uintptr_t; ++#endif ++ ++/* Are we big endian? */ ++#define APR_IS_BIGENDIAN 0 ++ ++/* Mechanisms to properly type numeric literals */ ++#define APR_INT64_C(val) INT64_C(val) ++#define APR_UINT64_C(val) UINT64_C(val) ++ ++#ifdef INT16_MIN ++#define APR_INT16_MIN INT16_MIN ++#else ++#define APR_INT16_MIN (-0x7fff - 1) ++#endif ++ ++#ifdef INT16_MAX ++#define APR_INT16_MAX INT16_MAX ++#else ++#define APR_INT16_MAX (0x7fff) ++#endif ++ ++#ifdef UINT16_MAX ++#define APR_UINT16_MAX UINT16_MAX ++#else ++#define APR_UINT16_MAX (0xffff) ++#endif ++ ++#ifdef INT32_MIN ++#define APR_INT32_MIN INT32_MIN ++#else ++#define APR_INT32_MIN (-0x7fffffff - 1) ++#endif ++ ++#ifdef INT32_MAX ++#define APR_INT32_MAX INT32_MAX ++#else ++#define APR_INT32_MAX 0x7fffffff ++#endif ++ ++#ifdef UINT32_MAX ++#define APR_UINT32_MAX UINT32_MAX ++#else ++#define APR_UINT32_MAX (0xffffffffU) ++#endif ++ ++#ifdef INT64_MIN ++#define APR_INT64_MIN INT64_MIN ++#else ++#define APR_INT64_MIN (APR_INT64_C(-0x7fffffffffffffff) - 1) ++#endif ++ ++#ifdef INT64_MAX ++#define APR_INT64_MAX INT64_MAX ++#else ++#define APR_INT64_MAX APR_INT64_C(0x7fffffffffffffff) ++#endif ++ ++#ifdef UINT64_MAX ++#define APR_UINT64_MAX UINT64_MAX ++#else ++#define APR_UINT64_MAX APR_UINT64_C(0xffffffffffffffff) ++#endif ++ ++#define APR_SIZE_MAX (~((apr_size_t)0)) ++ ++ ++/* Definitions that APR programs need to work properly. */ ++ ++/** ++ * APR public API wrap for C++ compilers. ++ */ ++#ifdef __cplusplus ++#define APR_BEGIN_DECLS extern "C" { ++#define APR_END_DECLS } ++#else ++#define APR_BEGIN_DECLS ++#define APR_END_DECLS ++#endif ++ ++/** ++ * Thread callbacks from APR functions must be declared with APR_THREAD_FUNC, ++ * so that they follow the platform's calling convention. ++ * 
++ *
++ * void* APR_THREAD_FUNC my_thread_entry_fn(apr_thread_t *thd, void *data);
++ *
++ *
++ */ ++#define APR_THREAD_FUNC ++ ++#if defined(DOXYGEN) || !defined(WIN32) ++ ++/** ++ * The public APR functions are declared with APR_DECLARE(), so they may ++ * use the most appropriate calling convention. Public APR functions with ++ * variable arguments must use APR_DECLARE_NONSTD(). ++ * ++ * @remark Both the declaration and implementations must use the same macro. ++ * ++ * 
++ * APR_DECLARE(rettype) apr_func(args)
++ *
++ * @see APR_DECLARE_NONSTD @see APR_DECLARE_DATA ++ * @remark Note that when APR compiles the library itself, it passes the ++ * symbol -DAPR_DECLARE_EXPORT to the compiler on some platforms (e.g. Win32) ++ * to export public symbols from the dynamic library build.\n ++ * The user must define the APR_DECLARE_STATIC when compiling to target ++ * the static APR library on some platforms (e.g. Win32.) The public symbols ++ * are neither exported nor imported when APR_DECLARE_STATIC is defined.\n ++ * By default, compiling an application and including the APR public ++ * headers, without defining APR_DECLARE_STATIC, will prepare the code to be ++ * linked to the dynamic library. ++ */ ++#define APR_DECLARE(type) type ++ ++/** ++ * The public APR functions using variable arguments are declared with ++ * APR_DECLARE_NONSTD(), as they must follow the C language calling convention. ++ * @see APR_DECLARE @see APR_DECLARE_DATA ++ * @remark Both the declaration and implementations must use the same macro. ++ * 
++ *
++ * APR_DECLARE_NONSTD(rettype) apr_func(args, ...);
++ *
++ *
++ */ ++#define APR_DECLARE_NONSTD(type) type ++ ++/** ++ * The public APR variables are declared with AP_MODULE_DECLARE_DATA. ++ * This assures the appropriate indirection is invoked at compile time. ++ * @see APR_DECLARE @see APR_DECLARE_NONSTD ++ * @remark Note that the declaration and implementations use different forms, ++ * but both must include the macro. ++ * ++ * 
++ *
++ * extern APR_DECLARE_DATA type apr_variable;\n
++ * APR_DECLARE_DATA type apr_variable = value;
++ *
++ *
++ */ ++#define APR_DECLARE_DATA ++ ++#elif defined(APR_DECLARE_STATIC) ++#define APR_DECLARE(type) type __stdcall ++#define APR_DECLARE_NONSTD(type) type __cdecl ++#define APR_DECLARE_DATA ++#elif defined(APR_DECLARE_EXPORT) ++#define APR_DECLARE(type) __declspec(dllexport) type __stdcall ++#define APR_DECLARE_NONSTD(type) __declspec(dllexport) type __cdecl ++#define APR_DECLARE_DATA __declspec(dllexport) ++#else ++#define APR_DECLARE(type) __declspec(dllimport) type __stdcall ++#define APR_DECLARE_NONSTD(type) __declspec(dllimport) type __cdecl ++#define APR_DECLARE_DATA __declspec(dllimport) ++#endif ++ ++/* Define APR_SSIZE_T_FMT. ++ * If ssize_t is an integer we define it to be "d", ++ * if ssize_t is a long int we define it to be "ld", ++ * if ssize_t is neither we declare an error here. ++ * I looked for a better way to define this here, but couldn't find one, so ++ * to find the logic for this definition search for "ssize_t_fmt" in ++ * configure.in. ++ */ ++ ++#define APR_SSIZE_T_FMT "ld" ++ ++/* And APR_SIZE_T_FMT */ ++#define APR_SIZE_T_FMT "lu" ++ ++/* And APR_OFF_T_FMT */ ++#define APR_OFF_T_FMT "ld" ++ ++/* And APR_PID_T_FMT */ ++#define APR_PID_T_FMT "d" ++ ++/* And APR_INT64_T_FMT */ ++#define APR_INT64_T_FMT "ld" ++ ++/* And APR_UINT64_T_FMT */ ++#define APR_UINT64_T_FMT "lu" ++ ++/* And APR_UINT64_T_HEX_FMT */ ++#define APR_UINT64_T_HEX_FMT "lx" ++ ++/* ++ * Ensure we work with universal binaries on Darwin ++ */ ++#ifdef DARWIN_10 ++ ++#undef APR_HAS_LARGE_FILES ++#undef APR_SIZEOF_VOIDP ++#undef APR_INT64_T_FMT ++#undef APR_UINT64_T_FMT ++#undef APR_UINT64_T_HEX_FMT ++ ++#ifdef __LP64__ ++ #define APR_HAS_LARGE_FILES 0 ++ #define APR_SIZEOF_VOIDP 8 ++ #define APR_INT64_T_FMT "ld" ++ #define APR_UINT64_T_FMT "lu" ++ #define APR_UINT64_T_HEX_FMT "lx" ++#else ++ #define APR_HAS_LARGE_FILES 1 ++ #define APR_SIZEOF_VOIDP 4 ++ #define APR_INT64_T_FMT "lld" ++ #define APR_UINT64_T_FMT "llu" ++ #define APR_UINT64_T_HEX_FMT "llx" ++#endif ++ ++#undef APR_IS_BIGENDIAN ++#ifdef __BIG_ENDIAN__ ++ #define APR_IS_BIGENDIAN 1 ++#else ++ #define APR_IS_BIGENDIAN 0 ++#endif ++ ++#undef APR_OFF_T_FMT ++#define APR_OFF_T_FMT "lld" ++ ++#endif /* DARWIN_10 */ ++ ++/* Does the proc mutex lock threads too */ ++#define APR_PROC_MUTEX_IS_GLOBAL 0 ++ ++/* Local machine definition for console and log output. */ ++#define APR_EOL_STR "\n" ++ ++#if APR_HAVE_SYS_WAIT_H ++#ifdef WEXITSTATUS ++#define apr_wait_t int ++#else ++#define apr_wait_t union wait ++#define WEXITSTATUS(status) (int)((status).w_retcode) ++#define WTERMSIG(status) (int)((status).w_termsig) ++#endif /* !WEXITSTATUS */ ++#elif defined(__MINGW32__) ++typedef int apr_wait_t; ++#endif /* HAVE_SYS_WAIT_H */ ++ ++#if defined(PATH_MAX) ++#define APR_PATH_MAX PATH_MAX ++#elif defined(_POSIX_PATH_MAX) ++#define APR_PATH_MAX _POSIX_PATH_MAX ++#else ++#error no decision has been made on APR_PATH_MAX for your platform ++#endif ++ ++#ifdef __APPLE__ ++#define APR_DSOPATH "DYLD_LIBRARY_PATH" ++#else ++#define APR_DSOPATH "LD_LIBRARY_PATH" ++#endif ++ ++/** @} */ ++ ++/* Definitions that only Win32 programs need to compile properly. */ ++ ++/* XXX These simply don't belong here, perhaps in apr_portable.h ++ * based on some APR_HAVE_PID/GID/UID? ++ */ ++#ifdef __MINGW32__ ++#ifndef __GNUC__ ++typedef int pid_t; ++#endif ++typedef int uid_t; ++typedef int gid_t; ++#endif ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* APR_H */ +diff -urN libapr1/include/arch/unix/apr_private.h libapr/include/arch/unix/apr_private.h +--- libapr1/include/arch/unix/apr_private.h 1970-01-01 00:00:00.000000000 +0000 ++++ libapr/include/arch/unix/apr_private.h 2019-02-09 23:46:52.000000000 +0000 +@@ -0,0 +1,1158 @@ ++/* include/arch/unix/apr_private.h. Generated from apr_private.h.in by configure. */ ++/* include/arch/unix/apr_private.h.in. Generated from configure.in by autoheader. */ ++ ++ ++#ifndef APR_PRIVATE_H ++#define APR_PRIVATE_H ++ ++ ++/* Define if building universal (internal helper macro) */ ++/* #undef AC_APPLE_UNIVERSAL_BUILD */ ++ ++/* Define if apr_allocator should use guard pages */ ++/* #undef APR_ALLOCATOR_GUARD_PAGES */ ++ ++/* Define if apr_allocator should use mmap */ ++/* #undef APR_ALLOCATOR_USES_MMAP */ ++ ++/* Define as function which can be used for conversion of strings to ++ apr_int64_t */ ++#define APR_INT64_STRFN strtol ++ ++/* Define as function used for conversion of strings to apr_off_t */ ++#define APR_OFF_T_STRFN strtol ++ ++/* Define if pool functions should abort if concurrent usage is detected */ ++/* #undef APR_POOL_CONCURRENCY_CHECK */ ++ ++/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP ++ systems. This function is required for `alloca.c' support on those systems. ++ */ ++/* #undef CRAY_STACKSEG_END */ ++ ++/* Define to 1 if using `alloca.c'. */ ++/* #undef C_ALLOCA */ ++ ++/* Define to path of random device */ ++#define DEV_RANDOM "/dev/urandom" ++ ++/* Define if struct dirent has an inode member */ ++#define DIRENT_INODE d_fileno ++ ++/* Define if struct dirent has a d_type member */ ++#define DIRENT_TYPE d_type ++ ++/* Define if DSO support uses dlfcn.h */ ++#ifdef __APPLE__ ++#define DSO_USE_DLFCN 1 ++#else ++/* #undef DSO_USE_DLFCN */ ++#endif ++ ++/* Define if DSO support uses dyld.h */ ++/* #undef DSO_USE_DYLD */ ++ ++/* Define if DSO support uses shl_load */ ++/* #undef DSO_USE_SHL */ ++ ++/* Define to list of paths to EGD sockets */ ++/* #undef EGD_DEFAULT_SOCKET */ ++ ++/* Define if fcntl locks affect threads within the process */ ++/* #undef FCNTL_IS_GLOBAL */ ++ ++/* Define if fcntl returns EACCES when F_SETLK is already held */ ++/* #undef FCNTL_TRYACQUIRE_EACCES */ ++ ++/* Define if flock locks affect threads within the process */ ++/* #undef FLOCK_IS_GLOBAL */ ++ ++/* Define if gethostbyaddr is thread safe */ ++/* #undef GETHOSTBYADDR_IS_THREAD_SAFE */ ++ ++/* Define if gethostbyname is thread safe */ ++/* #undef GETHOSTBYNAME_IS_THREAD_SAFE */ ++ ++/* Define if gethostbyname_r has the glibc style */ ++#ifdef __APPLE__ ++/* #undef GETHOSTBYNAME_R_GLIBC2 */ ++#else ++#define GETHOSTBYNAME_R_GLIBC2 1 ++#endif ++ ++/* Define if gethostbyname_r has the hostent_data for the third argument */ ++/* #undef GETHOSTBYNAME_R_HOSTENT_DATA */ ++ ++/* Define if getservbyname is thread safe */ ++/* #undef GETSERVBYNAME_IS_THREAD_SAFE */ ++ ++/* Define if getservbyname_r has the glibc style */ ++#ifdef __APPLE__ ++/* #undef GETSERVBYNAME_R_GLIBC2 */ ++#else ++#define GETSERVBYNAME_R_GLIBC2 1 ++#endif ++ ++/* Define if getservbyname_r has the OSF/1 style */ ++/* #undef GETSERVBYNAME_R_OSF1 */ ++ ++/* Define if getservbyname_r has the Solaris style */ ++/* #undef GETSERVBYNAME_R_SOLARIS */ ++ ++/* Define if accept4 function is supported */ ++#ifdef __APPLE__ ++/* #undef HAVE_ACCEPT4 */ ++#else ++#define HAVE_ACCEPT4 1 ++#endif ++ ++/* Define if async i/o supports message q's */ ++/* #undef HAVE_AIO_MSGQ */ ++ ++/* Define to 1 if you have `alloca', as a function or macro. */ ++#define HAVE_ALLOCA 1 ++ ++/* Define to 1 if you have and it should be used (not on Ultrix). ++ */ ++#define HAVE_ALLOCA_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_ARPA_INET_H 1 ++ ++/* Define if compiler provides atomic builtins */ ++#define HAVE_ATOMIC_BUILTINS 1 ++ ++/* Define if BONE_VERSION is defined in sys/socket.h */ ++/* #undef HAVE_BONE_VERSION */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_BYTEORDER_H */ ++ ++/* Define to 1 if you have the `calloc' function. */ ++#define HAVE_CALLOC 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_CONIO_H */ ++ ++/* Define to 1 if you have the `create_area' function. */ ++/* #undef HAVE_CREATE_AREA */ ++ ++/* Define to 1 if you have the `create_sem' function. */ ++/* #undef HAVE_CREATE_SEM */ ++ ++/* Define to 1 if you have the header file. */ ++#ifdef __APPLE__ ++/* #undef HAVE_CRYPT_H */ ++#else ++#define HAVE_CRYPT_H 1 ++#endif ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_CTYPE_H 1 ++ ++/* Define to 1 if you have the declaration of `sys_siglist', and to 0 if you ++ don't. */ ++#define HAVE_DECL_SYS_SIGLIST 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_DIRENT_H 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_DIR_H */ ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_DLFCN_H 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_DL_H */ ++ ++/* Define if dup3 function is supported */ ++#ifdef __APPLE__ ++/* #undef HAVE_DUP3 */ ++#else ++#define HAVE_DUP3 1 ++#endif ++ ++/* Define if EGD is supported */ ++/* #undef HAVE_EGD */ ++ ++/* Define if the epoll interface is supported */ ++#ifdef __APPLE__ ++/* #undef HAVE_EPOLL */ ++#else ++#define HAVE_EPOLL 1 ++#endif ++ ++/* Define if epoll_create1 function is supported */ ++#ifdef __APPLE__ ++/* #undef HAVE_EPOLL_CREATE1 */ ++#else ++#define HAVE_EPOLL_CREATE1 1 ++#endif ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_ERRNO_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_FCNTL_H 1 ++ ++/* Define to 1 if you have the `fdatasync' function. */ ++#ifdef __APPLE__ ++/* #undef HAVE_FDATASYNC */ ++#else ++#define HAVE_FDATASYNC 1 ++#endif ++ ++/* Define to 1 if you have the `flock' function. */ ++#define HAVE_FLOCK 1 ++ ++/* Define to 1 if you have the `fork' function. */ ++#define HAVE_FORK 1 ++ ++/* Define if F_SETLK is defined in fcntl.h */ ++#define HAVE_F_SETLK 1 ++ ++/* Define if getaddrinfo accepts the AI_ADDRCONFIG flag */ ++#define HAVE_GAI_ADDRCONFIG 1 ++ ++/* Define to 1 if you have the `gai_strerror' function. */ ++#define HAVE_GAI_STRERROR 1 ++ ++/* Define if getaddrinfo exists and works well enough for APR */ ++#define HAVE_GETADDRINFO 1 ++ ++/* Define to 1 if you have the `getenv' function. */ ++#define HAVE_GETENV 1 ++ ++/* Define to 1 if you have the `getgrgid_r' function. */ ++#define HAVE_GETGRGID_R 1 ++ ++/* Define to 1 if you have the `getgrnam_r' function. */ ++#define HAVE_GETGRNAM_R 1 ++ ++/* Define to 1 if you have the `gethostbyaddr_r' function. */ ++#ifdef __APPLE__ ++/* #undef HAVE_GETHOSTBYADDR_R */ ++#else ++#define HAVE_GETHOSTBYADDR_R 1 ++#endif ++ ++/* Define to 1 if you have the `gethostbyname_r' function. */ ++#ifdef __APPLE__ ++/* #undef HAVE_GETHOSTBYNAME_R */ ++#else ++#define HAVE_GETHOSTBYNAME_R 1 ++#endif ++ ++/* Define to 1 if you have the `getifaddrs' function. */ ++#define HAVE_GETIFADDRS 1 ++ ++/* Define if getnameinfo exists */ ++#define HAVE_GETNAMEINFO 1 ++ ++/* Define to 1 if you have the `getpass' function. */ ++#define HAVE_GETPASS 1 ++ ++/* Define to 1 if you have the `getpassphrase' function. */ ++/* #undef HAVE_GETPASSPHRASE */ ++ ++/* Define to 1 if you have the `getpwnam_r' function. */ ++#define HAVE_GETPWNAM_R 1 ++ ++/* Define to 1 if you have the `getpwuid_r' function. */ ++#define HAVE_GETPWUID_R 1 ++ ++/* Define to 1 if you have the `getrlimit' function. */ ++#define HAVE_GETRLIMIT 1 ++ ++/* Define to 1 if you have the `getservbyname_r' function. */ ++#ifdef __APPLE__ ++/* #undef HAVE_GETSERVBYNAME_R */ ++#else ++#define HAVE_GETSERVBYNAME_R 1 ++#endif ++ ++/* Define to 1 if you have the `gmtime_r' function. */ ++#define HAVE_GMTIME_R 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_GRP_H 1 ++ ++/* Define if hstrerror is present */ ++/* #undef HAVE_HSTRERROR */ ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_INTTYPES_H 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_IO_H */ ++ ++/* Define to 1 if you have the `isinf' function. */ ++#define HAVE_ISINF 1 ++ ++/* Define to 1 if you have the `isnan' function. */ ++#define HAVE_ISNAN 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_KERNEL_OS_H */ ++ ++/* Define to 1 if you have the `kqueue' function. */ ++#ifdef __APPLE__ ++#define HAVE_KQUEUE 1 ++#else ++/* #undef HAVE_KQUEUE */ ++#endif ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_LANGINFO_H 1 ++ ++/* Define to 1 if you have the `bsd' library (-lbsd). */ ++/* #undef HAVE_LIBBSD */ ++ ++/* Define to 1 if you have the `sendfile' library (-lsendfile). */ ++/* #undef HAVE_LIBSENDFILE */ ++ ++/* Define to 1 if you have the `truerand' library (-ltruerand). */ ++/* #undef HAVE_LIBTRUERAND */ ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_LIMITS_H 1 ++ ++/* Define to 1 if you have the `localtime_r' function. */ ++#define HAVE_LOCALTIME_R 1 ++ ++/* Define if LOCK_EX is defined in sys/file.h */ ++#define HAVE_LOCK_EX 1 ++ ++/* Define to 1 if you have the header file. */ ++#ifdef __APPLE__ ++#define HAVE_MACH_O_DYLD_H 1 ++#else ++/* #undef HAVE_MACH_O_DYLD_H */ ++#endif ++ ++/* Define to 1 if you have the header file. */ ++#ifdef __APPLE__ ++/* #undef HAVE_MALLOC_H */ ++#else ++#define HAVE_MALLOC_H 1 ++#endif ++ ++/* Define if MAP_ANON is defined in sys/mman.h */ ++#define HAVE_MAP_ANON 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_MEMCHECK_H */ ++ ++/* Define to 1 if you have the `memchr' function. */ ++#define HAVE_MEMCHR 1 ++ ++/* Define to 1 if you have the `memmove' function. */ ++#define HAVE_MEMMOVE 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_MEMORY_H 1 ++ ++/* Define to 1 if you have the `mkstemp' function. */ ++#define HAVE_MKSTEMP 1 ++ ++/* Define to 1 if you have the `mkstemp64' function. */ ++/* #undef HAVE_MKSTEMP64 */ ++ ++/* Define to 1 if you have the `mmap' function. */ ++#define HAVE_MMAP 1 ++ ++/* Define to 1 if you have the `mmap64' function. */ ++/* #undef HAVE_MMAP64 */ ++ ++/* Define to 1 if you have the `mprotect' function. */ ++#define HAVE_MPROTECT 1 ++ ++/* Define to 1 if you have the `munmap' function. */ ++#define HAVE_MUNMAP 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_NETDB_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_NETINET_IN_H 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_NETINET_SCTP_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_NETINET_SCTP_UIO_H */ ++ ++/* Defined if netinet/tcp.h is present */ ++#define HAVE_NETINET_TCP_H 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_NET_ERRNO_H */ ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_NET_IF_H 1 ++ ++/* Define to 1 if you have the `nl_langinfo' function. */ ++#define HAVE_NL_LANGINFO 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_OS2_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_OSRELDATE_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_OS_H */ ++ ++/* Define to 1 if you have the `poll' function. */ ++#define HAVE_POLL 1 ++ ++/* Define if POLLIN is defined */ ++#define HAVE_POLLIN 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_POLL_H 1 ++ ++/* Define to 1 if you have the `port_create' function. */ ++/* #undef HAVE_PORT_CREATE */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_PROCESS_H */ ++ ++/* Define to 1 if you have the `pthread_attr_setguardsize' function. */ ++#define HAVE_PTHREAD_ATTR_SETGUARDSIZE 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_PTHREAD_H 1 ++ ++/* Define to 1 if you have the `pthread_key_delete' function. */ ++#define HAVE_PTHREAD_KEY_DELETE 1 ++ ++/* Define to 1 if you have the `pthread_mutexattr_setpshared' function. */ ++#define HAVE_PTHREAD_MUTEXATTR_SETPSHARED 1 ++ ++/* Define if recursive pthread mutexes are available */ ++#define HAVE_PTHREAD_MUTEX_RECURSIVE 1 ++ ++/* Define if cross-process robust mutexes are available */ ++#ifdef __APPLE__ ++/* #undef HAVE_PTHREAD_MUTEX_ROBUST */ ++#else ++#define HAVE_PTHREAD_MUTEX_ROBUST 1 ++#endif ++ ++/* Define if PTHREAD_PROCESS_SHARED is defined in pthread.h */ ++#define HAVE_PTHREAD_PROCESS_SHARED 1 ++ ++/* Define if pthread rwlocks are available */ ++#define HAVE_PTHREAD_RWLOCKS 1 ++ ++/* Define to 1 if you have the `pthread_rwlock_init' function. */ ++#define HAVE_PTHREAD_RWLOCK_INIT 1 ++ ++/* Define to 1 if you have the `pthread_yield' function. */ ++#ifdef __APPLE__ ++/* #undef HAVE_PTHREAD_YIELD */ ++#else ++#define HAVE_PTHREAD_YIELD 1 ++#endif ++ ++/* Define to 1 if you have the `putenv' function. */ ++#define HAVE_PUTENV 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_PWD_H 1 ++ ++/* Define to 1 if you have the `readdir64_r' function. */ ++/* #undef HAVE_READDIR64_R */ ++ ++/* Define to 1 if you have the header file. */ ++#ifdef __APPLE__ ++#define HAVE_SCHED_H 1 ++#else ++/* #undef HAVE_SCHED_H */ ++#endif ++ ++/* Define to 1 if you have the `sched_yield' function. */ ++#ifdef __APPLE__ ++#define HAVE_SCHED_YIELD 1 ++#else ++/* #undef HAVE_SCHED_YIELD */ ++#endif ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SEMAPHORE_H 1 ++ ++/* Define to 1 if you have the `semctl' function. */ ++#define HAVE_SEMCTL 1 ++ ++/* Define to 1 if you have the `semget' function. */ ++#define HAVE_SEMGET 1 ++ ++/* Define to 1 if you have the `sem_close' function. */ ++#define HAVE_SEM_CLOSE 1 ++ ++/* Define to 1 if you have the `sem_post' function. */ ++#define HAVE_SEM_POST 1 ++ ++/* Define if SEM_UNDO is defined in sys/sem.h */ ++#define HAVE_SEM_UNDO 1 ++ ++/* Define to 1 if you have the `sem_unlink' function. */ ++#define HAVE_SEM_UNLINK 1 ++ ++/* Define to 1 if you have the `sem_wait' function. */ ++#define HAVE_SEM_WAIT 1 ++ ++/* Define to 1 if you have the `sendfile' function. */ ++#define HAVE_SENDFILE 1 ++ ++/* Define to 1 if you have the `sendfile64' function. */ ++/* #undef HAVE_SENDFILE64 */ ++ ++/* Define to 1 if you have the `sendfilev' function. */ ++/* #undef HAVE_SENDFILEV */ ++ ++/* Define to 1 if you have the `sendfilev64' function. */ ++/* #undef HAVE_SENDFILEV64 */ ++ ++/* Define to 1 if you have the `send_file' function. */ ++/* #undef HAVE_SEND_FILE */ ++ ++/* Define to 1 if you have the `setenv' function. */ ++#define HAVE_SETENV 1 ++ ++/* Define to 1 if you have the `setrlimit' function. */ ++#define HAVE_SETRLIMIT 1 ++ ++/* Define to 1 if you have the `setsid' function. */ ++#define HAVE_SETSID 1 ++ ++/* Define to 1 if you have the `set_h_errno' function. */ ++/* #undef HAVE_SET_H_ERRNO */ ++ ++/* Define to 1 if you have the `shmat' function. */ ++#define HAVE_SHMAT 1 ++ ++/* Define to 1 if you have the `shmctl' function. */ ++#define HAVE_SHMCTL 1 ++ ++/* Define to 1 if you have the `shmdt' function. */ ++#define HAVE_SHMDT 1 ++ ++/* Define to 1 if you have the `shmget' function. */ ++#define HAVE_SHMGET 1 ++ ++/* Define to 1 if you have the `shm_open' function. */ ++#define HAVE_SHM_OPEN 1 ++ ++/* Define to 1 if you have the `shm_unlink' function. */ ++#define HAVE_SHM_UNLINK 1 ++ ++/* Define to 1 if you have the `sigaction' function. */ ++#define HAVE_SIGACTION 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SIGNAL_H 1 ++ ++/* Define to 1 if you have the `sigsuspend' function. */ ++#define HAVE_SIGSUSPEND 1 ++ ++/* Define to 1 if you have the `sigwait' function. */ ++#define HAVE_SIGWAIT 1 ++ ++/* Whether you have socklen_t */ ++#define HAVE_SOCKLEN_T 1 ++ ++/* Define if the SOCK_CLOEXEC flag is supported */ ++#ifdef __APPLE__ ++/* #undef HAVE_SOCK_CLOEXEC */ ++#else ++#define HAVE_SOCK_CLOEXEC 1 ++#endif ++ ++/* Define if SO_ACCEPTFILTER is defined in sys/socket.h */ ++/* #undef HAVE_SO_ACCEPTFILTER */ ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_STDARG_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_STDDEF_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_STDINT_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_STDIO_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_STDLIB_H 1 ++ ++/* Define to 1 if you have the `strcasecmp' function. */ ++#define HAVE_STRCASECMP 1 ++ ++/* Define to 1 if you have the `strdup' function. */ ++#define HAVE_STRDUP 1 ++ ++/* Define to 1 if you have the `strerror_r' function. */ ++#define HAVE_STRERROR_R 1 ++ ++/* Define to 1 if you have the `stricmp' function. */ ++/* #undef HAVE_STRICMP */ ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_STRINGS_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_STRING_H 1 ++ ++/* Define to 1 if you have the `strncasecmp' function. */ ++#define HAVE_STRNCASECMP 1 ++ ++/* Define to 1 if you have the `strnicmp' function. */ ++/* #undef HAVE_STRNICMP */ ++ ++/* Define to 1 if you have the `strstr' function. */ ++#define HAVE_STRSTR 1 ++ ++/* Define if struct impreq was found */ ++#define HAVE_STRUCT_IPMREQ 1 ++ ++/* Define to 1 if `st_atimensec' is a member of `struct stat'. */ ++/* #undef HAVE_STRUCT_STAT_ST_ATIMENSEC */ ++ ++/* Define to 1 if `st_atime_n' is a member of `struct stat'. */ ++/* #undef HAVE_STRUCT_STAT_ST_ATIME_N */ ++ ++/* Define to 1 if `st_atim.tv_nsec' is a member of `struct stat'. */ ++#ifdef __APPLE__ ++/* #undef HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC */ ++#else ++#define HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC 1 ++#endif ++ ++/* Define to 1 if `st_blocks' is a member of `struct stat'. */ ++#define HAVE_STRUCT_STAT_ST_BLOCKS 1 ++ ++/* Define to 1 if `st_ctimensec' is a member of `struct stat'. */ ++/* #undef HAVE_STRUCT_STAT_ST_CTIMENSEC */ ++ ++/* Define to 1 if `st_ctime_n' is a member of `struct stat'. */ ++/* #undef HAVE_STRUCT_STAT_ST_CTIME_N */ ++ ++/* Define to 1 if `st_ctim.tv_nsec' is a member of `struct stat'. */ ++#ifdef __APPLE__ ++/* #undef HAVE_STRUCT_STAT_ST_CTIM_TV_NSEC */ ++#else ++#define HAVE_STRUCT_STAT_ST_CTIM_TV_NSEC 1 ++#endif ++ ++/* Define to 1 if `st_mtimensec' is a member of `struct stat'. */ ++/* #undef HAVE_STRUCT_STAT_ST_MTIMENSEC */ ++ ++/* Define to 1 if `st_mtime_n' is a member of `struct stat'. */ ++/* #undef HAVE_STRUCT_STAT_ST_MTIME_N */ ++ ++/* Define to 1 if `st_mtim.tv_nsec' is a member of `struct stat'. */ ++#ifdef __APPLE__ ++/* #undef HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC */ ++#else ++#define HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC 1 ++#endif ++ ++/* Define to 1 if `tm_gmtoff' is a member of `struct tm'. */ ++#define HAVE_STRUCT_TM_TM_GMTOFF 1 ++ ++/* Define to 1 if `__tm_gmtoff' is a member of `struct tm'. */ ++/* #undef HAVE_STRUCT_TM___TM_GMTOFF */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_SYSAPI_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_SYSGTIME_H */ ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_FILE_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_IOCTL_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_IPC_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_MMAN_H 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_SYS_MUTEX_H */ ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_PARAM_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_POLL_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_RESOURCE_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_SELECT_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_SEM_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#ifdef __APPLE__ ++/* #undef HAVE_SYS_SENDFILE_H */ ++#else ++#define HAVE_SYS_SENDFILE_H 1 ++#endif ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_SHM_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_SIGNAL_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_SOCKET_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#ifdef __APPLE__ ++#define HAVE_SYS_SOCKIO_H 1 ++#else ++/* #undef HAVE_SYS_SOCKIO_H */ ++#endif ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_STAT_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_SYSCTL_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#ifdef __APPLE__ ++#define HAVE_SYS_SYSLIMITS_H 1 ++#else ++/* #undef HAVE_SYS_SYSLIMITS_H */ ++#endif ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_TIME_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_TYPES_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_UIO_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_UN_H 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_SYS_UUID_H */ ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_WAIT_H 1 ++ ++/* Define if TCP_CORK is defined in netinet/tcp.h */ ++#ifdef __APPLE__ ++/* #undef HAVE_TCP_CORK */ ++#else ++#define HAVE_TCP_CORK 1 ++#endif ++ ++/* Define if TCP_NODELAY and TCP_CORK can be enabled at the same time */ ++#ifdef __APPLE__ ++/* #undef HAVE_TCP_NODELAY_WITH_CORK */ ++#else ++#define HAVE_TCP_NODELAY_WITH_CORK 1 ++#endif ++ ++/* Define if TCP_NOPUSH is defined in netinet/tcp.h */ ++#ifdef __APPLE__ ++#define HAVE_TCP_NOPUSH 1 ++#else ++/* #undef HAVE_TCP_NOPUSH */ ++#endif ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_TERMIOS_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_TIME_H 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_TPFEQ_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_TPFIO_H */ ++ ++/* Define if truerand is supported */ ++/* #undef HAVE_TRUERAND */ ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_UNISTD_H 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_UNIX_H */ ++ ++/* Define to 1 if you have the `unsetenv' function. */ ++#define HAVE_UNSETENV 1 ++ ++/* Define to 1 if you have the `utime' function. */ ++#define HAVE_UTIME 1 ++ ++/* Define to 1 if you have the `utimes' function. */ ++#define HAVE_UTIMES 1 ++ ++/* Define to 1 if you have the `uuid_create' function. */ ++/* #undef HAVE_UUID_CREATE */ ++ ++/* Define to 1 if you have the `uuid_generate' function. */ ++#ifdef __APPLE__ ++#define HAVE_UUID_GENERATE 1 ++#else ++/* #undef HAVE_UUID_GENERATE */ ++#endif ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_UUID_H */ ++ ++/* Define to 1 if you have the header file. */ ++#ifdef __APPLE__ ++#define HAVE_UUID_UUID_H 1 ++#else ++/* #undef HAVE_UUID_UUID_H */ ++#endif ++ ++/* Compile in valgrind support */ ++/* #undef HAVE_VALGRIND */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_VALGRIND_H */ ++ ++/* Define if C compiler supports VLA */ ++#define HAVE_VLA 1 ++ ++/* Define to 1 if you have the `waitpid' function. */ ++#define HAVE_WAITPID 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_WINDOWS_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_WINSOCK2_H */ ++ ++/* Define to 1 if you have the `writev' function. */ ++#define HAVE_WRITEV 1 ++ ++/* Define for z/OS pthread API nuances */ ++/* #undef HAVE_ZOS_PTHREADS */ ++ ++/* Define to the sub-directory in which libtool stores uninstalled libraries. ++ */ ++#define LT_OBJDIR ".libs/" ++ ++/* Define if EAI_ error codes from getaddrinfo are negative */ ++#ifdef __APPLE__ ++/* #undef NEGATIVE_EAI */ ++#else ++#define NEGATIVE_EAI 1 ++#endif ++ ++/* Define to the address where bug reports for this package should be sent. */ ++#define PACKAGE_BUGREPORT "" ++ ++/* Define to the full name of this package. */ ++#define PACKAGE_NAME "" ++ ++/* Define to the full name and version of this package. */ ++#define PACKAGE_STRING "" ++ ++/* Define to the one symbol short name of this package. */ ++#define PACKAGE_TARNAME "" ++ ++/* Define to the home page for this package. */ ++#define PACKAGE_URL "" ++ ++/* Define to the version of this package. */ ++#define PACKAGE_VERSION "" ++ ++/* Define if POSIX semaphores affect threads within the process */ ++#ifdef __APPLE__ ++#define POSIXSEM_IS_GLOBAL 1 ++#else ++/* #undef POSIXSEM_IS_GLOBAL */ ++#endif ++ ++/* Define on PowerPC 405 where errata 77 applies */ ++/* #undef PPC405_ERRATA */ ++ ++/* Define if pthread_attr_getdetachstate() has one arg */ ++/* #undef PTHREAD_ATTR_GETDETACHSTATE_TAKES_ONE_ARG */ ++ ++/* Define if pthread_getspecific() has two args */ ++/* #undef PTHREAD_GETSPECIFIC_TAKES_TWO_ARGS */ ++ ++/* Define if readdir is thread safe */ ++/* #undef READDIR_IS_THREAD_SAFE */ ++ ++/* Define to 1 if the `setpgrp' function takes no argument. */ ++#define SETPGRP_VOID 1 ++ ++/* */ ++/* #undef SIGWAIT_TAKES_ONE_ARG */ ++ ++/* The size of `char', as computed by sizeof. */ ++#define SIZEOF_CHAR 1 ++ ++/* The size of ino_t */ ++#define SIZEOF_INO_T 8 ++ ++/* The size of `int', as computed by sizeof. */ ++#define SIZEOF_INT 4 ++ ++/* The size of `long', as computed by sizeof. */ ++#define SIZEOF_LONG 8 ++ ++/* The size of `long long', as computed by sizeof. */ ++#define SIZEOF_LONG_LONG 8 ++ ++/* The size of off_t */ ++#define SIZEOF_OFF_T 8 ++ ++/* The size of pid_t */ ++#define SIZEOF_PID_T 4 ++ ++/* The size of `short', as computed by sizeof. */ ++#define SIZEOF_SHORT 2 ++ ++/* The size of size_t */ ++#define SIZEOF_SIZE_T 8 ++ ++/* The size of ssize_t */ ++#define SIZEOF_SSIZE_T 8 ++ ++/* The size of struct iovec */ ++#define SIZEOF_STRUCT_IOVEC 16 ++ ++/* The size of `void*', as computed by sizeof. */ ++#define SIZEOF_VOIDP 8 ++ ++/* If using the C implementation of alloca, define if you know the ++ direction of stack growth for your system; otherwise it will be ++ automatically deduced at runtime. ++ STACK_DIRECTION > 0 => grows toward higher addresses ++ STACK_DIRECTION < 0 => grows toward lower addresses ++ STACK_DIRECTION = 0 => direction of growth unknown */ ++/* #undef STACK_DIRECTION */ ++ ++/* Define to 1 if you have the ANSI C header files. */ ++#define STDC_HEADERS 1 ++ ++/* Define if strerror returns int */ ++#ifdef __APPLE__ ++#define STRERROR_R_RC_INT 1 ++#else ++/* #undef STRERROR_R_RC_INT */ ++#endif ++ ++/* Define if SysV semaphores affect threads within the process */ ++/* #undef SYSVSEM_IS_GLOBAL */ ++ ++/* Define if use of generic atomics is requested */ ++/* #undef USE_ATOMICS_GENERIC */ ++ ++/* Define if BeOS Semaphores will be used */ ++/* #undef USE_BEOSSEM */ ++ ++/* Define if SVR4-style fcntl() will be used */ ++/* #undef USE_FCNTL_SERIALIZE */ ++ ++/* Define if 4.2BSD-style flock() will be used */ ++/* #undef USE_FLOCK_SERIALIZE */ ++ ++/* Define if BeOS areas will be used */ ++/* #undef USE_SHMEM_BEOS */ ++ ++/* Define if BeOS areas will be used */ ++/* #undef USE_SHMEM_BEOS_ANON */ ++ ++/* Define if 4.4BSD-style mmap() via MAP_ANON will be used */ ++#define USE_SHMEM_MMAP_ANON 1 ++ ++/* Define if mmap() via POSIX.1 shm_open() on temporary file will be used */ ++/* #undef USE_SHMEM_MMAP_SHM */ ++ ++/* Define if Classical mmap() on temporary file will be used */ ++/* #undef USE_SHMEM_MMAP_TMP */ ++ ++/* Define if SVR4-style mmap() on /dev/zero will be used */ ++/* #undef USE_SHMEM_MMAP_ZERO */ ++ ++/* Define if OS/2 DosAllocSharedMem() will be used */ ++/* #undef USE_SHMEM_OS2 */ ++ ++/* Define if OS/2 DosAllocSharedMem() will be used */ ++/* #undef USE_SHMEM_OS2_ANON */ ++ ++/* Define if SysV IPC shmget() will be used */ ++#define USE_SHMEM_SHMGET 1 ++ ++/* Define if SysV IPC shmget() will be used */ ++/* #undef USE_SHMEM_SHMGET_ANON */ ++ ++/* Define if Windows shared memory will be used */ ++/* #undef USE_SHMEM_WIN32 */ ++ ++/* Define if Windows CreateFileMapping() will be used */ ++/* #undef USE_SHMEM_WIN32_ANON */ ++ ++/* Enable extensions on AIX 3, Interix. */ ++#ifndef _ALL_SOURCE ++# define _ALL_SOURCE 1 ++#endif ++/* Enable GNU extensions on systems that have them. */ ++#ifndef _GNU_SOURCE ++# define _GNU_SOURCE 1 ++#endif ++/* Enable threading extensions on Solaris. */ ++#ifndef _POSIX_PTHREAD_SEMANTICS ++# define _POSIX_PTHREAD_SEMANTICS 1 ++#endif ++/* Enable extensions on HP NonStop. */ ++#ifndef _TANDEM_SOURCE ++# define _TANDEM_SOURCE 1 ++#endif ++/* Enable general extensions on Solaris. */ ++#ifndef __EXTENSIONS__ ++# define __EXTENSIONS__ 1 ++#endif ++ ++ ++/* Define if SysV IPC semget() will be used */ ++#define USE_SYSVSEM_SERIALIZE 1 ++ ++/* Define if apr_wait_for_io_or_timeout() uses poll(2) */ ++#define WAITIO_USES_POLL 1 ++ ++/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most ++ significant byte first (like Motorola and SPARC, unlike Intel). */ ++#if defined AC_APPLE_UNIVERSAL_BUILD ++# if defined __BIG_ENDIAN__ ++# define WORDS_BIGENDIAN 1 ++# endif ++#else ++# ifndef WORDS_BIGENDIAN ++/* # undef WORDS_BIGENDIAN */ ++# endif ++#endif ++ ++/* Define to 1 if on MINIX. */ ++/* #undef _MINIX */ ++ ++/* Define to 2 if the system does not provide POSIX.1 features except with ++ this defined. */ ++/* #undef _POSIX_1_SOURCE */ ++ ++/* Define to 1 if you need to in order for `stat' and other things to work. */ ++/* #undef _POSIX_SOURCE */ ++ ++/* Define to empty if `const' does not conform to ANSI C. */ ++/* #undef const */ ++ ++/* Define to `int' if doesn't define. */ ++/* #undef gid_t */ ++ ++/* Define to `__inline__' or `__inline' if that's what the C compiler ++ calls it, or to nothing if 'inline' is not supported under any name. */ ++#ifndef __cplusplus ++/* #undef inline */ ++#endif ++ ++/* Define to `long int' if does not define. */ ++/* #undef off_t */ ++ ++/* Define to `int' if does not define. */ ++/* #undef pid_t */ ++ ++/* Define to `unsigned int' if does not define. */ ++/* #undef size_t */ ++ ++/* Define to `int' if does not define. */ ++/* #undef ssize_t */ ++ ++/* Define to `int' if doesn't define. */ ++/* #undef uid_t */ ++ ++ ++/* switch this on if we have a BeOS version below BONE */ ++#if defined(BEOS) && !defined(HAVE_BONE_VERSION) ++#define BEOS_R5 1 ++#else ++#define BEOS_BONE 1 ++#endif ++ ++/* ++ * Darwin 10's default compiler (gcc42) builds for both 64 and ++ * 32 bit architectures unless specifically told not to. ++ * In those cases, we need to override types depending on how ++ * we're being built at compile time. ++ * NOTE: This is an ugly work-around for Darwin's ++ * concept of universal binaries, a single package ++ * (executable, lib, etc...) which contains both 32 ++ * and 64 bit versions. The issue is that if APR is ++ * built universally, if something else is compiled ++ * against it, some bit sizes will depend on whether ++ * it is 32 or 64 bit. This is determined by the __LP64__ ++ * flag. Since we need to support both, we have to ++ * handle OS X unqiuely. ++ */ ++#ifdef DARWIN_10 ++ ++#define APR_OFF_T_STRFN strtol ++#define APR_INT64_STRFN strtol ++#define SIZEOF_LONG 8 ++#define SIZEOF_SIZE_T 8 ++#define SIZEOF_SSIZE_T 8 ++#define SIZEOF_VOIDP 8 ++#define SIZEOF_STRUCT_IOVEC 16 ++ ++#ifdef __LP64__ ++ #define APR_INT64_STRFN strtol ++ #define SIZEOF_LONG 8 ++ #define SIZEOF_SIZE_T 8 ++ #define SIZEOF_SSIZE_T 8 ++ #define SIZEOF_VOIDP 8 ++ #define SIZEOF_STRUCT_IOVEC 16 ++#else ++ #define APR_INT64_STRFN strtol ++ #define SIZEOF_LONG 8 ++ #define SIZEOF_SIZE_T 8 ++ #define SIZEOF_SSIZE_T 8 ++ #define SIZEOF_VOIDP 8 ++ #define SIZEOF_STRUCT_IOVEC 16 ++#endif ++ ++#define APR_OFF_T_STRFN strtol ++#define APR_OFF_T_STRFN strtol ++ ++ ++#define SETPGRP_VOID 1 ++#ifdef __DARWIN_UNIX03 ++ #define SETPGRP_VOID 1 ++#else ++/* #undef SETPGRP_VOID */ ++#endif ++ ++#endif /* DARWIN_10 */ ++ ++/* ++ * Include common private declarations. ++ */ ++#include "../apr_private_common.h" ++#endif /* APR_PRIVATE_H */ ++ +diff -urN libapr1/include/private/apr_escape_test_char.h libapr/include/private/apr_escape_test_char.h +--- libapr1/include/private/apr_escape_test_char.h 1970-01-01 00:00:00.000000000 +0000 ++++ libapr/include/private/apr_escape_test_char.h 2019-02-09 23:45:36.000000000 +0000 +@@ -0,0 +1,25 @@ ++/* this file is automatically generated by gen_test_char, do not edit. "make include/private/apr_escape_test_char.h" to regenerate. */ ++#define T_ESCAPE_SHELL_CMD (1) ++#define T_ESCAPE_PATH_SEGMENT (2) ++#define T_OS_ESCAPE_PATH (4) ++#define T_ESCAPE_ECHO (8) ++#define T_ESCAPE_URLENCODED (16) ++#define T_ESCAPE_XML (32) ++#define T_ESCAPE_LDAP_DN (64) ++#define T_ESCAPE_LDAP_FILTER (128) ++ ++static const unsigned char test_char_table[256] = { ++ 224,222,222,222,222,222,222,222,222,222,223,222,222,222,222,222,222,222,222,222, ++ 222,222,222,222,222,222,222,222,222,222,222,222,6,16,127,22,17,22,49,17, ++ 145,145,129,80,80,0,0,18,0,0,0,0,0,0,0,0,0,0,16,87, ++ 119,16,119,23,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, ++ 0,0,0,0,0,0,0,0,0,0,0,23,223,23,23,0,23,0,0,0, ++ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, ++ 0,0,0,23,23,23,17,222,222,222,222,222,222,222,222,222,222,222,222,222, ++ 222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222, ++ 222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222, ++ 222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222, ++ 222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222, ++ 222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222, ++ 222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222 ++}; diff --git a/third_party/libaprutil1.BUILD b/third_party/libaprutil1.BUILD new file mode 100644 index 000000000..ecf9c2f46 --- /dev/null +++ b/third_party/libaprutil1.BUILD @@ -0,0 +1,121 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache license (for libapr1) + +cc_library( + name = "libaprutil1", + srcs = glob([ + "include/private/apu_*.h", + "include/private/apr_*.h", + ]) + [ + "buckets/apr_brigade.c", + "buckets/apr_buckets.c", + "buckets/apr_buckets_alloc.c", + "buckets/apr_buckets_eos.c", + "buckets/apr_buckets_file.c", + "buckets/apr_buckets_flush.c", + "buckets/apr_buckets_heap.c", + "buckets/apr_buckets_mmap.c", + "buckets/apr_buckets_pipe.c", + "buckets/apr_buckets_pool.c", + "buckets/apr_buckets_refcount.c", + "buckets/apr_buckets_simple.c", + "buckets/apr_buckets_socket.c", + "crypto/apr_crypto.c", + "crypto/apr_md4.c", + "crypto/apr_md5.c", + "crypto/apr_sha1.c", + "crypto/apr_siphash.c", + "crypto/crypt_blowfish.c", + "crypto/crypt_blowfish.h", + "crypto/getuuid.c", + "crypto/uuid.c", + "dbd/apr_dbd.c", + "dbd/apr_dbd_pgsql.c", + "dbd/apr_dbd_sqlite3.c", + "dbm/apr_dbm.c", + "dbm/apr_dbm_sdbm.c", + "dbm/sdbm/sdbm.c", + "dbm/sdbm/sdbm_hash.c", + "dbm/sdbm/sdbm_lock.c", + "dbm/sdbm/sdbm_pair.c", + "dbm/sdbm/sdbm_pair.h", + "dbm/sdbm/sdbm_private.h", + "dbm/sdbm/sdbm_tune.h", + "encoding/apr_base64.c", + "hooks/apr_hooks.c", + "ldap/apr_ldap_stub.c", + "ldap/apr_ldap_url.c", + "memcache/apr_memcache.c", + "misc/apr_date.c", + "misc/apr_queue.c", + "misc/apr_reslist.c", + "misc/apr_rmm.c", + "misc/apr_thread_pool.c", + "misc/apu_dso.c", + "misc/apu_version.c", + "redis/apr_redis.c", + "strmatch/apr_strmatch.c", + "uri/apr_uri.c", + "xlate/xlate.c", + "xml/apr_xml.c", + ], + hdrs = [ + "include/apr_anylock.h", + "include/apr_base64.h", + "include/apr_buckets.h", + "include/apr_crypto.h", + "include/apr_date.h", + "include/apr_dbd.h", + "include/apr_dbm.h", + "include/apr_hooks.h", + "include/apr_ldap.h", + "include/apr_ldap_init.h", + "include/apr_ldap_option.h", + "include/apr_ldap_rebind.h", + "include/apr_ldap_url.h", + "include/apr_md4.h", + "include/apr_md5.h", + "include/apr_memcache.h", + "include/apr_optional.h", + "include/apr_optional_hooks.h", + "include/apr_queue.h", + "include/apr_redis.h", + "include/apr_reslist.h", + "include/apr_rmm.h", + "include/apr_sdbm.h", + "include/apr_sha1.h", + "include/apr_siphash.h", + "include/apr_strmatch.h", + "include/apr_thread_pool.h", + "include/apr_uri.h", + "include/apr_uuid.h", + "include/apr_xlate.h", + "include/apr_xml.h", + "include/apu.h", + "include/apu_errno.h", + "include/apu_version.h", + "include/apu_want.h", + ], + copts = [ + "-pthread", + ], + defines = [ + "_GNU_SOURCE", + "HAVE_CONFIG_H", + "LINUX", + "REENTRANT", + ], + includes = [ + "include", + "include/private", + ], + linkopts = [ + "-lpthread", + ], + strip_include_prefix = "include", + deps = [ + "@libapr1", + "@libexpat", + ], +) diff --git a/third_party/libaprutil1.patch b/third_party/libaprutil1.patch new file mode 100644 index 000000000..a3a371cbc --- /dev/null +++ b/third_party/libaprutil1.patch @@ -0,0 +1,608 @@ +diff -urN libaprutil1/include/apr_ldap.h libaprutil/include/apr_ldap.h +--- libaprutil1/include/apr_ldap.h 1970-01-01 00:00:00.000000000 +0000 ++++ libaprutil/include/apr_ldap.h 2019-02-09 23:57:54.000000000 +0000 +@@ -0,0 +1,197 @@ ++/* Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++/* ++ * apr_ldap.h is generated from apr_ldap.h.in by configure -- do not edit apr_ldap.h ++ */ ++/** ++ * @file apr_ldap.h ++ * @brief APR-UTIL LDAP ++ */ ++#ifndef APU_LDAP_H ++#define APU_LDAP_H ++ ++/** ++ * @defgroup APR_Util_LDAP LDAP ++ * @ingroup APR_Util ++ * @{ ++ */ ++ ++/* this will be defined if LDAP support was compiled into apr-util */ ++#define APR_HAS_LDAP 0 ++ ++/* identify the LDAP toolkit used */ ++#define APR_HAS_NETSCAPE_LDAPSDK 0 ++#define APR_HAS_SOLARIS_LDAPSDK 0 ++#define APR_HAS_NOVELL_LDAPSDK 0 ++#define APR_HAS_MOZILLA_LDAPSDK 0 ++#define APR_HAS_OPENLDAP_LDAPSDK 0 ++#define APR_HAS_MICROSOFT_LDAPSDK 0 ++#define APR_HAS_TIVOLI_LDAPSDK 0 ++#define APR_HAS_ZOS_LDAPSDK 0 ++#define APR_HAS_OTHER_LDAPSDK 0 ++ ++ ++/* ++ * Handle the case when LDAP is enabled ++ */ ++#if APR_HAS_LDAP ++ ++/* ++ * The following #defines are DEPRECATED and should not be used for ++ * anything. They remain to maintain binary compatibility. ++ * The original code defined the OPENLDAP SDK as present regardless ++ * of what really was there, which was way bogus. In addition, the ++ * apr_ldap_url_parse*() functions have been rewritten specifically for ++ * APR, so the APR_HAS_LDAP_URL_PARSE macro is forced to zero. ++ */ ++#if APR_HAS_TIVOLI_LDAPSDK ++#define APR_HAS_LDAP_SSL 0 ++#else ++#define APR_HAS_LDAP_SSL 1 ++#endif ++#define APR_HAS_LDAP_URL_PARSE 0 ++ ++#if APR_HAS_OPENLDAP_LDAPSDK && !defined(LDAP_DEPRECATED) ++/* Ensure that the "deprecated" interfaces are still exposed ++ * with OpenLDAP >= 2.3; these were exposed by default in earlier ++ * releases. */ ++#define LDAP_DEPRECATED 1 ++#endif ++ ++/* ++ * Include the standard LDAP header files. ++ */ ++ ++ ++ ++ ++ ++ ++/* ++ * Detected standard functions ++ */ ++#define APR_HAS_LDAPSSL_CLIENT_INIT 0 ++#define APR_HAS_LDAPSSL_CLIENT_DEINIT 0 ++#define APR_HAS_LDAPSSL_ADD_TRUSTED_CERT 0 ++#define APR_HAS_LDAP_START_TLS_S 0 ++#define APR_HAS_LDAP_SSLINIT 0 ++#define APR_HAS_LDAPSSL_INIT 0 ++#define APR_HAS_LDAPSSL_INSTALL_ROUTINES 0 ++ ++/* ++ * Make sure the secure LDAP port is defined ++ */ ++#ifndef LDAPS_PORT ++#define LDAPS_PORT 636 /* ldaps:/// default LDAP over TLS port */ ++#endif ++ ++/* ++ * For ldap function calls that input a size limit on the number of returned elements ++ * Some SDKs do not have the define for LDAP_DEFAULT_LIMIT (-1) or LDAP_NO_LIMIT (0) ++ * LDAP_DEFAULT_LIMIT is preferred as it allows inheritance from whatever the SDK ++ * or process is configured for. ++ */ ++#ifdef LDAP_DEFAULT_LIMIT ++#define APR_LDAP_SIZELIMIT LDAP_DEFAULT_LIMIT ++#else ++#ifdef LDAP_NO_LIMIT ++#define APR_LDAP_SIZELIMIT LDAP_NO_LIMIT ++#endif ++#endif ++ ++#ifndef APR_LDAP_SIZELIMIT ++#define APR_LDAP_SIZELIMIT 0 /* equivalent to LDAP_NO_LIMIT, and what goes on the wire */ ++#endif ++ ++/* ++ * z/OS is missing some defines ++ */ ++#ifndef LDAP_VERSION_MAX ++#define LDAP_VERSION_MAX LDAP_VERSION ++#endif ++#if APR_HAS_ZOS_LDAPSDK ++#define LDAP_VENDOR_NAME "IBM z/OS" ++#endif ++ ++/* Note: Macros defining const casting has been removed in APR v1.0, ++ * pending real support for LDAP v2.0 toolkits. ++ * ++ * In the mean time, please use an LDAP v3.0 toolkit. ++ */ ++#if LDAP_VERSION_MAX <= 2 ++#error Support for LDAP v2.0 toolkits has been removed from apr-util. Please use an LDAP v3.0 toolkit. ++#endif ++ ++#ifdef __cplusplus ++extern "C" { ++#endif /* __cplusplus */ ++ ++/** ++ * This structure allows the C LDAP API error codes to be returned ++ * along with plain text error messages that explain to us mere mortals ++ * what really happened. ++ */ ++typedef struct apr_ldap_err_t { ++ const char *reason; ++ const char *msg; ++ int rc; ++} apr_ldap_err_t; ++ ++#ifdef __cplusplus ++} ++#endif ++ ++/* The MS SDK returns LDAP_UNAVAILABLE when the backend has closed the connection ++ * between LDAP calls. Protect with APR_HAS_MICROSOFT_LDAPSDK in case someone ++ * manually chooses another SDK on Windows ++ */ ++#if APR_HAS_MICROSOFT_LDAPSDK ++#define APR_LDAP_IS_SERVER_DOWN(s) ((s) == LDAP_SERVER_DOWN \ ++ || (s) == LDAP_UNAVAILABLE) ++#else ++#define APR_LDAP_IS_SERVER_DOWN(s) ((s) == LDAP_SERVER_DOWN) ++#endif ++ ++/* These symbols are not actually exported in a DSO build, but mapped into ++ * a private exported function array for apr_ldap_stub to bind dynamically. ++ * Rename them appropriately to protect the global namespace. ++ */ ++#ifdef APU_DSO_LDAP_BUILD ++ ++#define apr_ldap_info apr__ldap_info ++#define apr_ldap_init apr__ldap_init ++#define apr_ldap_ssl_init apr__ldap_ssl_init ++#define apr_ldap_ssl_deinit apr__ldap_ssl_deinit ++#define apr_ldap_get_option apr__ldap_get_option ++#define apr_ldap_set_option apr__ldap_set_option ++#define apr_ldap_rebind_init apr__ldap_rebind_init ++#define apr_ldap_rebind_add apr__ldap_rebind_add ++#define apr_ldap_rebind_remove apr__ldap_rebind_remove ++ ++#define APU_DECLARE_LDAP(type) type ++#else ++#define APU_DECLARE_LDAP(type) APU_DECLARE(type) ++#endif ++ ++#include "apr_ldap_url.h" ++#include "apr_ldap_init.h" ++#include "apr_ldap_option.h" ++#include "apr_ldap_rebind.h" ++ ++#endif /* APR_HAS_LDAP */ ++/** @} */ ++#endif /* APU_LDAP_H */ +diff -urN libaprutil1/include/apu.h libaprutil/include/apu.h +--- libaprutil1/include/apu.h 1970-01-01 00:00:00.000000000 +0000 ++++ libaprutil/include/apu.h 2019-02-09 23:58:29.000000000 +0000 +@@ -0,0 +1,128 @@ ++/* Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++/* ++ * apu.h is generated from apu.h.in by configure -- do not edit apu.h ++ */ ++/* @file apu.h ++ * @brief APR-Utility main file ++ */ ++/** ++ * @defgroup APR_Util APR Utility Functions ++ * @{ ++ */ ++ ++ ++#ifndef APU_H ++#define APU_H ++ ++/** ++ * APU_DECLARE_EXPORT is defined when building the APR-UTIL dynamic library, ++ * so that all public symbols are exported. ++ * ++ * APU_DECLARE_STATIC is defined when including the APR-UTIL public headers, ++ * to provide static linkage when the dynamic library may be unavailable. ++ * ++ * APU_DECLARE_STATIC and APU_DECLARE_EXPORT are left undefined when ++ * including the APR-UTIL public headers, to import and link the symbols from ++ * the dynamic APR-UTIL library and assure appropriate indirection and calling ++ * conventions at compile time. ++ */ ++ ++#if defined(DOXYGEN) || !defined(WIN32) ++/** ++ * The public APR-UTIL functions are declared with APU_DECLARE(), so they may ++ * use the most appropriate calling convention. Public APR functions with ++ * variable arguments must use APU_DECLARE_NONSTD(). ++ * ++ * @fn APU_DECLARE(rettype) apr_func(args); ++ */ ++#define APU_DECLARE(type) type ++/** ++ * The public APR-UTIL functions using variable arguments are declared with ++ * APU_DECLARE_NONSTD(), as they must use the C language calling convention. ++ * ++ * @fn APU_DECLARE_NONSTD(rettype) apr_func(args, ...); ++ */ ++#define APU_DECLARE_NONSTD(type) type ++/** ++ * The public APR-UTIL variables are declared with APU_DECLARE_DATA. ++ * This assures the appropriate indirection is invoked at compile time. ++ * ++ * @fn APU_DECLARE_DATA type apr_variable; ++ * @note APU_DECLARE_DATA extern type apr_variable; syntax is required for ++ * declarations within headers to properly import the variable. ++ */ ++#define APU_DECLARE_DATA ++#elif defined(APU_DECLARE_STATIC) ++#define APU_DECLARE(type) type __stdcall ++#define APU_DECLARE_NONSTD(type) type __cdecl ++#define APU_DECLARE_DATA ++#elif defined(APU_DECLARE_EXPORT) ++#define APU_DECLARE(type) __declspec(dllexport) type __stdcall ++#define APU_DECLARE_NONSTD(type) __declspec(dllexport) type __cdecl ++#define APU_DECLARE_DATA __declspec(dllexport) ++#else ++#define APU_DECLARE(type) __declspec(dllimport) type __stdcall ++#define APU_DECLARE_NONSTD(type) __declspec(dllimport) type __cdecl ++#define APU_DECLARE_DATA __declspec(dllimport) ++#endif ++ ++#if !defined(WIN32) || defined(APU_MODULE_DECLARE_STATIC) ++/** ++ * Declare a dso module's exported module structure as APU_MODULE_DECLARE_DATA. ++ * ++ * Unless APU_MODULE_DECLARE_STATIC is defined at compile time, symbols ++ * declared with APU_MODULE_DECLARE_DATA are always exported. ++ * @code ++ * module APU_MODULE_DECLARE_DATA mod_tag ++ * @endcode ++ */ ++#define APU_MODULE_DECLARE_DATA ++#else ++#define APU_MODULE_DECLARE_DATA __declspec(dllexport) ++#endif ++ ++/* ++ * we always have SDBM (it's in our codebase) ++ */ ++#define APU_HAVE_SDBM 1 ++#define APU_HAVE_GDBM 0 ++#define APU_HAVE_NDBM 0 ++#define APU_HAVE_DB 0 ++ ++#if APU_HAVE_DB ++#define APU_HAVE_DB_VERSION 0 ++#endif ++ ++#define APU_HAVE_PGSQL 0 ++#define APU_HAVE_MYSQL 0 ++#define APU_HAVE_SQLITE3 0 ++#define APU_HAVE_SQLITE2 0 ++#define APU_HAVE_ORACLE 0 ++#define APU_HAVE_ODBC 0 ++ ++#define APU_HAVE_CRYPTO 0 ++#define APU_HAVE_OPENSSL 0 ++#define APU_HAVE_NSS 0 ++#define APU_HAVE_COMMONCRYPTO 0 ++ ++#define APU_HAVE_APR_ICONV 0 ++#define APU_HAVE_ICONV 1 ++#define APR_HAS_XLATE (APU_HAVE_APR_ICONV || APU_HAVE_ICONV) ++ ++#endif /* APU_H */ ++/** @} */ +diff -urN libaprutil1/include/apu_want.h libaprutil/include/apu_want.h +--- libaprutil1/include/apu_want.h 1970-01-01 00:00:00.000000000 +0000 ++++ libaprutil/include/apu_want.h 2019-02-09 23:58:54.000000000 +0000 +@@ -0,0 +1,51 @@ ++/* Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++#include "apu.h" /* configuration data */ ++ ++/** ++ * @file apu_want.h ++ * @brief APR Standard Headers Support ++ * ++ * 
++ * Features:
++ *
++ *   APU_WANT_DB:       
++ *
++ * Typical usage:
++ *
++ *   #define APU_WANT_DB
++ *   #include "apu_want.h"
++ *
++ * The appropriate headers will be included.
++ *
++ * Note: it is safe to use this in a header (it won't interfere with other
++ *       headers' or source files' use of apu_want.h)
++ *
++ */ ++ ++/* --------------------------------------------------------------------- */ ++ ++#ifdef APU_WANT_DB ++ ++#if APU_HAVE_DB ++#include ++#endif ++ ++#undef APU_WANT_DB ++#endif ++ ++/* --------------------------------------------------------------------- */ +diff -urN libaprutil1/include/private/apu_config.h libaprutil/include/private/apu_config.h +--- libaprutil1/include/private/apu_config.h 1970-01-01 00:00:00.000000000 +0000 ++++ libaprutil/include/private/apu_config.h 2019-02-09 23:59:45.000000000 +0000 +@@ -0,0 +1,184 @@ ++/* include/private/apu_config.h. Generated from apu_config.h.in by configure. */ ++/* include/private/apu_config.h.in. Generated from configure.in by autoheader. */ ++ ++/* Define if the system crypt() function is threadsafe */ ++/* #undef APU_CRYPT_THREADSAFE */ ++ ++/* Define to 1 if modular components are built as DSOs */ ++#define APU_DSO_BUILD 0 ++ ++/* Define to be absolute path to DSO directory */ ++#define APU_DSO_LIBDIR "/usr/local/apr/lib/apr-util-1" ++ ++/* Define if the inbuf parm to iconv() is const char ** */ ++/* #undef APU_ICONV_INBUF_CONST */ ++ ++/* Define that OpenSSL uses const buffers */ ++/* #undef CRYPTO_OPENSSL_CONST_BUFFERS */ ++ ++/* Define if crypt_r has uses CRYPTD */ ++/* #undef CRYPT_R_CRYPTD */ ++ ++/* Define if crypt_r uses struct crypt_data */ ++#define CRYPT_R_STRUCT_CRYPT_DATA 1 ++ ++/* Define if CODESET is defined in langinfo.h */ ++#define HAVE_CODESET 1 ++ ++/* Define to 1 if you have the header ++ file. */ ++/* #undef HAVE_COMMONCRYPTO_COMMONKEYDERIVATION_H */ ++ ++/* Define to 1 if you have the `crypt_r' function. */ ++#define HAVE_CRYPT_R 1 ++ ++/* Define to 1 if you have the declaration of `EVP_PKEY_CTX_new', and to 0 if ++ you don't. */ ++/* #undef HAVE_DECL_EVP_PKEY_CTX_NEW */ ++ ++/* Define if expat.h is available */ ++#define HAVE_EXPAT_H 1 ++ ++/* Define if explicit_bzero function is supported */ ++/* #undef HAVE_EXPLICIT_BZERO */ ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_ICONV_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_INTTYPES_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_LANGINFO_H 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_LBER_H */ ++ ++/* Defined if ldap.h is present */ ++/* #undef HAVE_LDAP_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_LDAP_SSL_H */ ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_LIBPQ_FE_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_MEMORY_H 1 ++ ++/* Define if memset_s function is supported */ ++/* #undef HAVE_MEMSET_S */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_MYSQL_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_MYSQL_MYSQL_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_MYSQL_MY_GLOBAL_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_MYSQL_MY_SYS_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_MY_GLOBAL_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_MY_SYS_H */ ++ ++/* Define to 1 if you have the `nl_langinfo' function. */ ++#define HAVE_NL_LANGINFO 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_NSS_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_NSS_NSS_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_NSS_PK11PUB_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_OCI_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_ODBC_SQL_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_OPENSSL_X509_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_PK11PUB_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_POSTGRESQL_LIBPQ_FE_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_PRERROR_H */ ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SQLITE3_H 1 ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_SQLITE_H */ ++ ++/* Define to 1 if you have the header file. */ ++/* #undef HAVE_SQL_H */ ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_STDINT_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_STDLIB_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_STRINGS_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_STRING_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_STAT_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_SYS_TYPES_H 1 ++ ++/* Define to 1 if you have the header file. */ ++#define HAVE_UNISTD_H 1 ++ ++/* Define if compiler handles weak symbols */ ++#define HAVE_WEAK_SYMBOLS 1 ++ ++/* Define if xmlparse/xmlparse.h is available */ ++/* #undef HAVE_XMLPARSE_XMLPARSE_H */ ++ ++/* Define if xmltok/xmlparse.h is available */ ++/* #undef HAVE_XMLTOK_XMLPARSE_H */ ++ ++/* Define if xml/xmlparse.h is available */ ++/* #undef HAVE_XML_XMLPARSE_H */ ++ ++/* Define if ldap_set_rebind_proc takes three arguments */ ++/* #undef LDAP_SET_REBIND_PROC_THREE */ ++ ++/* Define to the address where bug reports for this package should be sent. */ ++#define PACKAGE_BUGREPORT "" ++ ++/* Define to the full name of this package. */ ++#define PACKAGE_NAME "" ++ ++/* Define to the full name and version of this package. */ ++#define PACKAGE_STRING "" ++ ++/* Define to the one symbol short name of this package. */ ++#define PACKAGE_TARNAME "" ++ ++/* Define to the home page for this package. */ ++#define PACKAGE_URL "" ++ ++/* Define to the version of this package. */ ++#define PACKAGE_VERSION "" ++ ++/* Define to 1 if you have the ANSI C header files. */ ++#define STDC_HEADERS 1 +diff -urN libaprutil1/include/private/apu_select_dbm.h libaprutil/include/private/apu_select_dbm.h +--- libaprutil1/include/private/apu_select_dbm.h 1970-01-01 00:00:00.000000000 +0000 ++++ libaprutil/include/private/apu_select_dbm.h 2019-02-09 23:56:50.000000000 +0000 +@@ -0,0 +1,28 @@ ++/* Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++#ifndef APU_SELECT_DBM_H ++#define APU_SELECT_DBM_H ++ ++/* ++** The following macros control what features APRUTIL will use ++*/ ++#define APU_USE_SDBM 1 ++#define APU_USE_NDBM 0 ++#define APU_USE_GDBM 0 ++#define APU_USE_DB 0 ++ ++#endif /* !APU_SELECT_DBM_H */ diff --git a/third_party/minini.BUILD b/third_party/minini.BUILD new file mode 100644 index 000000000..186954fef --- /dev/null +++ b/third_party/minini.BUILD @@ -0,0 +1,21 @@ +licenses(["notice"]) # Apache 2.0 + +cc_library( + name = "minini", + srcs = [ + "dev/minGlue.h", + "dev/minGlue-FatFs.h", + "dev/minGlue-ccs.h", + "dev/minGlue-efsl.h", + "dev/minGlue-ffs.h", + "dev/minGlue-mdd.h", + "dev/minGlue-stdio.h", + "dev/minIni.c", + "dev/wxMinIni.h", + ], + hdrs = ["dev/minIni.h"], + includes = [ + "dev", + ], + visibility = ["//visibility:public"], +) diff --git a/third_party/mxml.BUILD b/third_party/mxml.BUILD new file mode 100644 index 000000000..6922f6976 --- /dev/null +++ b/third_party/mxml.BUILD @@ -0,0 +1,33 @@ +licenses(["notice"]) # Apache 2.0 + +cc_library( + name = "mxml", + srcs = [ + "config.h", + "mxml-attr.c", + "mxml-entity.c", + "mxml-file.c", + "mxml-get.c", + "mxml-index.c", + "mxml-node.c", + "mxml-private.c", + "mxml-private.h", + "mxml-search.c", + "mxml-set.c", + "mxml-string.c", + ], + hdrs = [ + "mxml.h", + ], + copts = ["-pthread"], + defines = [ + "_GNU_SOURCE", + "_THREAD_SAFE", + "_REENTRANT", + ], + includes = [ + ".", + ], + linkopts = ["-lpthread"], + visibility = ["//visibility:public"], +) diff --git a/third_party/mxml.patch b/third_party/mxml.patch new file mode 100644 index 000000000..8abda5da5 --- /dev/null +++ b/third_party/mxml.patch @@ -0,0 +1,114 @@ +diff -urN mxml/config.h mxml.org/config.h +--- mxml/config.h 1970-01-01 00:00:00.000000000 +0000 ++++ mxml.org/config.h 2019-02-10 22:55:59.000000000 +0000 +@@ -0,0 +1,110 @@ ++/* config.h. Generated from config.h.in by configure. */ ++/* ++ * Configuration file for Mini-XML, a small XML file parsing library. ++ * ++ * Copyright 2003-2017 by Michael R Sweet. ++ * ++ * These coded instructions, statements, and computer programs are the ++ * property of Michael R Sweet and are protected by Federal copyright ++ * law. Distribution and use rights are outlined in the file "COPYING" ++ * which should have been included with this file. If this file is ++ * missing or damaged, see the license at: ++ * ++ * https://michaelrsweet.github.io/mxml ++ */ ++ ++/* ++ * Include necessary headers... ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++ ++/* ++ * Version number... ++ */ ++ ++#define MXML_VERSION "Mini-XML v2.12" ++ ++ ++/* ++ * Inline function support... ++ */ ++ ++#define inline ++ ++ ++/* ++ * Long long support... ++ */ ++ ++#define HAVE_LONG_LONG 1 ++ ++ ++/* ++ * Do we have ? ++ */ ++ ++#define HAVE_ZLIB_H 1 ++ ++ ++/* ++ * Do we have the *printf() functions? ++ */ ++ ++#define HAVE_SNPRINTF 1 ++#define HAVE_VASPRINTF 1 ++#define HAVE_VSNPRINTF 1 ++ ++ ++/* ++ * Do we have the strXXX() functions? ++ */ ++ ++#define HAVE_STRDUP 1 ++/* #undef HAVE_STRLCAT */ ++/* #undef HAVE_STRLCPY */ ++ ++ ++/* ++ * Do we have threading support? ++ */ ++ ++#define HAVE_PTHREAD_H 1 ++ ++ ++/* ++ * Define prototypes for string functions as needed... ++ */ ++ ++# ifndef HAVE_STRDUP ++extern char *_mxml_strdup(const char *); ++# define strdup _mxml_strdup ++# endif /* !HAVE_STRDUP */ ++ ++# ifndef HAVE_STRLCAT ++extern size_t _mxml_strlcat(char *, const char *, size_t); ++# define strlcat _mxml_strlcat ++# endif /* !HAVE_STRLCAT */ ++ ++# ifndef HAVE_STRLCPY ++extern size_t _mxml_strlcpy(char *, const char *, size_t); ++# define strlcpy _mxml_strlcpy ++# endif /* !HAVE_STRLCPY */ ++ ++extern char *_mxml_strdupf(const char *, ...); ++extern char *_mxml_vstrdupf(const char *, va_list); ++ ++# ifndef HAVE_SNPRINTF ++extern int _mxml_snprintf(char *, size_t, const char *, ...); ++# define snprintf _mxml_snprintf ++# endif /* !HAVE_SNPRINTF */ ++ ++# ifndef HAVE_VSNPRINTF ++extern int _mxml_vsnprintf(char *, size_t, const char *, va_list); ++# define vsnprintf _mxml_vsnprintf ++# endif /* !HAVE_VSNPRINTF */ diff --git a/third_party/oss_c_sdk.BUILD b/third_party/oss_c_sdk.BUILD new file mode 100644 index 000000000..46d1df3d7 --- /dev/null +++ b/third_party/oss_c_sdk.BUILD @@ -0,0 +1,55 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +cc_library( + name = "aliyun_oss_c_sdk", + srcs = [ + "oss_c_sdk/aos_buf.c", + "oss_c_sdk/aos_buf.h", + "oss_c_sdk/aos_crc64.c", + "oss_c_sdk/aos_crc64.h", + "oss_c_sdk/aos_fstack.c", + "oss_c_sdk/aos_fstack.h", + "oss_c_sdk/aos_http_io.c", + "oss_c_sdk/aos_http_io.h", + "oss_c_sdk/aos_list.h", + "oss_c_sdk/aos_log.c", + "oss_c_sdk/aos_status.c", + "oss_c_sdk/aos_string.c", + "oss_c_sdk/aos_transport.c", + "oss_c_sdk/aos_transport.h", + "oss_c_sdk/aos_util.c", + "oss_c_sdk/oss_auth.c", + "oss_c_sdk/oss_bucket.c", + "oss_c_sdk/oss_define.c", + "oss_c_sdk/oss_define.h", + "oss_c_sdk/oss_live.c", + "oss_c_sdk/oss_multipart.c", + "oss_c_sdk/oss_object.c", + "oss_c_sdk/oss_resumable.c", + "oss_c_sdk/oss_resumable.h", + "oss_c_sdk/oss_util.c", + "oss_c_sdk/oss_xml.c", + "oss_c_sdk/oss_xml.h", + ], + hdrs = [ + "oss_c_sdk/aos_define.h", + "oss_c_sdk/aos_log.h", + "oss_c_sdk/aos_status.h", + "oss_c_sdk/aos_string.h", + "oss_c_sdk/aos_util.h", + "oss_c_sdk/oss_api.h", + "oss_c_sdk/oss_auth.h", + "oss_c_sdk/oss_util.h", + ], + includes = [ + "oss_c_sdk", + ], + deps = [ + "@curl", + "@libapr1", + "@libaprutil1", + "@mxml", + ], +)