Skip to content

Commit 6ba765e

Browse files
DzkolDzkol
authored
Merge pull request #182 from tencentyun/feature_gavinhgchen_96947d0b
opt cpu&mem for MultiPutObject&PutObjectResumableSingleThreadSync,add cmake option USE_OPENSSL_MD5
2 parents 3efcdf2 + a4c9bb8 commit 6ba765e

File tree

15 files changed

+149
-56
lines changed

15 files changed

+149
-56
lines changed

‎CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ option(BUILD_UNITTEST "Build unittest" OFF)
77
option(BUILD_DEMO "Build demo" ON)
88
option(BUILD_SHARED_LIB "Build shared library" OFF)
99
option(ENABLE_COVERAGE "Enable Coverage" OFF)
10+
option(USE_OPENSSL_MD5 "Use Openssl Md5" OFF)
1011

1112
if(APPLE)
1213
set(OS_TYPE "APPLE")
@@ -70,6 +71,7 @@ else()
7071
endif()
7172

7273
set(SYSTEM_LIBS stdc++ pthread)
74+
set(OPENSSL_LIBS ssl crypto)
7375
endif()
7476

7577
if(ENABLE_COVERAGE)

‎demo/CMakeLists.txt‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -43,23 +43,23 @@ add_executable(put_bucket_demo ${put_bucket_demo_src})
4343

4444
set(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/bin)
4545

46-
target_link_libraries(cos_demo cossdk ssl crypto ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
47-
target_link_libraries(put_object_demo ssl crypto cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
48-
target_link_libraries(get_object_url_demo ssl crypto cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
49-
target_link_libraries(head_object_demo ssl crypto cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
50-
target_link_libraries(get_object_demo ssl crypto cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
51-
target_link_libraries(delete_object_demo ssl crypto cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
52-
target_link_libraries(put_get_object_acl_demo ssl crypto cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
53-
target_link_libraries(put_get_delete_object_tagging_demo ssl crypto cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
54-
target_link_libraries(restore_object_demo cossdk ssl crypto ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
55-
target_link_libraries(copy_move_object_demo cossdk ssl crypto ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
56-
target_link_libraries(multi_put_object_demo cossdk ssl crypto ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
57-
target_link_libraries(multi_get_object_demo cossdk ssl crypto ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
58-
target_link_libraries(select_objec_demo ssl crypto cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
59-
target_link_libraries(get_bucket_list_demo ssl crypto cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
60-
target_link_libraries(delete_bucket_demo ssl crypto cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
61-
target_link_libraries(head_bucket_demo ssl crypto cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
62-
target_link_libraries(put_bucket_demo ssl crypto cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
46+
target_link_libraries(cos_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
47+
target_link_libraries(put_object_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
48+
target_link_libraries(get_object_url_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
49+
target_link_libraries(head_object_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
50+
target_link_libraries(get_object_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
51+
target_link_libraries(delete_object_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
52+
target_link_libraries(put_get_object_acl_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
53+
target_link_libraries(put_get_delete_object_tagging_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
54+
target_link_libraries(restore_object_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
55+
target_link_libraries(copy_move_object_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
56+
target_link_libraries(multi_put_object_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
57+
target_link_libraries(multi_get_object_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
58+
target_link_libraries(select_objec_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
59+
target_link_libraries(get_bucket_list_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
60+
target_link_libraries(delete_bucket_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
61+
target_link_libraries(head_bucket_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
62+
target_link_libraries(put_bucket_demo cossdk ${POCO_LIBS} ${OPENSSL_LIBS} ${SYSTEM_LIBS})
6363

6464
include_directories(${CMAKE_SOURCE_DIR}/include/ ${POCO_INCLUDE_DIR})
6565

‎gen_lcov.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ rm UTResport.tar
88

99
mkdir -p build
1010
cd build
11-
cmake -DENABLE_COVERAGE=ON -DBUILD_UNITTEST=ON ..
11+
cmake -DENABLE_COVERAGE=ON -DBUILD_UNITTEST=ON -DUSE_OPENSSL_MD5=ON ..
1212
gmake all -j 4
1313

1414
# init

‎include/op/object_op.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -436,7 +436,8 @@ class ObjectOp : public BaseOp {
436436
CosResult SingleThreadUpload(const PutObjectByFileReq& req, const std::string& upload_id,
437437
const std::vector<std::string>& already_exist_parts,
438438
bool resume_flag, std::vector<std::string>* etags_ptr,
439-
std::vector<uint64_t>* part_numbers_ptr, PutObjectByFileResp* resp);
439+
std::vector<uint64_t>* part_numbers_ptr, PutObjectByFileResp* resp,
440+
uint64_t& crc64);
440441

441442
/// \brief 读取文件内容, 并返回读取的长度
442443
// uint64_t GetContent(const std::string& src, std::string* file_content) const;

‎include/trsf/transfer_handler.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,11 @@ class HandleStreamCopier {
205205
std::istream& istr,
206206
std::ostream& ostr,
207207
std::size_t bufferSize = 8192);
208+
209+
static std::streamsize handleCopyStream(const SharedTransferHandler& handler,
210+
const char *buf, size_t buf_len,
211+
std::ostream& ostr,
212+
std::size_t bufferSize = 8192);
208213
};
209214

210215
class UserCancelException : public std::exception {

‎include/util/codec_util.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ class CodecUtil {
8181
static std::string RawMd51(const std::string& plainText);
8282

8383
static std::string HexToBin(const std::string& strHex);
84+
85+
static std::string DigestToHex(const unsigned char *digest, size_t len);
8486
};
8587

8688
} // namespace qcloud_cos

‎include/util/http_sender.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,8 @@ class HttpSender {
8080
bool is_verify_cert = true,
8181
const std::string& ca_location = "",
8282
SSLCtxCallback ssl_ctx_cb = nullptr,
83-
void *user_data = nullptr);
83+
void *user_data = nullptr,
84+
const char *req_body_buf = nullptr, size_t req_body_len = 0);
8485

8586
static int SendRequest(const SharedTransferHandler& handler,
8687
const std::string& http_method,

‎src/CMakeLists.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
project(cossdk)
22

3+
if(USE_OPENSSL_MD5)
4+
message(STATUS "Use Openssl Md5 Func")
5+
add_definitions(-DUSE_OPENSSL_MD5)
6+
endif()
7+
38
file(GLOB sdk_common_header "${CMAKE_SOURCE_DIR}/include/*.h")
49
file(GLOB sdk_op_header "${CMAKE_SOURCE_DIR}/include/op/*.h")
510
file(GLOB sdk_request_header "${CMAKE_SOURCE_DIR}/include/request/*.h")

‎src/op/file_upload_task.cpp‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@
55
#include "Poco/StreamCopier.h"
66
#include "util/http_sender.h"
77
#include "util/string_util.h"
8+
#include "util/codec_util.h"
9+
#ifdef USE_OPENSSL_MD5
10+
#include <openssl/md5.h>
11+
#endif
812

913
namespace qcloud_cos {
1014

@@ -149,14 +153,23 @@ void FileUploadTask::SetSslCtxCb(SSLCtxCallback cb, void *data) {
149153
}
150154

151155
void FileUploadTask::UploadTask() {
152-
std::string body((const char*)m_data_buf_ptr, m_data_len);
153-
// 计算上传的md5
154-
Poco::MD5Engine md5;
155-
std::istringstream istr(body);
156-
Poco::DigestOutputStream dos(md5);
157-
Poco::StreamCopier::copyStream(istr, dos);
158-
dos.close();
159-
const std::string& md5_str = Poco::DigestEngine::digestToHex(md5.digest());
156+
std::string md5_str;
157+
#ifdef USE_OPENSSL_MD5
158+
unsigned char digest[MD5_DIGEST_LENGTH];
159+
MD5((const unsigned char *)m_data_buf_ptr, m_data_len, digest);
160+
md5_str = CodecUtil::DigestToHex(digest, MD5_DIGEST_LENGTH);
161+
#else
162+
{
163+
// 计算上传的md5
164+
Poco::MD5Engine md5;
165+
std::string body((const char*)m_data_buf_ptr, m_data_len);
166+
std::istringstream istr(body);
167+
Poco::DigestOutputStream dos(md5);
168+
Poco::StreamCopier::copyStream(istr, dos);
169+
dos.close();
170+
md5_str = Poco::DigestEngine::digestToHex(md5.digest());
171+
}
172+
#endif
160173

161174
int loop = 0;
162175
do {
@@ -174,11 +187,15 @@ void FileUploadTask::UploadTask() {
174187
// &m_err_msg, false);
175188
// m_resp = oss.str();
176189
// } else {
190+
std::istringstream is;
191+
std::ostringstream oss;
177192
m_http_status = HttpSender::SendRequest(
178-
m_handler, "PUT", m_full_url, m_params, m_headers, body,
179-
m_conn_timeout_in_ms, m_recv_timeout_in_ms, &m_resp_headers, &m_resp,
180-
&m_err_msg, false, m_verify_cert, m_ca_location, m_ssl_ctx_cb, m_user_data);
193+
m_handler, "PUT", m_full_url, m_params, m_headers, is,
194+
m_conn_timeout_in_ms, m_recv_timeout_in_ms, &m_resp_headers, oss,
195+
&m_err_msg, false, m_verify_cert, m_ca_location, m_ssl_ctx_cb, m_user_data,
196+
(const char*)m_data_buf_ptr, m_data_len);
181197
//}
198+
m_resp = oss.str();
182199

183200
if (m_http_status != 200) {
184201
SDK_LOG_ERR("FileUpload: url(%s) fail, httpcode:%d, resp: %s",

‎src/op/object_op.cpp‎

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616
#else
1717
#include <unistd.h>
1818
#endif
19+
#ifdef USE_OPENSSL_MD5
20+
#include <openssl/md5.h>
21+
#endif
1922

2023
#include "Poco/DigestStream.h"
2124
#include "Poco/DirectoryIterator.h"
@@ -854,9 +857,10 @@ CosResult ObjectOp::UploadObjectResumableSingleThreadSync(const PutObjectByFileR
854857
std::vector<uint64_t> part_numbers;
855858

856859
PutObjectByFileResp upload_resp;
860+
uint64_t crc64_origin = 0;
857861
CosResult upload_result =
858862
SingleThreadUpload(req, resume_uploadid, already_exist_parts, resume_flag,
859-
&etags, &part_numbers, &upload_resp);
863+
&etags, &part_numbers, &upload_resp, crc64_origin);
860864

861865
// Notice the cancel way not need to abort the uploadid
862866
if (!upload_result.IsSucc()) {
@@ -885,16 +889,6 @@ CosResult ObjectOp::UploadObjectResumableSingleThreadSync(const PutObjectByFileR
885889
// check crc64 if needed
886890
if (req.CheckCRC64() && comp_result.IsSucc() &&
887891
!comp_resp.GetXCosHashCrc64Ecma().empty()) {
888-
uint64_t crc64_origin = 0;
889-
#if defined(_WIN32)
890-
if (req.IsWideCharPath()) {
891-
crc64_origin = FileUtil::GetFileCrc64(req.GetWideCharLocalFilePath());
892-
} else {
893-
crc64_origin = FileUtil::GetFileCrc64(req.GetLocalFilePath());
894-
}
895-
#else
896-
crc64_origin = FileUtil::GetFileCrc64(req.GetLocalFilePath());
897-
#endif
898892
uint64_t crc64_server_resp =
899893
StringUtil::StringToUint64(comp_resp.GetXCosHashCrc64Ecma());
900894
if (crc64_server_resp != crc64_origin) {
@@ -1915,7 +1909,8 @@ CosResult ObjectOp::SingleThreadUpload(
19151909
const PutObjectByFileReq& req, const std::string& upload_id,
19161910
const std::vector<std::string>& already_exist_parts, bool resume_flag,
19171911
std::vector<std::string>* etags_ptr,
1918-
std::vector<uint64_t>* part_numbers_ptr, PutObjectByFileResp* resp) {
1912+
std::vector<uint64_t>* part_numbers_ptr, PutObjectByFileResp* resp,
1913+
uint64_t& crc64) {
19191914
CosResult result;
19201915
std::string path = "/" + req.GetObjectName();
19211916
std::string host = CosSysConfig::GetHost(GetAppId(), m_config->GetRegion(),
@@ -1976,6 +1971,7 @@ CosResult ObjectOp::SingleThreadUpload(
19761971
SDK_LOG_DBG("upload data, part_size=%" PRIu64
19771972
", file_size=%" PRIu64, part_size, file_size);
19781973

1974+
crc64 = 0;
19791975
// 3. 单线程upload
19801976
{
19811977
uint64_t part_number = 1;
@@ -1990,6 +1986,10 @@ CosResult ObjectOp::SingleThreadUpload(
19901986
", offset=%" PRIu64 ", len=%" PRIu64,
19911987
part_number, file_size, offset, read_len);
19921988

1989+
// 提前计算整个文件的crc64，用于整个合并分块完成后做crc64校验
1990+
crc64 = CRC64::CalcCRC(crc64, static_cast<void*>(file_content_buf),
1991+
static_cast<size_t>(read_len));
1992+
19931993
// Check the resume
19941994

19951995
if (resume_flag && !already_exist_parts[part_number].empty()) {
@@ -2013,6 +2013,15 @@ CosResult ObjectOp::SingleThreadUpload(
20132013
if (req.GetHeader("x-cos-traffic-limit") != "") {
20142014
upload_part_req.SetTrafficLimit(req.GetHeader("x-cos-traffic-limit"));
20152015
}
2016+
2017+
#ifdef USE_OPENSSL_MD5
2018+
// 提前计算Content-MD5
2019+
unsigned char digest[MD5_DIGEST_LENGTH];
2020+
MD5((const unsigned char *)file_content_buf, read_len, digest);
2021+
std::string digest_str((const char*)digest, MD5_DIGEST_LENGTH);
2022+
upload_part_req.AddHeader("Content-MD5", CodecUtil::Base64Encode(digest_str));
2023+
#endif
2024+
20162025
qcloud_cos::UploadPartDataResp upload_part_resp;
20172026
qcloud_cos::CosResult upload_part_result = UploadPartData(upload_part_req, &upload_part_resp);
20182027

0 commit comments

Comments
 (0)