From 6d18e3ba101823ff86590c6d33ec616d71d128ae Mon Sep 17 00:00:00 2001 From: huberyxxiao Date: Tue, 14 Jan 2025 17:08:01 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=8D=95=E7=BA=BF?= =?UTF-8?q?=E7=A8=8B=E6=96=AD=E7=82=B9=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- demo/object_op_demo/multi_put_object_demo.cpp | 109 +++++++- include/cos_api.h | 3 + include/cos_defines.h | 2 +- include/op/object_op.h | 11 + include/request/object_req.h | 11 + include/response/object_resp.h | 35 +++ src/cos_api.cpp | 5 + src/op/object_op.cpp | 263 ++++++++++++++++++ src/response/bucket_resp.cpp | 10 +- src/response/object_resp.cpp | 24 ++ src/util/auth_tool.cpp | 3 + src/util/log_util.cpp | 29 +- 12 files changed, 489 insertions(+), 16 deletions(-) diff --git a/demo/object_op_demo/multi_put_object_demo.cpp b/demo/object_op_demo/multi_put_object_demo.cpp index b99fc19..35475b6 100644 --- a/demo/object_op_demo/multi_put_object_demo.cpp +++ b/demo/object_op_demo/multi_put_object_demo.cpp @@ -8,6 +8,7 @@ #include "cos_api.h" #include "cos_sys_config.h" #include "util/auth_tool.h" +#include /** * 本样例演示了如何使用 COS C++ SDK 进行对象的高级上传 @@ -258,9 +259,22 @@ void AsyncPutObjectDemo(qcloud_cos::CosAPI& cos) { /* * 该 Demo 示范用户如何自行组合分块上传各接口进行对象上传 * 分块操作基于初始化、上传分块、完成分块三个接口可以实现将对象切分为多个分块，然后再将这些分块上传到 cos，最后发起 Complete 完成分块上传 - * 与本节中的高级上传接口配置一样，可通过全局设置上传线程池大小、分块大小。该上传线程池上是每次上传独立的。 * 本 Demo 中的上传分块接口 UploadPartData 仅支持传入流，最多支持10000分块，每个分块大小为1MB - 5GB，最后一个分块可以小于1MB */ + +int SslCtxCallback(void *ssl_ctx, void *data) { + std::cout << "ssl_ctx: " << ssl_ctx << " data: " << data << std::endl; + + SSL_CTX *ctx = (SSL_CTX *)ssl_ctx; + std::cout << "ssl_ctx in" << std::endl; + SSL_CTX_use_PrivateKey_file(ctx, "/data/cert/client_key.key", SSL_FILETYPE_PEM); + SSL_CTX_use_certificate_chain_file(ctx, "/data/cert/client_cert.cer"); + std::cout << "ssl_ctx out" << std::endl; + + return 0; +} + + void PutPartDemo(qcloud_cos::CosAPI& cos) { std::string object_name = "big_file.txt"; @@ -275,19 +289,36 @@ void PutPartDemo(qcloud_cos::CosAPI& cos) { // 2. UploadPartData // UploadPartData 部分，可以根据实际选择分块数量和分块大小，这里以 2 个分块为例 + + // Complete 需要的两个列表： std::vector etags; std::vector part_numbers; + std::string upload_id = init_resp.GetUploadId(); { uint64_t part_number = 1; - // 模拟上传分块数据，这里以 1M 为例 - std::vector data(1024 * 1024, 'A'); + // 模拟上传分块数据，这里以 100M 为例 + uint64_t copy_size = 1024 * 1024 * 100; + std::vector data(copy_size, 'A'); std::string content(data.begin(), data.end()); std::istringstream iss(content); qcloud_cos::UploadPartDataReq req(bucket_name, object_name, upload_id, iss); req.SetPartNumber(part_number); + // 限速上传对象，默认单位为 bit/s，限速值设置范围为 819200 - 838860800, 即800Kb/s-800Mb/s + uint64_t traffic_limit = 8192*1024*10; // 100MB 文件 5M + req.SetTrafficLimit(traffic_limit); qcloud_cos::UploadPartDataResp resp; + std::chrono::time_point start_ts, end_ts; + start_ts = std::chrono::steady_clock::now(); qcloud_cos::CosResult result = cos.UploadPartData(req, &resp); + end_ts = std::chrono::steady_clock::now(); + auto time_consumed_ms = + std::chrono::duration_cast(end_ts - start_ts) + .count(); + float rate = + ((float)copy_size / 1024 / 1024) / ((float)time_consumed_ms / 1000); + SDK_LOG_ERR("send part_number: %d, send_size: %" PRIu64 " MB, time_consumed: %" PRIu64 + " ms, rate: %.2f MB/s , traffic_limit : %.2f MB", part_number, copy_size/ 1024 / 1024, time_consumed_ms, rate, traffic_limit/1024/1024/8.0); std::cout << "==================UploadPartDataResp1=====================" << std::endl; PrintResult(result, resp); std::cout << "==========================================================" << std::endl; @@ -298,12 +329,38 @@ void PutPartDemo(qcloud_cos::CosAPI& cos) { } { uint64_t part_number = 2; - std::istringstream iss("The last part can be smaller than 1MB"); + uint64_t copy_size = 1024 * 1024 * 100; + std::vector data(copy_size, 'A'); + std::string content(data.begin(), data.end()); + std::istringstream iss(content); qcloud_cos::UploadPartDataReq req(bucket_name, object_name, upload_id, iss); req.SetPartNumber(part_number); + // 限速上传对象，默认单位为 bit/s，限速值设置范围为 819200 - 838860800, 即800Kb/s-800Mb/s + uint64_t traffic_limit = 8192 * 1024 * 5 ; + req.SetTrafficLimit(traffic_limit); + qcloud_cos::UploadPartDataResp resp; + qcloud_cos::CosResult result = cos.UploadPartData(req, &resp); + std::cout << "==================UploadPartDataResp2=====================" << std::endl; + PrintResult(result, resp); + std::cout << "==========================================================" << std::endl; + if (result.IsSucc()) { + part_numbers.push_back(part_number); + etags.push_back(resp.GetEtag()); + } + } + { + uint64_t part_number = 3; + uint64_t copy_size = 1024 * 1024 * 10; + std::vector data(copy_size, 'A'); + std::string content(data.begin(), data.end()); + std::istringstream iss(content); + qcloud_cos::UploadPartDataReq req(bucket_name, object_name, upload_id, iss); + req.SetPartNumber(part_number); + // 限速上传对象，默认单位为 bit/s，限速值设置范围为 819200 - 838860800, 即800Kb/s-800Mb/s + uint64_t traffic_limit = 8192 * 1024; + req.SetTrafficLimit(traffic_limit); qcloud_cos::UploadPartDataResp resp; qcloud_cos::CosResult result = cos.UploadPartData(req, &resp); - std::cout << "==================UploadPartDataResp2=====================" << std::endl; PrintResult(result, resp); std::cout << "==========================================================" << std::endl; @@ -328,6 +385,45 @@ void PutPartDemo(qcloud_cos::CosAPI& cos) { return; } +void PutObjectResumableSingleThreadSyncDemo(qcloud_cos::CosAPI& cos) { + std::string local_file = "SingleThreadSync.txt"; + std::string object_name = "SingleThreadSync.txt"; + + qcloud_cos::PutObjectResumableSingleSyncReq req(bucket_name, object_name, local_file); + req.SetHttps(); + req.AddHeader("x-cos-meta-ssss1","1xxxxxxx"); + req.AddHeader("x-cos-meta-ssss2","2xxxxxxx"); + req.AddHeader("x-cos-meta-ssss3","3xxxxxxx"); + req.AddHeader("x-cos-meta-ssss4","4xxxxxxx"); + uint64_t traffic_limit = 8192 * 1024;//1MB + req.SetTrafficLimit(traffic_limit); + req.SetCheckCRC64(true); + qcloud_cos::PutObjectResumableSingleSyncResp resp; + std::chrono::time_point start_ts, end_ts; + start_ts = std::chrono::steady_clock::now(); + qcloud_cos::CosResult result = cos.PutObjectResumableSingleThreadSync(req, &resp); + end_ts = std::chrono::steady_clock::now(); + if (result.IsSucc()) { + std::cout << "MultiUpload Succ." << std::endl; + std::cout << resp.GetLocation() << std::endl; + std::cout << resp.GetKey() << std::endl; + std::cout << resp.GetBucket() << std::endl; + std::cout << resp.GetEtag() << std::endl; + } else { + std::cout << "MultiUpload Fail." << std::endl; + // 获取具体失败在哪一步 + std::string resp_tag = resp.GetRespTag(); + if ("Init" == resp_tag) { + // print result + } else if ("Upload" == resp_tag) { + // print result + } else if ("Complete" == resp_tag) { + // print result + } + PrintResult(result, resp); + } + std::cout << "===========================================================" << std::endl; +} int main() { qcloud_cos::CosAPI cos = InitCosAPI(); CosSysConfig::SetLogLevel((LOG_LEVEL)COS_LOG_ERR); @@ -335,4 +431,5 @@ int main() { AsyncMultiPutObjectDemo(cos); AsyncPutObjectDemo(cos); PutPartDemo(cos); -} \ No newline at end of file + PutObjectResumableSingleThreadSyncDemo(cos); +} diff --git a/include/cos_api.h b/include/cos_api.h index efadeb8..ed16ef9 100644 --- a/include/cos_api.h +++ b/include/cos_api.h @@ -758,6 +758,9 @@ class CosAPI { CosResult MultiGetObject(const MultiGetObjectReq& req, MultiGetObjectResp* resp); + CosResult PutObjectResumableSingleThreadSync(const PutObjectResumableSingleSyncReq& req, + PutObjectResumableSingleSyncResp* resp); + /* Resumable接口 */ /// \brief 封装了初始化分块上传、分块上传、完成分块上传三步，支持断点续传 diff --git a/include/cos_defines.h b/include/cos_defines.h index 70432bc..2ee52a8 100644 --- a/include/cos_defines.h +++ b/include/cos_defines.h @@ -12,7 +12,7 @@ namespace qcloud_cos { -#define COS_CPP_SDK_VERSON "v5.5.15" +#define COS_CPP_SDK_VERSON "v5.5.16" /// 路径分隔符 const char kPathDelimiter[] = "/"; diff --git a/include/op/object_op.h b/include/op/object_op.h index c0117db..55acbe5 100644 --- a/include/op/object_op.h +++ b/include/op/object_op.h @@ -180,6 +180,12 @@ class ObjectOp : public BaseOp { const SharedTransferHandler& handler = nullptr, bool change_backup_domain = false); + /// \brief 单线程同步分块上传 + /// + /// \return result + CosResult UploadObjectResumableSingleThreadSync(const PutObjectByFileReq& req, + PutObjectResumableSingleSyncResp* resp); + /// \brief 舍弃一个分块上传并删除已上传的块 /// /// \param req AbortMultiUpload请求 @@ -427,6 +433,11 @@ class ObjectOp : public BaseOp { const SharedTransferHandler& handler = nullptr, bool change_backup_domain = false); + CosResult SingleThreadUpload(const PutObjectByFileReq& req, const std::string& upload_id, + const std::vector& already_exist_parts, + bool resume_flag, std::vector* etags_ptr, + std::vector* part_numbers_ptr, PutObjectByFileResp* resp); + /// \brief 读取文件内容, 并返回读取的长度 // uint64_t GetContent(const std::string& src, std::string* file_content) const; diff --git a/include/request/object_req.h b/include/request/object_req.h index 8904079..368c0be 100644 --- a/include/request/object_req.h +++ b/include/request/object_req.h @@ -1722,6 +1722,17 @@ class MultiGetObjectReq : public GetObjectByFileReq { virtual ~MultiGetObjectReq() {} }; +class PutObjectResumableSingleSyncReq : public PutObjectByFileReq { + public: + PutObjectResumableSingleSyncReq(const std::string& bucket_name, + const std::string& object_name, + const std::string& local_file_path) + : PutObjectByFileReq(bucket_name, object_name, local_file_path) { + SetCheckCRC64(true); + } + virtual ~PutObjectResumableSingleSyncReq() {} +}; + /* Async接口 */ #if 0 diff --git a/include/response/object_resp.h b/include/response/object_resp.h index 7287819..0ef5915 100644 --- a/include/response/object_resp.h +++ b/include/response/object_resp.h @@ -670,6 +670,41 @@ class MultiGetObjectResp : public GetObjectByFileResp { ~MultiGetObjectResp() {} }; +class PutObjectResumableSingleSyncResp : public GetObjectByFileResp { + public: + PutObjectResumableSingleSyncResp() {} + + virtual ~PutObjectResumableSingleSyncResp() {} + + std::string GetRespTag() { return m_resp_tag; } + + std::string GetLocation() const { return m_location; } + + std::string GetKey() const { return m_key; } + + std::string GetBucket() const { return m_bucket; } + + void CopyFrom(const InitMultiUploadResp& resp); + + void CopyFrom(const PutObjectByFileResp& resp); + + void CopyFrom(const CompleteMultiUploadResp& resp); + + /// \brief Server端加密使用的算法 + std::string GetXCosServerSideEncryption() const { + return GetHeader("x-cos-server-side-encryption"); + } + + private: + std::string m_location; // Object的外网访问域名 + std::string m_bucket; + std::string m_key; + std::string m_upload_id; + + // FIXME(sevenyou) 先这么搞吧 + std::string m_resp_tag; // 用于区分是哪一种response +}; + /* Async接口 */ //typedef PutObjectByFileResp PutObjectAsyncResp; diff --git a/src/cos_api.cpp b/src/cos_api.cpp index e4bf10b..ca30528 100644 --- a/src/cos_api.cpp +++ b/src/cos_api.cpp @@ -586,6 +586,11 @@ CosResult CosAPI::MultiPutObject(const MultiPutObjectReq& req, return result; } +CosResult CosAPI::PutObjectResumableSingleThreadSync(const PutObjectResumableSingleSyncReq& req, + PutObjectResumableSingleSyncResp* resp){ + return m_object_op.UploadObjectResumableSingleThreadSync(static_cast(req), resp); +} + CosResult CosAPI::AbortMultiUpload(const AbortMultiUploadReq& req, AbortMultiUploadResp* resp) { CosResult result = m_object_op.AbortMultiUpload(req, resp); diff --git a/src/op/object_op.cpp b/src/op/object_op.cpp index 1033680..158a16a 100644 --- a/src/op/object_op.cpp +++ b/src/op/object_op.cpp @@ -797,6 +797,122 @@ CosResult ObjectOp::MultiUploadObject(const PutObjectByFileReq& req, return comp_result; } +CosResult ObjectOp::UploadObjectResumableSingleThreadSync(const PutObjectByFileReq& req, + PutObjectResumableSingleSyncResp* resp) { + if (!resp) { + CosResult result; + SetResultAndLogError(result, "Invalid input parameter"); + return result; + } + std::string bucket_name = req.GetBucketName(); + std::string object_name = req.GetObjectName(); + + bool resume_flag = false; + std::vector already_exist_parts(kMaxPartNumbers); + // check the breakpoint + std::string resume_uploadid = GetResumableUploadID(req ,bucket_name, object_name); + if (!resume_uploadid.empty()) { + resume_flag = CheckUploadPart(req, bucket_name, object_name, + resume_uploadid, already_exist_parts); + } + + if (!resume_flag) { + // 1. Init + InitMultiUploadReq init_req(bucket_name, object_name); + + CosResult init_result; + InitMultiUploadResp init_resp; + if (req.IsHttps()) { + init_req.SetHttps(); + init_req.SetVerifyCert(req.GetVerifyCert()); + init_req.SetCaLocation(req.GetCaLocation()); + init_req.SetSSLCtxCallback(req.GetSSLCtxCallback(), req.GetSSLCtxCbData()); + } + init_req.AddHeaders(req.GetHeaders()); + init_req.SetConnTimeoutInms(req.GetConnTimeoutInms()); + init_req.SetRecvTimeoutInms(req.GetRecvTimeoutInms()); + init_result = InitMultiUpload(init_req, &init_resp); + if (!init_result.IsSucc()) { + std::string err_msg = "Init multipart upload failed"; + SetResultAndLogError(init_result, err_msg); + resp->CopyFrom(init_resp); + return init_result; + } + resume_uploadid = init_resp.GetUploadId(); + if (resume_uploadid.empty()) { + std::string err_msg = "upload id empty"; + SetResultAndLogError(init_result, err_msg); + resp->CopyFrom(init_resp); + return init_result; + } + } + SDK_LOG_INFO("Multi upload object, resume_uploadid:%s, resumed:%d", + resume_uploadid.c_str(), resume_flag); + + // 2. Upload + std::vector etags; + std::vector part_numbers; + + PutObjectByFileResp upload_resp; + CosResult upload_result = + SingleThreadUpload(req, resume_uploadid, already_exist_parts, resume_flag, + &etags, &part_numbers, &upload_resp); + + // Notice the cancel way not need to abort the uploadid + if (!upload_result.IsSucc()) { + // 失败了不abort,再次上传走断点续传 + resp->CopyFrom(upload_resp); + return upload_result; + } + + // 3. Complete + CosResult comp_result; + CompleteMultiUploadReq comp_req(bucket_name, object_name, resume_uploadid); + CompleteMultiUploadResp comp_resp; + comp_req.SetConnTimeoutInms(req.GetConnTimeoutInms()); + // Double timeout time + comp_req.SetRecvTimeoutInms(req.GetRecvTimeoutInms() * 2); + comp_req.SetEtags(etags); + comp_req.SetPartNumbers(part_numbers); + if (req.IsHttps()) { + comp_req.SetHttps(); + comp_req.SetCaLocation(req.GetCaLocation()); + comp_req.SetVerifyCert(req.GetVerifyCert()); + comp_req.SetSSLCtxCallback(req.GetSSLCtxCallback(), req.GetSSLCtxCbData()); + } + + comp_result = CompleteMultiUpload(comp_req, &comp_resp); + // check crc64 if needed + if (req.CheckCRC64() && comp_result.IsSucc() && + !comp_resp.GetXCosHashCrc64Ecma().empty()) { + uint64_t crc64_origin = 0; +#if defined(_WIN32) + if (req.IsWideCharPath()) { + crc64_origin = FileUtil::GetFileCrc64(req.GetWideCharLocalFilePath()); + } else { + crc64_origin = FileUtil::GetFileCrc64(req.GetLocalFilePath()); + } +#else + crc64_origin = FileUtil::GetFileCrc64(req.GetLocalFilePath()); +#endif + uint64_t crc64_server_resp = + StringUtil::StringToUint64(comp_resp.GetXCosHashCrc64Ecma()); + if (crc64_server_resp != crc64_origin) { + std::string err_msg = + "MultiUploadObject failed, crc64 check failed, crc64_origin: " + + std::to_string(crc64_origin) + + ", crc64_server_resp: " + std::to_string(crc64_server_resp); + SetResultAndLogError(comp_result, err_msg); + } + SDK_LOG_DBG("crc64 check crc64_server_resp:[%llu], crc64_origin:[%llu], is same:[%s]", crc64_server_resp, crc64_origin, crc64_server_resp == crc64_origin ? "true" : "false"); + } + + if (comp_result.IsSucc()) { + resp->CopyFrom(comp_resp); + } + return comp_result; +} + CosResult ObjectOp::InitMultiUpload(const InitMultiUploadReq& req, InitMultiUploadResp* resp, bool change_backup_domain) { std::string host = CosSysConfig::GetHost(GetAppId(), m_config->GetRegion(), @@ -1795,6 +1911,153 @@ CosResult ObjectOp::MultiThreadUpload( return result; } +CosResult ObjectOp::SingleThreadUpload( + const PutObjectByFileReq& req, const std::string& upload_id, + const std::vector& already_exist_parts, bool resume_flag, + std::vector* etags_ptr, + std::vector* part_numbers_ptr, PutObjectByFileResp* resp) { + CosResult result; + std::string path = "/" + req.GetObjectName(); + std::string host = CosSysConfig::GetHost(GetAppId(), m_config->GetRegion(), + req.GetBucketName()); + + // 1. 获取文件大小 + std::string local_file_path = req.GetLocalFilePath(); + std::ifstream fin; +#if defined(_WIN32) + if (req.IsWideCharPath()) { + fin.open(req.GetWideCharLocalFilePath(), std::ios::in | std::ios::binary); + } else { + fin.open(req.GetLocalFilePath(), std::ios::in | std::ios::binary); + } +#else + fin.open(req.GetLocalFilePath(), std::ios::in | std::ios::binary); +#endif + if (!fin) { + std::string err_msg = "Failed to open file " + req.GetLocalFilePath(); + SetResultAndLogError(result, err_msg); + return result; + } +#if defined(_WIN32) + uint64_t file_size = + req.IsWideCharPath() + ? FileUtil::GetFileLen(req.GetWideCharLocalFilePath()) + : FileUtil::GetFileLen(req.GetLocalFilePath()); +#else + uint64_t file_size = FileUtil::GetFileLen(req.GetLocalFilePath()); +#endif + // 2. 初始化upload + uint64_t offset = 0; + bool task_fail_flag = false; + + std::map headers = req.GetHeaders(); + std::map params = req.GetParams(); + + uint64_t part_size = CosSysConfig::GetUploadPartSize(); + + // Check the part number + uint64_t part_number = file_size / part_size; + uint64_t last_part_size = file_size % part_size; + if (0 != last_part_size) { + part_number += 1; + } else { + last_part_size = part_size; + } + + if (part_number > kMaxPartNumbers) { + std::string err_msg = + "Upload failed, part number: " + std::to_string(part_number) + + " larger than 10000"; + SetResultAndLogError(result, err_msg); + return result; + } + + unsigned char* file_content_buf = new unsigned char[(size_t)part_size]; + SDK_LOG_DBG("upload data, part_size=%" PRIu64 + ", file_size=%" PRIu64, part_size, file_size); + + // 3. 单线程upload + { + uint64_t part_number = 1; + while (offset < file_size) { + fin.read((char*)file_content_buf, part_size); + std::streamsize read_len = fin.gcount(); + if (read_len == 0 && fin.eof()) { + SDK_LOG_DBG("read over"); + break; + } + SDK_LOG_DBG("upload data, part_number=%d, file_size=%" PRIu64 + ", offset=%" PRIu64 ", len=%" PRIu64, + part_number, file_size, offset, read_len); + + // Check the resume + + if (resume_flag && !already_exist_parts[part_number].empty()) { + // Already has this part + SDK_LOG_INFO("part etag: %s", + already_exist_parts[part_number].c_str()); + SDK_LOG_INFO("upload data part:%" PRIu64 " has resumed", part_number); + etags_ptr->push_back(already_exist_parts[part_number]); + } else { + // 上传未上传的分块 + std::string body((const char*)file_content_buf, read_len); + std::istringstream istr(body); + qcloud_cos::UploadPartDataReq upload_part_req(req.GetBucketName(), req.GetObjectName(), upload_id, istr); + upload_part_req.SetPartNumber(part_number); + if (req.IsHttps()) { + upload_part_req.SetHttps(); + upload_part_req.SetCaLocation(req.GetCaLocation()); + upload_part_req.SetVerifyCert(req.GetVerifyCert()); + upload_part_req.SetSSLCtxCallback(req.GetSSLCtxCallback(), req.GetSSLCtxCbData()); + } + if (req.GetHeader("x-cos-traffic-limit") != ""){ + upload_part_req.SetTrafficLimit(req.GetHeader("x-cos-traffic-limit")); + } + qcloud_cos::UploadPartDataResp upload_part_resp; + qcloud_cos::CosResult upload_part_result = UploadPartData(upload_part_req, &upload_part_resp); + + if (upload_part_result.IsSucc()) { + //未包含 etag 也算失败 + std::string upload_par_etag = upload_part_resp.GetEtag(); + if (upload_par_etag != "") { + etags_ptr->push_back(upload_par_etag); + }else { + std::string err_msg = "upload failed response header missing etag"; + SetResultAndLogError(result, err_msg); + result.SetHttpStatus(upload_part_result.GetHttpStatus()); + task_fail_flag = true; + break; + } + }else { + SDK_LOG_ERR("upload data, upload task fail, rsp:%s", + upload_part_resp.DebugString().c_str()); + result.SetHttpStatus(upload_part_result.GetHttpStatus()); + if (upload_part_result.GetHttpStatus() == -1) { + result.SetErrorMsg(upload_part_result.GetErrorMsg()); + } else if (!result.ParseFromHttpResponse(upload_part_resp.GetHeaders(), + upload_part_resp.GetBody())) { + result.SetErrorMsg(upload_part_resp.GetBody()); + } + task_fail_flag = true; + break; + } + } + offset += read_len; + part_numbers_ptr->push_back(part_number); + ++part_number; + } + } + + if (!task_fail_flag) { + result.SetSucc(); + } + + // 释放相关资源 + fin.close(); + delete[] file_content_buf; + return result; +} + #if 0 uint64_t ObjectOp::GetContent(const std::string& src, std::string* file_content) const { diff --git a/src/response/bucket_resp.cpp b/src/response/bucket_resp.cpp index b9cd728..66f3a2a 100644 --- a/src/response/bucket_resp.cpp +++ b/src/response/bucket_resp.cpp @@ -162,9 +162,10 @@ bool ListMultipartUploadResp::ParseFromXmlString(const std::string& body) { rapidxml::xml_node<>* id_node = upload_node->first_node(); for (; id_node != NULL; id_node = id_node->next_sibling()) { Owner own; - if (id_node->name() == kListMultipartUploadID) { + const std::string& id_node_name = id_node->name(); + if (id_node_name == kListMultipartUploadID) { own.m_id = id_node->value(); - } else if (id_node->name() == kListMultipartUploadDisplayName) { + } else if (id_node_name == kListMultipartUploadDisplayName) { own.m_display_name = id_node->value(); } else { SDK_LOG_WARN( @@ -178,9 +179,10 @@ bool ListMultipartUploadResp::ParseFromXmlString(const std::string& body) { rapidxml::xml_node<>* id_node = upload_node->first_node(); for (; id_node != NULL; id_node = id_node->next_sibling()) { Owner own; - if (id_node->name() == kListMultipartUploadID) { + const std::string& id_node_name = id_node->name(); + if (id_node_name == kListMultipartUploadID) { own.m_id = id_node->value(); - } else if (id_node->name() == kListMultipartUploadDisplayName) { + } else if (id_node_name == kListMultipartUploadDisplayName) { own.m_display_name = id_node->value(); } else { SDK_LOG_WARN("Unknown field in KListMultipartUploadOwner node."); diff --git a/src/response/object_resp.cpp b/src/response/object_resp.cpp index c247eb9..00ff586 100644 --- a/src/response/object_resp.cpp +++ b/src/response/object_resp.cpp @@ -184,6 +184,30 @@ void MultiPutObjectResp::CopyFrom(const CompleteMultiUploadResp& resp) { SetEtag(resp.GetEtag()); } + +void PutObjectResumableSingleSyncResp::CopyFrom(const InitMultiUploadResp& resp) { + m_resp_tag = "Init"; + InternalCopyFrom(resp); + m_upload_id = resp.GetUploadId(); + m_bucket = resp.GetBucket(); + m_key = resp.GetKey(); +} + +// TODO(sevenyou) +void PutObjectResumableSingleSyncResp::CopyFrom(const PutObjectByFileResp& resp) { + m_resp_tag = "Upload"; + InternalCopyFrom(resp); +} + +void PutObjectResumableSingleSyncResp::CopyFrom(const CompleteMultiUploadResp& resp) { + m_resp_tag = "Complete"; + InternalCopyFrom(resp); + m_location = resp.GetLocation(); + m_bucket = resp.GetBucket(); + m_key = resp.GetKey(); + SetEtag(resp.GetEtag()); +} + bool ListPartsResp::ParseFromXmlString(const std::string& body) { rapidxml::xml_document<> doc; char* cstr = new char[body.size() + 1]; diff --git a/src/util/auth_tool.cpp b/src/util/auth_tool.cpp index 9eab264..5bf6798 100644 --- a/src/util/auth_tool.cpp +++ b/src/util/auth_tool.cpp @@ -153,16 +153,19 @@ std::string AuthTool::Sign(const std::string& access_key, std::string format_str = lower_method + "\n" + uri + "\n" + param_value_list + "\n" + header_value_list + "\n"; + SDK_LOG_DBG("format string :%s", format_str.c_str()); // 4. StringToSign Sha1 sha1; sha1.Append(format_str.c_str(), format_str.size()); std::string string_to_sign = "sha1\n" + start_end_time_str + "\n" + sha1.Final() + "\n"; + SDK_LOG_DBG("string_to_sign :%s", string_to_sign.c_str()); // 5. signature std::string sign_key = CodecUtil::HmacSha1Hex(start_end_time_str, secret_key); std::transform(sign_key.begin(), sign_key.end(), sign_key.begin(), ::tolower); + SDK_LOG_DBG("sign_key :%s", sign_key.c_str()); std::string signature = CodecUtil::HmacSha1Hex(string_to_sign, sign_key); std::transform(signature.begin(), signature.end(), signature.begin(), ::tolower); diff --git a/src/util/log_util.cpp b/src/util/log_util.cpp index a8e7c0f..d0685b8 100644 --- a/src/util/log_util.cpp +++ b/src/util/log_util.cpp @@ -23,15 +23,34 @@ std::string LogUtil::GetLogPrefix(int level) { return ss.str(); } +// std::string LogUtil::FormatLog(int level, const char* fmt, ...) { +// std::stringstream ss; +// ss << GetLogPrefix(level); +// char buf[1024]; +// va_list ap; +// va_start(ap, fmt); +// vsnprintf(buf, 1024, fmt, ap); +// va_end(ap); +// ss << buf; +// return ss.str(); +// } + std::string LogUtil::FormatLog(int level, const char* fmt, ...) { std::stringstream ss; ss << GetLogPrefix(level); - char buf[1024]; + std::vector buf(1024); va_list ap; - va_start(ap, fmt); - vsnprintf(buf, 1024, fmt, ap); - va_end(ap); - ss << buf; + + while (true) { + va_start(ap, fmt); + int needed = vsnprintf(buf.data(), buf.size(), fmt, ap); + va_end(ap); + if (needed > -1 && static_cast(needed) < buf.size()) { + break; + } + buf.resize(needed > -1 ? needed + 1 : buf.size() * 2); + } + ss << buf.data(); return ss.str(); } From e8e642ec223c3c7b6c05da45203236f19e6b4752 Mon Sep 17 00:00:00 2001 From: huberyxxiao Date: Tue, 14 Jan 2025 19:20:49 +0800 Subject: [PATCH 2/2] ut --- demo/object_op_demo/multi_put_object_demo.cpp | 6 +- unittest/src/object_op_test.cpp | 67 +++++++++++++++++++ 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/demo/object_op_demo/multi_put_object_demo.cpp b/demo/object_op_demo/multi_put_object_demo.cpp index 35475b6..86a25f4 100644 --- a/demo/object_op_demo/multi_put_object_demo.cpp +++ b/demo/object_op_demo/multi_put_object_demo.cpp @@ -48,6 +48,8 @@ void PrintResult(const qcloud_cos::CosResult& result, const qcloud_cos::BaseResp qcloud_cos::CosAPI InitCosAPI() { qcloud_cos::CosConfig config(appid, tmp_secret_id, tmp_secret_key, region); config.SetTmpToken(tmp_token); // 推荐使用临时密钥初始化 CosAPI 对象, 如果您使用永久密钥初始化 CosAPI 对象，请注释 + config.SetDestDomain("xxxxx.xxxxx.com"); + config.SetDomainSameToHost(true); qcloud_cos::CosAPI cos_tmp(config); return cos_tmp; } @@ -398,11 +400,9 @@ void PutObjectResumableSingleThreadSyncDemo(qcloud_cos::CosAPI& cos) { uint64_t traffic_limit = 8192 * 1024;//1MB req.SetTrafficLimit(traffic_limit); req.SetCheckCRC64(true); + req.SetSSLCtxCallback(SslCtxCallback, nullptr); qcloud_cos::PutObjectResumableSingleSyncResp resp; - std::chrono::time_point start_ts, end_ts; - start_ts = std::chrono::steady_clock::now(); qcloud_cos::CosResult result = cos.PutObjectResumableSingleThreadSync(req, &resp); - end_ts = std::chrono::steady_clock::now(); if (result.IsSucc()) { std::cout << "MultiUpload Succ." << std::endl; std::cout << resp.GetLocation() << std::endl; diff --git a/unittest/src/object_op_test.cpp b/unittest/src/object_op_test.cpp index e7a41fb..ffd1ab0 100644 --- a/unittest/src/object_op_test.cpp +++ b/unittest/src/object_op_test.cpp @@ -2197,6 +2197,73 @@ TEST_F(ObjectOpTest, MultiPutObjectTest_OneStep) { } } +TEST_F(ObjectOpTest, PutObjectResumableSingleThreadSyncTest) { + { + std::string filename = "single_upload_object_one_step"; + std::string object_name = filename; + // 1. 生成个临时文件, 用于分块上传 + { + std::ofstream fs; + fs.open(filename.c_str(), std::ios::out | std::ios::binary); + std::string str(10 * 1000 * 1000, 'b'); + for (int idx = 0; idx < 10; ++idx) { + fs << str; + } + fs.close(); + } + + // 2. 上传 + qcloud_cos::PutObjectResumableSingleSyncReq req(m_bucket_name, object_name, filename); + req.SetHttps(); + req.AddHeader("x-cos-meta-ssss1","1xxxxxxx"); + req.AddHeader("x-cos-meta-ssss2","2xxxxxxx"); + req.AddHeader("x-cos-meta-ssss3","3xxxxxxx"); + req.AddHeader("x-cos-meta-ssss4","4xxxxxxx"); + uint64_t traffic_limit = 8192 * 1024*100;//100MB + req.SetTrafficLimit(traffic_limit); + req.SetCheckCRC64(true); + qcloud_cos::PutObjectResumableSingleSyncResp resp; + std::chrono::time_point start_ts, end_ts; + start_ts = std::chrono::steady_clock::now(); + qcloud_cos::CosResult result = m_client->PutObjectResumableSingleThreadSync(req, &resp); + EXPECT_TRUE(result.IsSucc()); + + // 3. 删除临时文件 + if (-1 == remove(filename.c_str())) { + std::cout << "Remove temp file=" << filename << " fail." << std::endl; + } + } + + { + std::string filename = "multi_upload_object_enc_one_step"; + std::string object_name = filename; + // 1. 生成个临时文件, 用于分块上传 + { + std::ofstream fs; + fs.open(filename.c_str(), std::ios::out | std::ios::binary); + std::string str(10 * 1000 * 1000, 'b'); + for (int idx = 0; idx < 10; ++idx) { + fs << str; + } + fs.close(); + } + + // 2. 上传 + MultiPutObjectReq req(m_bucket_name, object_name, filename); + req.SetXCosServerSideEncryption("AES256"); + MultiPutObjectResp resp; + + CosResult result = m_client->MultiPutObject(req, &resp); + ASSERT_TRUE(result.IsSucc()); + EXPECT_EQ("AES256", resp.GetXCosServerSideEncryption()); + + // 3. 删除临时文件 + if (-1 == remove(filename.c_str())) { + std::cout << "Remove temp file=" << filename << " fail." << std::endl; + } + } +} + TEST_F(ObjectOpTest, UploadPartCopyDataTest) { //上传一个对象 {