diff --git a/conf/config.ini b/conf/config.ini index 67caf5f..455dee5 100644 --- a/conf/config.ini +++ b/conf/config.ini @@ -30,12 +30,16 @@ https=off # 默认存储在工具下的tmp目录, 请确保磁盘空间充足，取决于要迁移的文件的大小与并发度。 tmpFolder=./tmp # 小文件阈值的字节，大于等于这个阈值使用分块上传，否则使用简单上传, 默认5MB +# 注意：最大能上传5GB的小文件 smallFileThreshold=5242880 # 小文件(文件小于smallFileThreshold)的并发度，使用简单上传，此值对于带宽充足或小文件过多时，可以适当增大调整为128或者256等。 # 同时如果从友商或者URL迁移，下载是使用的线程池并发度也由smallFileExecutorNum来决定，因此可以通过增大下载速度。 smallFileExecutorNum=64 # 大文件(文件大于等于smallFileThreshold)的并发度，使用分块上传,此值不宜过大，建议不大于32 bigFileExecutorNum=8 +# 用来指定分块上传时单个分块的大小, 单位字节,，默认分块大小是5MB +# 由于分块上传对单个文件块的数目有最大限制（10000块），所以对于超出5MB*10000大小的文件，需要根据具体情况调整该参数 +bigFileUploadPartSize=5242880 # 表示迁移工具将全文的MD5计算后，存入文件的自定义头部x-cos-meta-md5中, 用于后续的校验，因为COS的分块上传的大文件的etag不是全文的md5 # on 打开, off关闭 entireFileMd5Attached=on diff --git a/dep/cos_migrate_tool-1.0-jar-with-dependencies.jar b/dep/cos_migrate_tool-1.0-jar-with-dependencies.jar index cbf502d..1248bff 100644 Binary files a/dep/cos_migrate_tool-1.0-jar-with-dependencies.jar and b/dep/cos_migrate_tool-1.0-jar-with-dependencies.jar differ diff --git a/result/.gitignore b/result/.gitignore new file mode 100644 index 0000000..5e7d273 --- /dev/null +++ b/result/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/src/main/java/com/qcloud/cos_migrate_tool/config/CommonConfig.java b/src/main/java/com/qcloud/cos_migrate_tool/config/CommonConfig.java index 4ed9d1c..8fe8e5a 100644 --- a/src/main/java/com/qcloud/cos_migrate_tool/config/CommonConfig.java +++ b/src/main/java/com/qcloud/cos_migrate_tool/config/CommonConfig.java @@ -24,6 +24,7 @@ public class CommonConfig { private StorageClass storageClass = StorageClass.Standard; private int smallFileExecutorNumber = 64; private int bigFileExecutorNum = 4; + private long bigFileUploadPartSize = 5 * 1024 * 1024; private long smallFileThreshold = 5 * 1024 * 1024; private boolean damonMode = false; private long damonInterVal = 60; @@ -414,4 +415,12 @@ public void setOutputFinishedFilePath(String path) throws IllegalArgumentExcepti public String getOutputFinishedFilePath() { return this.outputFinishedFilePath; } + + public long getBigFileUploadPartSize() { + return bigFileUploadPartSize; + } + + public void setBigFileUploadPartSize(long bigFileUploadPartSize) { + this.bigFileUploadPartSize = bigFileUploadPartSize; + } } diff --git a/src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java b/src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java index 08cf32a..7441a64 100644 --- a/src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java +++ b/src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java @@ -42,6 +42,7 @@ public class ConfigParser { private static final String COMMON_STORAGE_CLASS = "storageClass"; private static final String COMMON_SMALL_FILE_EXECUTOR_NUM = "smallFileExecutorNum"; private static final String COMMON_BIG_FILE_EXECUTOR_NUM = "bigFileExecutorNum"; + private static final String COMMON_BIG_FILE_UPLOAD_PART_SIZE = "bigFileUploadPartSize"; private static final String COMMON_ENTIRE_FILE_MD5_ATTACHED = "entireFileMd5Attached"; private static final String COMMON_DAEMON_MODE = "daemonMode"; private static final String COMMON_DAEMON_MODE_INTERVAL = "daemonModeInterVal"; @@ -497,6 +498,12 @@ private boolean initCommonConfig(Preferences prefs, CommonConfig commonConfig) { assert (bigFileExecutorNumberStr != null); commonConfig.setBigFileUploadExecutorNum(bigFileExecutorNumberStr); + String bigFileUploadPartSizeStr = + getConfigValue(prefs, COMMON_SECTION_NAME, COMMON_BIG_FILE_UPLOAD_PART_SIZE); + if(bigFileUploadPartSizeStr != null) { + commonConfig.setBigFileUploadPartSize(Long.parseLong(bigFileUploadPartSizeStr)); + } + String storageClassStr = getConfigValue(prefs, COMMON_SECTION_NAME, COMMON_STORAGE_CLASS); assert (storageClassStr != null); diff --git a/src/main/java/com/qcloud/cos_migrate_tool/task/Task.java b/src/main/java/com/qcloud/cos_migrate_tool/task/Task.java index b1d619d..d10dd49 100644 --- a/src/main/java/com/qcloud/cos_migrate_tool/task/Task.java +++ b/src/main/java/com/qcloud/cos_migrate_tool/task/Task.java @@ -1,5 +1,6 @@ package com.qcloud.cos_migrate_tool.task; +import java.io.ByteArrayInputStream; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileNotFoundException; @@ -204,24 +205,39 @@ private String uploadBigFile(PutObjectRequest putObjectRequest) throws Interrupt private String uploadSmallFile(PutObjectRequest putObjectRequest) throws InterruptedException { Upload upload = smallFileTransfer.upload(putObjectRequest); - return showTransferProgressAndGetRequestId(upload, false, putObjectRequest.getKey(), - putObjectRequest.getFile().lastModified()); + File file = putObjectRequest.getFile(); + long mtime = 0; + if(file != null){ + mtime = file.lastModified(); + } + return showTransferProgressAndGetRequestId(upload, false, putObjectRequest.getKey(), mtime); } public String uploadFile(String bucketName, String cosPath, File localFile, StorageClass storageClass, boolean entireMd5Attached, ObjectMetadata objectMetadata, AccessControlList acl) throws Exception { - PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, cosPath, localFile); + PutObjectRequest putObjectRequest; + if(localFile.isDirectory()) { + if(!cosPath.endsWith("/")) { + cosPath += "/"; + } + byte[] contentByteArray = new byte[0]; + objectMetadata.setContentType("application/x-directory"); + objectMetadata.setContentLength(0); + putObjectRequest = new PutObjectRequest(bucketName, cosPath, new ByteArrayInputStream(contentByteArray), + objectMetadata); + } else { + putObjectRequest = new PutObjectRequest(bucketName, cosPath, localFile); + } putObjectRequest.setStorageClass(storageClass); if (acl != null) { putObjectRequest.setAccessControlList(acl); } - if (entireMd5Attached) { + if (entireMd5Attached && !localFile.isDirectory()) { String md5 = Md5Utils.md5Hex(localFile); - String upyunTag = objectMetadata.getUserMetaDataOf("upyun-etag"); if (upyunTag != null) { if (!md5.equalsIgnoreCase(upyunTag)) { diff --git a/src/main/java/com/qcloud/cos_migrate_tool/task/TaskExecutor.java b/src/main/java/com/qcloud/cos_migrate_tool/task/TaskExecutor.java index 775d9f8..faa686b 100644 --- a/src/main/java/com/qcloud/cos_migrate_tool/task/TaskExecutor.java +++ b/src/main/java/com/qcloud/cos_migrate_tool/task/TaskExecutor.java @@ -87,6 +87,9 @@ public TaskExecutor(MigrateType migrateType, CommonConfig config) { this.bigFileTransferManager.getConfiguration() .setMultipartUploadThreshold(config.getSmallFileThreshold()); + + this.bigFileTransferManager.getConfiguration(). + setMinimumUploadPartSize(config.getBigFileUploadPartSize()); } protected abstract String buildTaskDbComment();