Devel branch

conf/config.ini

@@ -30,12 +30,16 @@ https=off
 # 默认存储在工具下的tmp目录, 请确保磁盘空间充足，取决于要迁移的文件的大小与并发度。
 tmpFolder=./tmp
 # 小文件阈值的字节，大于等于这个阈值使用分块上传，否则使用简单上传, 默认5MB
+# 注意：最大能上传5GB的小文件
 smallFileThreshold=5242880
 # 小文件(文件小于smallFileThreshold)的并发度，使用简单上传，此值对于带宽充足或小文件过多时，可以适当增大调整为128或者256等。
 # 同时如果从友商或者URL迁移，下载是使用的线程池并发度也由smallFileExecutorNum来决定，因此可以通过增大下载速度。
 smallFileExecutorNum=64
 # 大文件(文件大于等于smallFileThreshold)的并发度，使用分块上传,此值不宜过大，建议不大于32
 bigFileExecutorNum=8
+# 用来指定分块上传时单个分块的大小, 单位字节,，默认分块大小是5MB
+# 由于分块上传对单个文件块的数目有最大限制（10000块），所以对于超出5MB*10000大小的文件，需要根据具体情况调整该参数 
+bigFileUploadPartSize=5242880
 # 表示迁移工具将全文的MD5计算后，存入文件的自定义头部x-cos-meta-md5中, 用于后续的校验，因为COS的分块上传的大文件的etag不是全文的md5
 # on 打开, off关闭
 entireFileMd5Attached=on

result/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

src/main/java/com/qcloud/cos_migrate_tool/config/CommonConfig.java

@@ -24,6 +24,7 @@ public class CommonConfig {
     private StorageClass storageClass = StorageClass.Standard;
     private int smallFileExecutorNumber = 64;
     private int bigFileExecutorNum = 4;
+    private long bigFileUploadPartSize = 5 * 1024 * 1024;
     private long smallFileThreshold = 5 * 1024 * 1024;
     private boolean damonMode = false;
     private long damonInterVal = 60;
@@ -414,4 +415,12 @@ public void setOutputFinishedFilePath(String path) throws IllegalArgumentExcepti
     public String getOutputFinishedFilePath() {
         return this.outputFinishedFilePath;
     }
+
+    public long getBigFileUploadPartSize() {
+        return bigFileUploadPartSize;
+    }
+
+    public void setBigFileUploadPartSize(long bigFileUploadPartSize) {
+        this.bigFileUploadPartSize = bigFileUploadPartSize;
+    }
 }

src/main/java/com/qcloud/cos_migrate_tool/config/ConfigParser.java

@@ -42,6 +42,7 @@ public class ConfigParser {
     private static final String COMMON_STORAGE_CLASS = "storageClass";
     private static final String COMMON_SMALL_FILE_EXECUTOR_NUM = "smallFileExecutorNum";
     private static final String COMMON_BIG_FILE_EXECUTOR_NUM = "bigFileExecutorNum";
+    private static final String COMMON_BIG_FILE_UPLOAD_PART_SIZE = "bigFileUploadPartSize";
     private static final String COMMON_ENTIRE_FILE_MD5_ATTACHED = "entireFileMd5Attached";
     private static final String COMMON_DAEMON_MODE = "daemonMode";
     private static final String COMMON_DAEMON_MODE_INTERVAL = "daemonModeInterVal";
@@ -497,6 +498,12 @@ private boolean initCommonConfig(Preferences prefs, CommonConfig commonConfig) {
             assert (bigFileExecutorNumberStr != null);
             commonConfig.setBigFileUploadExecutorNum(bigFileExecutorNumberStr);
 
+            String bigFileUploadPartSizeStr =
+                    getConfigValue(prefs, COMMON_SECTION_NAME, COMMON_BIG_FILE_UPLOAD_PART_SIZE);
+            if(bigFileUploadPartSizeStr != null) {
+                commonConfig.setBigFileUploadPartSize(Long.parseLong(bigFileUploadPartSizeStr));
+            }
+
             String storageClassStr =
                     getConfigValue(prefs, COMMON_SECTION_NAME, COMMON_STORAGE_CLASS);
             assert (storageClassStr != null);

src/main/java/com/qcloud/cos_migrate_tool/task/Task.java

@@ -1,5 +1,6 @@
 package com.qcloud.cos_migrate_tool.task;
 
+import java.io.ByteArrayInputStream;
 import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -204,24 +205,39 @@ private String uploadBigFile(PutObjectRequest putObjectRequest) throws Interrupt
 
     private String uploadSmallFile(PutObjectRequest putObjectRequest) throws InterruptedException {
         Upload upload = smallFileTransfer.upload(putObjectRequest);
-        return showTransferProgressAndGetRequestId(upload, false, putObjectRequest.getKey(),
-                putObjectRequest.getFile().lastModified());
+        File file = putObjectRequest.getFile();
+        long mtime = 0;
+        if(file != null){
+            mtime = file.lastModified();
+        }
+        return showTransferProgressAndGetRequestId(upload, false, putObjectRequest.getKey(), mtime);
     }
 
 
     public String uploadFile(String bucketName, String cosPath, File localFile,
             StorageClass storageClass, boolean entireMd5Attached, ObjectMetadata objectMetadata,
             AccessControlList acl) throws Exception {
-        PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, cosPath, localFile);
+        PutObjectRequest putObjectRequest;
+        if(localFile.isDirectory()) {
+            if(!cosPath.endsWith("/")) {
+                cosPath += "/";
+            }
+            byte[] contentByteArray = new byte[0];
+            objectMetadata.setContentType("application/x-directory");
+            objectMetadata.setContentLength(0);
+            putObjectRequest = new PutObjectRequest(bucketName, cosPath, new ByteArrayInputStream(contentByteArray),
+                    objectMetadata);
+        } else {
+            putObjectRequest = new PutObjectRequest(bucketName, cosPath, localFile);
+        }
         putObjectRequest.setStorageClass(storageClass);
 
         if (acl != null) {
             putObjectRequest.setAccessControlList(acl);
         }
 
-        if (entireMd5Attached) {
+        if (entireMd5Attached && !localFile.isDirectory()) {
             String md5 = Md5Utils.md5Hex(localFile);
-
             String upyunTag = objectMetadata.getUserMetaDataOf("upyun-etag");
             if (upyunTag != null) {
                 if (!md5.equalsIgnoreCase(upyunTag)) {

src/main/java/com/qcloud/cos_migrate_tool/task/TaskExecutor.java

@@ -87,6 +87,9 @@ public TaskExecutor(MigrateType migrateType, CommonConfig config) {
 
         this.bigFileTransferManager.getConfiguration()
                 .setMultipartUploadThreshold(config.getSmallFileThreshold());
+
+        this.bigFileTransferManager.getConfiguration().
+                setMinimumUploadPartSize(config.getBigFileUploadPartSize());
     }
 
     protected abstract String buildTaskDbComment();