From 9a1310bee84ea0da733c02733fba4331a4cd6f0b Mon Sep 17 00:00:00 2001 From: kaznishi Date: Thu, 5 Jul 2018 08:24:44 +0900 Subject: [PATCH 1/3] =?UTF-8?q?WIP=20=E4=B8=8B=E8=AA=BF=E3=81=B9=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/main.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 kadai3-2/main.go diff --git a/kadai3-2/main.go b/kadai3-2/main.go new file mode 100644 index 0000000..30c5d6d --- /dev/null +++ b/kadai3-2/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "fmt" + "time" + "context" + "golang.org/x/net/context/ctxhttp" + "net/http" + "os" +) + +func main() { + url := "http://example.com" + dirname := "/tmp/pdownload" + ctx, _ := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) + fmt.Println(url) + res, _ := ctxhttp.Head(ctx, http.DefaultClient, url) + fmt.Println(res.Header.Get("Accept-Ranges")) + fmt.Println(res.Header.Get("Content-Length")) + os.MkdirAll(dirname, 0755) + req, _ := http.NewRequest("GET", url, nil) + low := 0 + high := 605 + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", low, high)) + res, _ = http.DefaultClient.Do(req) + + + return +} From 15b8d8a1661d7ed1129293b64d91da99349d7702 Mon Sep 17 00:00:00 2001 From: kaznishi Date: Wed, 11 Jul 2018 06:42:04 +0900 Subject: [PATCH 2/3] =?UTF-8?q?WIP=20=E7=B4=B0=E3=80=85=E3=81=A8=E3=81=97?= =?UTF-8?q?=E3=81=9F=E5=87=A6=E7=90=86=E3=82=92=E5=AE=9F=E8=A3=85=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/main.go | 144 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 135 insertions(+), 9 deletions(-) diff --git a/kadai3-2/main.go b/kadai3-2/main.go index 30c5d6d..cd81e1a 100644 --- a/kadai3-2/main.go +++ b/kadai3-2/main.go @@ -1,16 +1,23 @@ package main import ( - "fmt" - "time" "context" - "golang.org/x/net/context/ctxhttp" + "fmt" + "io" "net/http" "os" + "time" + + "golang.org/x/net/context/ctxhttp" + "strconv" +) + +const( + tmpdir = "/tmp/kaznishi_pdownload" ) func main() { - url := "http://example.com" + url := "http://abehiroshi.la.coocan.jp/abe-top2-4.jpg" dirname := "/tmp/pdownload" ctx, _ := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) fmt.Println(url) @@ -18,12 +25,131 @@ func main() { fmt.Println(res.Header.Get("Accept-Ranges")) fmt.Println(res.Header.Get("Content-Length")) os.MkdirAll(dirname, 0755) - req, _ := http.NewRequest("GET", url, nil) - low := 0 - high := 605 - req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", low, high)) - res, _ = http.DefaultClient.Do(req) + // req, _ := http.NewRequest("GET", url, nil) + // low := 0 + // high := 65172 + // req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", low, high)) + // res, _ = http.DefaultClient.Do(req) + + // ファイルを作って画像を保存する + // file, _ := os.OpenFile(dirname+"/abehiroshi.jpg", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + // defer file.Close() + // io.Copy(file, res.Body) + + // 2分割でファイルをダウンロードして、それぞれダウンロードし終わった後にマージファイルを作る + req1, _ := http.NewRequest("GET", url, nil) + low1 := 0 + high1 := 39999 + req1.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", low1, high1)) + res1, _ := http.DefaultClient.Do(req1) + file1, _ := os.OpenFile(dirname+"/abehiroshi.jpg-p1", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + io.Copy(file1, res1.Body) + file1.Close() + + req2, _ := http.NewRequest("GET", url, nil) + low2 := 40000 + high2 := 65172 + req2.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", low2, high2)) + res2, _ := http.DefaultClient.Do(req2) + file2, _ := os.OpenFile(dirname+"/abehiroshi.jpg-p2", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + io.Copy(file2, res2.Body) + file2.Close() + + file1, _ = os.Open(dirname + "/abehiroshi.jpg-p1") + file2, _ = os.Open(dirname + "/abehiroshi.jpg-p2") + file, _ := os.OpenFile(dirname+"/abehiroshi.jpg", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + io.Copy(file, file1) + io.Copy(file, file2) + file.Close() + + // 提出用に整えていく + //// チェック処理 + //// ファイルサイズ分割処理 + //// 分割ファイル名 + //// 分割ダウンロード処理 + //// 分割ファイルマージ処理 + //// 分割ファイルクリア処理 + //// goroutine使って書き換え + //// キャンセル時処理書く + //// テスト書く + + // return } + +func prepare () { + os.MkdirAll(tmpdir, 0755) +} + +func sizeCheck(url string) (int, error) { + ctx, _ := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) + res, err := ctxhttp.Head(ctx, http.DefaultClient, url) + if err != nil { + return 0, err + } + if res.Header.Get("Accept-Ranges") != "bytes" { + return 0, err // ここは自分でエラーを定義するように直す + } + + l, err := strconv.Atoi(res.Header.Get("Content-Length")) + return l, err +} + + +type part struct{ + Low int + High int + Filename string +} + +func split (pcount int, fullsize int, filename string) [...]part { + var result [pcount]part + + var low, high int + for i := 0; i < pcount; i++ { + if i == 0 { + low = 0 + } else { + low = high + 1 + } + if i == pcount - 1 { + high = fullsize + } else { + high = int(fullsize * (i+1) / pcount) + } + fn := filename + "_" + strconv.Itoa(i) + p := part{Low: low, High: high, Filename: fn} + result[i] = p + } + return result +} + +func download(p part, url string) { + req, _ := http.NewRequest("GET", url, nil) + low := p.Low + high := p.High + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", low, high)) + res, _ := http.DefaultClient.Do(req) + file, _ := os.OpenFile(tmpdir+"/"+p.Filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + io.Copy(file, res.Body) + file.Close() + return +} + +func merge (parts [...]part, newFilePath string) { + newFile, _ := os.OpenFile(newFilePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + for _, p := range parts { + pf, _ := os.Open(tmpdir + "/" + p.Filename) + io.Copy(newFile, pf) + pf.Close() + } + newFile.Close() +} + +func clearPartFiles (parts [...]part) { + for _, p := range parts { + os.Remove(tmpdir + "/" + p.Filename) + } +} \ No newline at end of file From 00d681636551ef8e439a084cabdd142bd5d80e94 Mon Sep 17 00:00:00 2001 From: kaznishi Date: Mon, 16 Jul 2018 11:25:07 +0900 Subject: [PATCH 3/3] =?UTF-8?q?=E4=B8=80=E9=80=9A=E3=82=8A=E5=AE=8C?= =?UTF-8?q?=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/README.md | 15 +++ kadai3-2/main.go | 155 ---------------------- kadai3-2/pdownload.go | 67 ++++++++++ kadai3-2/pdownload/pdownload.go | 223 ++++++++++++++++++++++++++++++++ 4 files changed, 305 insertions(+), 155 deletions(-) create mode 100644 kadai3-2/README.md delete mode 100644 kadai3-2/main.go create mode 100644 kadai3-2/pdownload.go create mode 100644 kadai3-2/pdownload/pdownload.go diff --git a/kadai3-2/README.md b/kadai3-2/README.md new file mode 100644 index 0000000..63e0657 --- /dev/null +++ b/kadai3-2/README.md @@ -0,0 +1,15 @@ +## Usage + +### 基本的な使い方 + +``` +pdownload [URL] +``` + +#### オプション + +``` +-p ダウンロード分割数の指定(デフォルト: 5) +-o 出力先ディレクトリの指定(デフォルト: ".") +-t 分割ファイル一時格納先の指定(デフォルト: "/tmp/kaznishi_pdownload") +``` diff --git a/kadai3-2/main.go b/kadai3-2/main.go deleted file mode 100644 index cd81e1a..0000000 --- a/kadai3-2/main.go +++ /dev/null @@ -1,155 +0,0 @@ -package main - -import ( - "context" - "fmt" - "io" - "net/http" - "os" - "time" - - "golang.org/x/net/context/ctxhttp" - "strconv" -) - -const( - tmpdir = "/tmp/kaznishi_pdownload" -) - -func main() { - url := "http://abehiroshi.la.coocan.jp/abe-top2-4.jpg" - dirname := "/tmp/pdownload" - ctx, _ := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) - fmt.Println(url) - res, _ := ctxhttp.Head(ctx, http.DefaultClient, url) - fmt.Println(res.Header.Get("Accept-Ranges")) - fmt.Println(res.Header.Get("Content-Length")) - os.MkdirAll(dirname, 0755) - // req, _ := http.NewRequest("GET", url, nil) - // low := 0 - // high := 65172 - // req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", low, high)) - // res, _ = http.DefaultClient.Do(req) - - // ファイルを作って画像を保存する - // file, _ := os.OpenFile(dirname+"/abehiroshi.jpg", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) - // defer file.Close() - // io.Copy(file, res.Body) - - // 2分割でファイルをダウンロードして、それぞれダウンロードし終わった後にマージファイルを作る - req1, _ := http.NewRequest("GET", url, nil) - low1 := 0 - high1 := 39999 - req1.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", low1, high1)) - res1, _ := http.DefaultClient.Do(req1) - file1, _ := os.OpenFile(dirname+"/abehiroshi.jpg-p1", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) - io.Copy(file1, res1.Body) - file1.Close() - - req2, _ := http.NewRequest("GET", url, nil) - low2 := 40000 - high2 := 65172 - req2.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", low2, high2)) - res2, _ := http.DefaultClient.Do(req2) - file2, _ := os.OpenFile(dirname+"/abehiroshi.jpg-p2", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) - io.Copy(file2, res2.Body) - file2.Close() - - file1, _ = os.Open(dirname + "/abehiroshi.jpg-p1") - file2, _ = os.Open(dirname + "/abehiroshi.jpg-p2") - - file, _ := os.OpenFile(dirname+"/abehiroshi.jpg", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) - io.Copy(file, file1) - io.Copy(file, file2) - file.Close() - - // 提出用に整えていく - //// チェック処理 - //// ファイルサイズ分割処理 - //// 分割ファイル名 - //// 分割ダウンロード処理 - //// 分割ファイルマージ処理 - //// 分割ファイルクリア処理 - //// goroutine使って書き換え - //// キャンセル時処理書く - //// テスト書く - - // - - return -} - -func prepare () { - os.MkdirAll(tmpdir, 0755) -} - -func sizeCheck(url string) (int, error) { - ctx, _ := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) - res, err := ctxhttp.Head(ctx, http.DefaultClient, url) - if err != nil { - return 0, err - } - if res.Header.Get("Accept-Ranges") != "bytes" { - return 0, err // ここは自分でエラーを定義するように直す - } - - l, err := strconv.Atoi(res.Header.Get("Content-Length")) - return l, err -} - - -type part struct{ - Low int - High int - Filename string -} - -func split (pcount int, fullsize int, filename string) [...]part { - var result [pcount]part - - var low, high int - for i := 0; i < pcount; i++ { - if i == 0 { - low = 0 - } else { - low = high + 1 - } - if i == pcount - 1 { - high = fullsize - } else { - high = int(fullsize * (i+1) / pcount) - } - fn := filename + "_" + strconv.Itoa(i) - p := part{Low: low, High: high, Filename: fn} - result[i] = p - } - return result -} - -func download(p part, url string) { - req, _ := http.NewRequest("GET", url, nil) - low := p.Low - high := p.High - req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", low, high)) - res, _ := http.DefaultClient.Do(req) - file, _ := os.OpenFile(tmpdir+"/"+p.Filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) - io.Copy(file, res.Body) - file.Close() - return -} - -func merge (parts [...]part, newFilePath string) { - newFile, _ := os.OpenFile(newFilePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) - for _, p := range parts { - pf, _ := os.Open(tmpdir + "/" + p.Filename) - io.Copy(newFile, pf) - pf.Close() - } - newFile.Close() -} - -func clearPartFiles (parts [...]part) { - for _, p := range parts { - os.Remove(tmpdir + "/" + p.Filename) - } -} \ No newline at end of file diff --git a/kadai3-2/pdownload.go b/kadai3-2/pdownload.go new file mode 100644 index 0000000..4fcc1ff --- /dev/null +++ b/kadai3-2/pdownload.go @@ -0,0 +1,67 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "sync" + "syscall" + + "github.com/gopherdojo/dojo2/kadai3-2/pdownload" +) + +var ( + pCountOpt = flag.Int("p", 5, "分割数") + outputDirOpt = flag.String("o", ".", "ダウンロードファイルの出力先ディレクトリ") + tmpDirOpt = flag.String("t", "/tmp/kaznishi_pdownload", "分割ファイルの一時格納ディレクトリ") +) + +func main() { + option := pdownload.Option{} + option.Init() + + flag.Parse() + if len(flag.Args()) == 0 { + fmt.Fprintf(os.Stderr, "ダウンロード対象のURLが指定されていません") + return + } + option.TargetURL = flag.Args()[0] + option.PCount = *pCountOpt + option.OutputDir = *outputDirOpt + option.TmpDir = *tmpDirOpt + + ctx, cancel := context.WithCancel(context.Background()) + + trapSignals := []os.Signal{ + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGQUIT, + } + sigCh := make(chan os.Signal, 1) + doneCh := make(chan int, 1) + signal.Notify(sigCh, trapSignals...) + + var wgMain sync.WaitGroup + go func() { + wgMain.Add(1) + if err := pdownload.Run(ctx, doneCh, option); err != nil { + fmt.Println(err) + } + wgMain.Done() + }() + select { + case sig := <-sigCh: + cancel() + wgMain.Wait() + fmt.Println("Got signal", sig) + case code := <-doneCh: + if code == 0 { + fmt.Println("Done!!!!!") + } else { + fmt.Println("Failed...") + } + } +} diff --git a/kadai3-2/pdownload/pdownload.go b/kadai3-2/pdownload/pdownload.go new file mode 100644 index 0000000..5a85dda --- /dev/null +++ b/kadai3-2/pdownload/pdownload.go @@ -0,0 +1,223 @@ +package pdownload + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path" + "strconv" + "time" + + "golang.org/x/net/context/ctxhttp" + "golang.org/x/sync/errgroup" +) + +// Option はプログラムに与えるオプションをまとめた構造体です +type Option struct { + TargetURL string // ダウンロードの対象URL + PCount int // 分割数 + OutputDir string // 結合後のファイルの格納場所 + TmpDir string // 分割ファイルの一時格納場所 +} + +// Init は新しく生成したオブジェクトにデフォルト値を設定するための関数です +func (o *Option) Init() { + o.PCount = 5 + o.OutputDir = "." + o.TmpDir = "/tmp/kaznishi_pdownload" +} + +var ( + tmpDir = "/tmp/kaznishi_pdownload" // 分割ファイルを一時的に格納するディレクトリ +) + +// Run はpdownloadの処理を実行します +func Run(ctx context.Context, doneCh chan<- int, option Option) error { + errCh := make(chan error, 1) + eg, _ := errgroup.WithContext(ctx) + + // 一時保存ディレクトリの作成 + setTmpDir(option.TmpDir) + if err := mkTmpDir(); err != nil { + doneCh <- 1 + return err + } + //// チェック処理 + fullSize, err := sizeCheck(option.TargetURL) + if err != nil { + doneCh <- 1 + return err + } + //// ファイルサイズ分割処理 + fileName := path.Base(option.TargetURL) + parts := split(option.PCount, fullSize, fileName) + + go func() { + //// 分割ダウンロード処理 + for _, p := range parts { + fmt.Println("Downloding Part File Started. :" + p.FileName) + p := p + eg.Go(func() error { + return download(p, option.TargetURL) + }) + } + if err := eg.Wait(); err != nil { + errCh <- err + } else { + fmt.Println("Downloading Part Files completed.") + } + + //// 分割ファイルマージ処理 + if err := merge(parts, getNewFilePath(option.OutputDir, fileName)); err != nil { + errCh <- err + } else { + fmt.Println("Combining Part Files completed.") + } + + //// 分割ファイルクリア処理 + if err = clearPartFiles(parts); err != nil { + errCh <- err + } + + errCh <- nil + }() + + for { + select { + case err := <-errCh: + if err != nil { + clearWhenCancel(parts, getNewFilePath(option.OutputDir, fileName)) + doneCh <- 1 + return err + } + doneCh <- 0 + return nil + case <-ctx.Done(): + clearWhenCancel(parts, getNewFilePath(option.OutputDir, fileName)) + doneCh <- 0 + return nil + } + } +} + +/////////////////////////////////////////////////////////////////////////// + +func getNewFilePath(outputDir, fileName string) string { + return outputDir + "/" + fileName +} + +func setTmpDir(dirPath string) { + if dirPath != "" { + tmpDir = dirPath + } +} + +func mkTmpDir() error { + return os.MkdirAll(tmpDir, 0755) +} + +func sizeCheck(url string) (int, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) + defer cancel() + res, err := ctxhttp.Head(ctx, http.DefaultClient, url) + if err != nil { + return 0, err + } + if res.Header.Get("Accept-Ranges") != "bytes" { + err = fmt.Errorf("Accept-Ranges = bytesではありません") + return 0, err + } + + l, err := strconv.Atoi(res.Header.Get("Content-Length")) + return l, err +} + +type part struct { + Low int + High int + FileName string +} + +func (p part) getFilePath() string { + return tmpDir + "/" + p.FileName +} + +func split(pCount int, fullSize int, fileName string) []part { + result := make([]part, pCount) + + var low, high int + for i := 0; i < pCount; i++ { + if i == 0 { + low = 0 + } else { + low = high + 1 + } + if i == pCount-1 { + high = fullSize - 1 + } else { + high = int(fullSize * (i + 1) / pCount) + } + fn := fileName + "_" + strconv.Itoa(i) + p := part{Low: low, High: high, FileName: fn} + result[i] = p + } + return result +} + +func download(p part, url string) error { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + + low := p.Low + high := p.High + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", low, high)) + + res, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + file, err := os.Create(p.getFilePath()) + if err != nil { + return err + } + + _, err = io.Copy(file, res.Body) + if err != nil { + return err + } + + return file.Close() +} + +func merge(parts []part, newFilePath string) error { + newFile, _ := os.Create(newFilePath) + for _, p := range parts { + pf, err := os.Open(p.getFilePath()) + if err != nil { + return err + } + io.Copy(newFile, pf) + pf.Close() + } + newFile.Close() + return nil +} + +func clearPartFiles(parts []part) error { + for _, p := range parts { + if err := os.Remove(p.getFilePath()); err != nil { + return err + } + } + return nil +} + +func clearWhenCancel(parts []part, newFilePath string) { + clearPartFiles(parts) + os.Remove(newFilePath) +}