From b9a62a0216a712a14283ef23066ca725437313e4 Mon Sep 17 00:00:00 2001 From: imura81gt Date: Tue, 12 Nov 2019 23:30:47 +0900 Subject: [PATCH 01/16] [rget] first commit (base code) --- kadai3/imura81gt/rget/README.md | 9 +++ kadai3/imura81gt/rget/cmd/rget/main.go | 26 +++++++++ kadai3/imura81gt/rget/rget.go | 78 ++++++++++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 kadai3/imura81gt/rget/README.md create mode 100644 kadai3/imura81gt/rget/cmd/rget/main.go create mode 100644 kadai3/imura81gt/rget/rget.go diff --git a/kadai3/imura81gt/rget/README.md b/kadai3/imura81gt/rget/README.md new file mode 100644 index 0000000..0a55a5e --- /dev/null +++ b/kadai3/imura81gt/rget/README.md @@ -0,0 +1,9 @@ +rget +========================================================= + +Command +----------------------------------------- + +``` +go run cmd/rget/main.go https://upload.wikimedia.org/wikipedia/commons/1/16/Notocactus_minimus.jpg +``` diff --git a/kadai3/imura81gt/rget/cmd/rget/main.go b/kadai3/imura81gt/rget/cmd/rget/main.go new file mode 100644 index 0000000..5579539 --- /dev/null +++ b/kadai3/imura81gt/rget/cmd/rget/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/gopherdojo/dojo7/kadai3/imura81gt/rget" +) + +func main() { + option := rget.Option{ + Concurrency: *flag.Int("c", 10, "concurrency"), + } + flag.Parse() + urls := flag.Args() + if len(urls) != 1 { + fmt.Fprintf(os.Stderr, "%s \n", os.Args[0]) + fmt.Fprintln(os.Stderr, "option:") + flag.PrintDefaults() + os.Exit(1) + } + + option.Url = urls[0] + rget.Run(option) +} diff --git a/kadai3/imura81gt/rget/rget.go b/kadai3/imura81gt/rget/rget.go new file mode 100644 index 0000000..feffe5b --- /dev/null +++ b/kadai3/imura81gt/rget/rget.go @@ -0,0 +1,78 @@ +package rget + +import ( + "fmt" + "net/http" + "os" +) + +type Option struct { + Concurrency int + Url string +} + +type Unit struct { + RangeStart int64 + RangeEnd int64 +} + +type Units []Unit + +func Run(option Option) { + fmt.Printf("%+v\n", option) + cl, err := contentLength(option.Url) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + units := divide(cl, option.Concurrency) + + //TODO: check errors + download(units) + +} + +func contentLength(url string) (int64, error) { + resp, err := http.Head(url) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 0, err + } + + if resp.Header.Get("Accept-Ranges") == "" { + err := fmt.Errorf("This URL cannot support Ranges Requests") + // fmt.Fprintln(os.Stderr, err) + return resp.ContentLength, err + } + if resp.Header["Accept-Ranges"][0] == "none" { + err := fmt.Errorf("This URL cannot support Ranges Requests") + // fmt.Fprintln(os.Stderr, err) + return resp.ContentLength, err + } + if resp.ContentLength == 0 { + err := fmt.Errorf("This URL size is %i", resp.Header["Content-Length"][0]) + // fmt.Fprintln(os.Stderr, err) + return resp.ContentLength, err + } + + return resp.ContentLength, nil +} + +func divide(contentLength int64, concurrency int) Units { + var units []Unit + + sbyte := contentLength / int64(concurrency) + for i := 0; i < concurrency; i++ { + units = append(units, Unit{ + RangeStart: int64(i) * sbyte, + RangeEnd: int64((i+1))*sbyte - 1, + }) + } + + return units +} + +func download(units Units) { + fmt.Println(units) +} From 8298b442f0233ef155e2baef0d543eef707991b0 Mon Sep 17 00:00:00 2001 From: imura81gt Date: Sun, 1 Dec 2019 21:26:06 +0900 Subject: [PATCH 02/16] [rget] bug fix for option flags --- kadai3/imura81gt/rget/cmd/rget/main.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/kadai3/imura81gt/rget/cmd/rget/main.go b/kadai3/imura81gt/rget/cmd/rget/main.go index 5579539..31a845c 100644 --- a/kadai3/imura81gt/rget/cmd/rget/main.go +++ b/kadai3/imura81gt/rget/cmd/rget/main.go @@ -9,10 +9,14 @@ import ( ) func main() { + concurrency := flag.Int("c", 1, "concurrency") + outputDir := flag.String("o", "./", "output directory") + + flag.Parse() option := rget.Option{ - Concurrency: *flag.Int("c", 10, "concurrency"), + Concurrency: *concurrency, + OutputDir: *outputDir, } - flag.Parse() urls := flag.Args() if len(urls) != 1 { fmt.Fprintf(os.Stderr, "%s \n", os.Args[0]) @@ -21,6 +25,7 @@ func main() { os.Exit(1) } - option.Url = urls[0] + option.URL = urls[0] + fmt.Println(option) rget.Run(option) } From b0b49ec8a37eddae8282f2b5921a9a1827726012 Mon Sep 17 00:00:00 2001 From: imura81gt Date: Mon, 9 Dec 2019 01:14:41 +0900 Subject: [PATCH 03/16] [rget] add parallelDownload --- kadai3/imura81gt/rget/.go-version | 1 + kadai3/imura81gt/rget/README.md | 39 +++++++ kadai3/imura81gt/rget/go.mod | 5 + kadai3/imura81gt/rget/go.sum | 2 + kadai3/imura81gt/rget/rget.go | 161 ++++++++++++++++++++++++----- kadai3/imura81gt/rget/rget_test.go | 1 + 6 files changed, 181 insertions(+), 28 deletions(-) create mode 100644 kadai3/imura81gt/rget/.go-version create mode 100644 kadai3/imura81gt/rget/go.mod create mode 100644 kadai3/imura81gt/rget/go.sum create mode 100644 kadai3/imura81gt/rget/rget_test.go diff --git a/kadai3/imura81gt/rget/.go-version b/kadai3/imura81gt/rget/.go-version new file mode 100644 index 0000000..80138e7 --- /dev/null +++ b/kadai3/imura81gt/rget/.go-version @@ -0,0 +1 @@ +1.13.4 diff --git a/kadai3/imura81gt/rget/README.md b/kadai3/imura81gt/rget/README.md index 0a55a5e..8eeba68 100644 --- a/kadai3/imura81gt/rget/README.md +++ b/kadai3/imura81gt/rget/README.md @@ -7,3 +7,42 @@ Command ``` go run cmd/rget/main.go https://upload.wikimedia.org/wikipedia/commons/1/16/Notocactus_minimus.jpg ``` + +Theme +----------------------------------------- + +分割ダウンロードを行う + +元ネタ: https://qiita.com/codehex/items/d0a500ac387d39a34401 + +- [x]Rangeアクセスを用いる +- [ ]いくつかのゴルーチンでダウンロードしてマージする +- [x]エラー処理を工夫する +- [x]golang.org/x/sync/errgroupパッケージなどを使ってみる +- [x]キャンセルが発生した場合の実装を行う + +ref: https://qiita.com/codehex/items/d0a500ac387d39a34401 + + + +Note. +------------------------------------------ + +### Range Request + +https://developer.mozilla.org/ja/docs/Web/HTTP/Range_requests + +> Accept-Ranges が HTTP レスポンスに存在した場合 (そして値が "none" ではない場合)、サーバーは範囲リクエストに対応しています。これは例えば、 HEAD リクエストを cURL で発行することで確認することができます。 + + +https://developer.mozilla.org/ja/docs/Web/HTTP/Headers/Accept-Ranges + +> Accept-Ranges: bytes +> Accept-Ranges: none + +https://developer.mozilla.org/ja/docs/Web/HTTP/Headers/Range + +> Range: =- +> Range: =- +> Range: =-, - +> Range: =-, -, - diff --git a/kadai3/imura81gt/rget/go.mod b/kadai3/imura81gt/rget/go.mod new file mode 100644 index 0000000..47bbf1f --- /dev/null +++ b/kadai3/imura81gt/rget/go.mod @@ -0,0 +1,5 @@ +module github.com/gopherdojo/dojo7/kadai3/imura81gt/rget + +go 1.13 + +require golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e diff --git a/kadai3/imura81gt/rget/go.sum b/kadai3/imura81gt/rget/go.sum new file mode 100644 index 0000000..0d5bc22 --- /dev/null +++ b/kadai3/imura81gt/rget/go.sum @@ -0,0 +1,2 @@ +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/kadai3/imura81gt/rget/rget.go b/kadai3/imura81gt/rget/rget.go index feffe5b..d0a9051 100644 --- a/kadai3/imura81gt/rget/rget.go +++ b/kadai3/imura81gt/rget/rget.go @@ -1,78 +1,183 @@ package rget import ( + "context" "fmt" + "io" "net/http" "os" + "path" + + "golang.org/x/sync/errgroup" ) type Option struct { - Concurrency int - Url string + Concurrency int + URL string + OutputDir string + ContentLength int64 + Units Units } type Unit struct { - RangeStart int64 - RangeEnd int64 + RangeStart int64 + RangeEnd int64 + TempFileName string } type Units []Unit func Run(option Option) { fmt.Printf("%+v\n", option) - cl, err := contentLength(option.Url) + err := option.contentLength() if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) + fmt.Errorf("%s", err) } - units := divide(cl, option.Concurrency) + option.divide() - //TODO: check errors - download(units) + err = option.parallelDownload() + if err != nil { + fmt.Errorf("%s", err) + } } -func contentLength(url string) (int64, error) { - resp, err := http.Head(url) +func (o *Option) contentLength() error { + //resp, err := http.Head(url) + resp, err := http.Head(o.URL) if err != nil { fmt.Fprintln(os.Stderr, err) - return 0, err + //return 0, err + return err } if resp.Header.Get("Accept-Ranges") == "" { - err := fmt.Errorf("This URL cannot support Ranges Requests") + err := fmt.Errorf("%s URL cannot support Ranges Requests", o.URL) // fmt.Fprintln(os.Stderr, err) - return resp.ContentLength, err + //return resp.ContentLength, err + return err } if resp.Header["Accept-Ranges"][0] == "none" { - err := fmt.Errorf("This URL cannot support Ranges Requests") + err := fmt.Errorf("%s cannot support Ranges Requests", o.URL) // fmt.Fprintln(os.Stderr, err) - return resp.ContentLength, err + //return resp.ContentLength, err + return err } if resp.ContentLength == 0 { - err := fmt.Errorf("This URL size is %i", resp.Header["Content-Length"][0]) + err := fmt.Errorf("%s size is %s", o.URL, resp.Header["Content-Length"][0]) // fmt.Fprintln(os.Stderr, err) - return resp.ContentLength, err + //return resp.ContentLength, err + return err } - return resp.ContentLength, nil + o.ContentLength = resp.ContentLength + //return resp.ContentLength, nil + return err } -func divide(contentLength int64, concurrency int) Units { +//func divide(contentLength int64, concurrency int) Units { +func (o *Option) divide() { var units []Unit - sbyte := contentLength / int64(concurrency) - for i := 0; i < concurrency; i++ { + //sbyte := contentLength / int64(concurrency) + sbyte := o.ContentLength / int64(o.Concurrency) + + // for i := 0; i < concurrency; i++ { + for i := 0; i < o.Concurrency; i++ { units = append(units, Unit{ - RangeStart: int64(i) * sbyte, - RangeEnd: int64((i+1))*sbyte - 1, + RangeStart: int64(i) * sbyte, + RangeEnd: int64((i+1))*sbyte - 1, + TempFileName: fmt.Sprintf("%d_%s", i, path.Base(o.URL)), + }) + } + + o.Units = units + //return units +} + +// func download(units Units) { +// filepath.Split() +// fmt.Println(units) +// } + +func (o *Option) parallelDownload() error { + fmt.Println("parallelDownload", o.Units) + + eg, ctx := errgroup.WithContext(context.Background()) + for i := range o.Units { + // https://godoc.org/golang.org/x/sync/errgroup#example-Group--Parallel + // https://golang.org/doc/faq#closures_and_goroutines + i := i + eg.Go(func() error { + return o.downloadWithContext(ctx, i) }) } - return units + if err := eg.Wait(); err != nil { + o.clearCache() + return err + } + + return nil +} + +func (o *Option) downloadWithContext(ctx context.Context, i int) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + fmt.Printf("Downloading: %v %+v\n", i, o.Units[i]) + + //v1.13 + req, err := http.NewRequestWithContext(ctx, http.MethodGet, o.URL, nil) + if err != nil { + return fmt.Errorf("Error: %v", err) + } + + // add range header + fmt.Printf(fmt.Sprintf("bytes=%d-%d", o.Units[i].RangeStart, o.Units[i].RangeEnd)) + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", o.Units[i].RangeStart, o.Units[i].RangeEnd)) + + client := &http.Client{} + resp, err := client.Do(req) + defer resp.Body.Close() + if err != nil { + return fmt.Errorf("Error: %v", err) + } + + select { + case <-ctx.Done(): + fmt.Printf("Done: %v %+v\n", i, o.Units[i]) + return nil + default: + fmt.Println("default:", i, o.Units[i]) + //return fmt.Errorf("Error: %v %+v", i, o.Units[i]) + } + + w, err := os.Create(o.Units[i].TempFileName) + if err != nil { + return fmt.Errorf("Error: %v", err) + } + defer func() error { + if err := w.Close(); err != nil { + return fmt.Errorf("Error: %v", err) + } + return nil + }() + + _, err = io.Copy(w, resp.Body) + if err != nil { + return fmt.Errorf("Error: %v", err) + } + + return nil +} + +func (o *Option) conbine() error { + return nil } -func download(units Units) { - fmt.Println(units) +func (o *Option) clearCache() error { + //TODO: remove temporary files + return nil } diff --git a/kadai3/imura81gt/rget/rget_test.go b/kadai3/imura81gt/rget/rget_test.go new file mode 100644 index 0000000..c4f182d --- /dev/null +++ b/kadai3/imura81gt/rget/rget_test.go @@ -0,0 +1 @@ +package rget From 54921b1159f9108c5459b74702fea65563fb6477 Mon Sep 17 00:00:00 2001 From: imura81gt Date: Sat, 14 Dec 2019 23:57:54 +0900 Subject: [PATCH 04/16] [rget] fix default value:concurrency --- kadai3/imura81gt/rget/cmd/rget/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kadai3/imura81gt/rget/cmd/rget/main.go b/kadai3/imura81gt/rget/cmd/rget/main.go index 31a845c..5f94566 100644 --- a/kadai3/imura81gt/rget/cmd/rget/main.go +++ b/kadai3/imura81gt/rget/cmd/rget/main.go @@ -9,7 +9,7 @@ import ( ) func main() { - concurrency := flag.Int("c", 1, "concurrency") + concurrency := flag.Int("c", 2, "concurrency") outputDir := flag.String("o", "./", "output directory") flag.Parse() From 4618cbefc3fa248d11f916d15578ac07f6a308d9 Mon Sep 17 00:00:00 2001 From: imura81gt Date: Sun, 15 Dec 2019 21:48:08 +0900 Subject: [PATCH 05/16] [rget] download temp file to temp dir, add combine function --- kadai3/imura81gt/rget/rget.go | 85 ++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 31 deletions(-) diff --git a/kadai3/imura81gt/rget/rget.go b/kadai3/imura81gt/rget/rget.go index d0a9051..cc12a36 100644 --- a/kadai3/imura81gt/rget/rget.go +++ b/kadai3/imura81gt/rget/rget.go @@ -4,9 +4,11 @@ import ( "context" "fmt" "io" + "io/ioutil" "net/http" "os" "path" + "path/filepath" "golang.org/x/sync/errgroup" ) @@ -36,7 +38,19 @@ func Run(option Option) { option.divide() - err = option.parallelDownload() + tmpDir, err := ioutil.TempDir("", "rget") + if err != nil { + fmt.Errorf("%s", err) + } + defer os.RemoveAll(tmpDir) + fmt.Println(tmpDir) + + err = option.parallelDownload(tmpDir) + if err != nil { + fmt.Errorf("%s", err) + } + + err = option.combine(tmpDir) if err != nil { fmt.Errorf("%s", err) } @@ -47,27 +61,19 @@ func (o *Option) contentLength() error { //resp, err := http.Head(url) resp, err := http.Head(o.URL) if err != nil { - fmt.Fprintln(os.Stderr, err) - //return 0, err return err } if resp.Header.Get("Accept-Ranges") == "" { err := fmt.Errorf("%s URL cannot support Ranges Requests", o.URL) - // fmt.Fprintln(os.Stderr, err) - //return resp.ContentLength, err return err } if resp.Header["Accept-Ranges"][0] == "none" { err := fmt.Errorf("%s cannot support Ranges Requests", o.URL) - // fmt.Fprintln(os.Stderr, err) - //return resp.ContentLength, err return err } if resp.ContentLength == 0 { err := fmt.Errorf("%s size is %s", o.URL, resp.Header["Content-Length"][0]) - // fmt.Fprintln(os.Stderr, err) - //return resp.ContentLength, err return err } @@ -80,10 +86,9 @@ func (o *Option) contentLength() error { func (o *Option) divide() { var units []Unit - //sbyte := contentLength / int64(concurrency) + //TODO: if o.ContentLength < int64(o.Concurrency) sbyte := o.ContentLength / int64(o.Concurrency) - // for i := 0; i < concurrency; i++ { for i := 0; i < o.Concurrency; i++ { units = append(units, Unit{ RangeStart: int64(i) * sbyte, @@ -92,16 +97,13 @@ func (o *Option) divide() { }) } + // TODO: should distribute the remainder to each unit + units[len(units)-1].RangeEnd = o.ContentLength + o.Units = units - //return units } -// func download(units Units) { -// filepath.Split() -// fmt.Println(units) -// } - -func (o *Option) parallelDownload() error { +func (o *Option) parallelDownload(tmpDir string) error { fmt.Println("parallelDownload", o.Units) eg, ctx := errgroup.WithContext(context.Background()) @@ -110,19 +112,22 @@ func (o *Option) parallelDownload() error { // https://golang.org/doc/faq#closures_and_goroutines i := i eg.Go(func() error { - return o.downloadWithContext(ctx, i) + return o.downloadWithContext(ctx, i, tmpDir) }) } if err := eg.Wait(); err != nil { - o.clearCache() return err } return nil } -func (o *Option) downloadWithContext(ctx context.Context, i int) error { +func (o *Option) downloadWithContext( + ctx context.Context, + i int, + dir string, +) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -135,15 +140,16 @@ func (o *Option) downloadWithContext(ctx context.Context, i int) error { } // add range header - fmt.Printf(fmt.Sprintf("bytes=%d-%d", o.Units[i].RangeStart, o.Units[i].RangeEnd)) + fmt.Printf(fmt.Sprintf("bytes=%d-%d\n", o.Units[i].RangeStart, o.Units[i].RangeEnd)) req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", o.Units[i].RangeStart, o.Units[i].RangeEnd)) - client := &http.Client{} + client := http.DefaultClient resp, err := client.Do(req) - defer resp.Body.Close() if err != nil { + fmt.Printf("client err: %s", err) return fmt.Errorf("Error: %v", err) } + defer resp.Body.Close() select { case <-ctx.Done(): @@ -151,10 +157,9 @@ func (o *Option) downloadWithContext(ctx context.Context, i int) error { return nil default: fmt.Println("default:", i, o.Units[i]) - //return fmt.Errorf("Error: %v %+v", i, o.Units[i]) } - w, err := os.Create(o.Units[i].TempFileName) + w, err := os.Create(filepath.Join(dir, o.Units[i].TempFileName)) if err != nil { return fmt.Errorf("Error: %v", err) } @@ -173,11 +178,29 @@ func (o *Option) downloadWithContext(ctx context.Context, i int) error { return nil } -func (o *Option) conbine() error { - return nil -} +func (o *Option) combine(dir string) error { + w, err := os.Create(path.Base(o.URL)) + if err != nil { + return fmt.Errorf("Error: %v", err) + } + defer func() error { + if err := w.Close(); err != nil { + return fmt.Errorf("Error: %v", err) + } + return nil + }() + + for _, unit := range o.Units { + r, err := os.Open(filepath.Join(dir, unit.TempFileName)) + if err != nil { + return fmt.Errorf("Error: %v", err) + } + + _, err = io.Copy(w, r) + if err != nil { + return fmt.Errorf("Error: %v", err) + } + } -func (o *Option) clearCache() error { - //TODO: remove temporary files return nil } From d932f91ef44a4a581c7289daf1136ed94ccbafb4 Mon Sep 17 00:00:00 2001 From: imura81gt Date: Sat, 21 Dec 2019 23:26:45 +0900 Subject: [PATCH 06/16] bugfix: return error if context is canceled. --- kadai3/imura81gt/rget/rget.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kadai3/imura81gt/rget/rget.go b/kadai3/imura81gt/rget/rget.go index cc12a36..13449e8 100644 --- a/kadai3/imura81gt/rget/rget.go +++ b/kadai3/imura81gt/rget/rget.go @@ -140,8 +140,9 @@ func (o *Option) downloadWithContext( } // add range header - fmt.Printf(fmt.Sprintf("bytes=%d-%d\n", o.Units[i].RangeStart, o.Units[i].RangeEnd)) - req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", o.Units[i].RangeStart, o.Units[i].RangeEnd)) + byteRange := fmt.Sprintf("bytes=%d-%d", o.Units[i].RangeStart, o.Units[i].RangeEnd) + fmt.Println(byteRange) + req.Header.Set("Range", byteRange) client := http.DefaultClient resp, err := client.Do(req) @@ -154,7 +155,7 @@ func (o *Option) downloadWithContext( select { case <-ctx.Done(): fmt.Printf("Done: %v %+v\n", i, o.Units[i]) - return nil + return fmt.Errorf("Error: %v", err) default: fmt.Println("default:", i, o.Units[i]) } From afac7082cc24f484eec2f240e14830ab6d941612 Mon Sep 17 00:00:00 2001 From: imura81gt Date: Sat, 21 Dec 2019 23:34:00 +0900 Subject: [PATCH 07/16] fix: change type: concurrency is uint --- kadai3/imura81gt/rget/cmd/rget/main.go | 2 +- kadai3/imura81gt/rget/rget.go | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/kadai3/imura81gt/rget/cmd/rget/main.go b/kadai3/imura81gt/rget/cmd/rget/main.go index 5f94566..3c2c62f 100644 --- a/kadai3/imura81gt/rget/cmd/rget/main.go +++ b/kadai3/imura81gt/rget/cmd/rget/main.go @@ -9,7 +9,7 @@ import ( ) func main() { - concurrency := flag.Int("c", 2, "concurrency") + concurrency := flag.Uint("c", 2, "concurrency") outputDir := flag.String("o", "./", "output directory") flag.Parse() diff --git a/kadai3/imura81gt/rget/rget.go b/kadai3/imura81gt/rget/rget.go index 13449e8..9f5354e 100644 --- a/kadai3/imura81gt/rget/rget.go +++ b/kadai3/imura81gt/rget/rget.go @@ -14,7 +14,7 @@ import ( ) type Option struct { - Concurrency int + Concurrency uint URL string OutputDir string ContentLength int64 @@ -86,10 +86,17 @@ func (o *Option) contentLength() error { func (o *Option) divide() { var units []Unit - //TODO: if o.ContentLength < int64(o.Concurrency) + if o.Concurrency == 0 { + o.Concurrency = 1 + } + + if o.ContentLength < int64(o.Concurrency) { + o.Concurrency = uint(o.ContentLength) + } + sbyte := o.ContentLength / int64(o.Concurrency) - for i := 0; i < o.Concurrency; i++ { + for i := 0; i < int(o.Concurrency); i++ { units = append(units, Unit{ RangeStart: int64(i) * sbyte, RangeEnd: int64((i+1))*sbyte - 1, From 3202b1d1153c10d3e1a66099349dc4073f8e25b0 Mon Sep 17 00:00:00 2001 From: imura81gt Date: Sat, 21 Dec 2019 23:42:01 +0900 Subject: [PATCH 08/16] add testing for divid and bugfix --- kadai3/imura81gt/rget/go.mod | 5 +- kadai3/imura81gt/rget/go.sum | 2 + kadai3/imura81gt/rget/rget.go | 19 +++-- kadai3/imura81gt/rget/rget_test.go | 114 +++++++++++++++++++++++++++++ 4 files changed, 134 insertions(+), 6 deletions(-) diff --git a/kadai3/imura81gt/rget/go.mod b/kadai3/imura81gt/rget/go.mod index 47bbf1f..a46d6f7 100644 --- a/kadai3/imura81gt/rget/go.mod +++ b/kadai3/imura81gt/rget/go.mod @@ -2,4 +2,7 @@ module github.com/gopherdojo/dojo7/kadai3/imura81gt/rget go 1.13 -require golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e +require ( + github.com/google/go-cmp v0.3.1 + golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e +) diff --git a/kadai3/imura81gt/rget/go.sum b/kadai3/imura81gt/rget/go.sum index 0d5bc22..33b824b 100644 --- a/kadai3/imura81gt/rget/go.sum +++ b/kadai3/imura81gt/rget/go.sum @@ -1,2 +1,4 @@ +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/kadai3/imura81gt/rget/rget.go b/kadai3/imura81gt/rget/rget.go index 9f5354e..974b904 100644 --- a/kadai3/imura81gt/rget/rget.go +++ b/kadai3/imura81gt/rget/rget.go @@ -22,9 +22,18 @@ type Option struct { } type Unit struct { - RangeStart int64 - RangeEnd int64 - TempFileName string + RangeStart int64 + RangeEnd int64 + TempFileName string + DownloadedSize int64 +} + +func (u *Unit) Write(data []byte) (int, error) { + d := len(data) + u.DownloadedSize += int64(d) + // fmt.Printf("%v is downloaded %v/%v \n", + // u.TempFileName, u.DownloadedSize, u.RangeEnd-u.RangeStart+1) + return d, nil } type Units []Unit @@ -105,7 +114,7 @@ func (o *Option) divide() { } // TODO: should distribute the remainder to each unit - units[len(units)-1].RangeEnd = o.ContentLength + units[len(units)-1].RangeEnd = (o.ContentLength - 1) o.Units = units } @@ -178,7 +187,7 @@ func (o *Option) downloadWithContext( return nil }() - _, err = io.Copy(w, resp.Body) + _, err = io.Copy(w, io.TeeReader(resp.Body, &o.Units[i])) if err != nil { return fmt.Errorf("Error: %v", err) } diff --git a/kadai3/imura81gt/rget/rget_test.go b/kadai3/imura81gt/rget/rget_test.go index c4f182d..7718827 100644 --- a/kadai3/imura81gt/rget/rget_test.go +++ b/kadai3/imura81gt/rget/rget_test.go @@ -1 +1,115 @@ package rget + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestWrite(t *testing.T) { +} + +func TestRun(t *testing.T) { +} + +func TestContentLength(t *testing.T) { +} + +func TestAcceptRangesHeaderCheck(t *testing.T) { +} + +//func divide(contentLength int64, concurrency int) Units { +func TestDivide(t *testing.T) { + + const ( + url = "https://example.com/test.iso" + contentLength = 5 + ) + + testCases := []struct { + caseName string + concurrency uint + expected Option + }{ + { + caseName: "ContentLength:5/5", + concurrency: 5, + expected: Option{ + URL: url, + ContentLength: contentLength, + Concurrency: 5, + Units: []Unit{ + {TempFileName: "0_test.iso", RangeStart: 0, RangeEnd: 0}, + {TempFileName: "1_test.iso", RangeStart: 1, RangeEnd: 1}, + {TempFileName: "2_test.iso", RangeStart: 2, RangeEnd: 2}, + {TempFileName: "3_test.iso", RangeStart: 3, RangeEnd: 3}, + {TempFileName: "4_test.iso", RangeStart: 4, RangeEnd: 4}, + }, + }, + }, + { + caseName: "ContentLength:5/1", + concurrency: 1, + expected: Option{ + URL: url, + ContentLength: contentLength, + Concurrency: 1, + Units: []Unit{ + {TempFileName: "0_test.iso", RangeStart: 0, RangeEnd: 4}, + }, + }, + }, + { + caseName: "ContentLength:5/10", + concurrency: 10, + expected: Option{ + URL: url, + ContentLength: contentLength, + Concurrency: 5, + Units: []Unit{ + {TempFileName: "0_test.iso", RangeStart: 0, RangeEnd: 0}, + {TempFileName: "1_test.iso", RangeStart: 1, RangeEnd: 1}, + {TempFileName: "2_test.iso", RangeStart: 2, RangeEnd: 2}, + {TempFileName: "3_test.iso", RangeStart: 3, RangeEnd: 3}, + {TempFileName: "4_test.iso", RangeStart: 4, RangeEnd: 4}, + }, + }, + }, + { + caseName: "ContentLength:5/0", + concurrency: 0, + expected: Option{ + URL: url, + ContentLength: contentLength, + Concurrency: 1, + Units: []Unit{ + {TempFileName: "0_test.iso", RangeStart: 0, RangeEnd: 4}, + }, + }, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.caseName, func(t *testing.T) { + t.Parallel() + + o := Option{URL: url, ContentLength: contentLength, Concurrency: tc.concurrency} + o.divide() + + if !cmp.Equal(o, tc.expected) { + t.Errorf("actual: %+v\nexpected: %+v\n", o, tc.expected) + } + + }) + } +} + +func TestParallelDownload(t *testing.T) { +} + +func TestDownloadWithContext(t *testing.T) { +} + +func TestCombine(t *testing.T) { +} From d00487ac695aec9145ddd9ab414c4947533eca8d Mon Sep 17 00:00:00 2001 From: imura81gt Date: Sat, 21 Dec 2019 23:58:17 +0900 Subject: [PATCH 09/16] fix testCase Name with TestDivide --- kadai3/imura81gt/rget/rget_test.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/kadai3/imura81gt/rget/rget_test.go b/kadai3/imura81gt/rget/rget_test.go index 7718827..a5b3a4e 100644 --- a/kadai3/imura81gt/rget/rget_test.go +++ b/kadai3/imura81gt/rget/rget_test.go @@ -32,7 +32,7 @@ func TestDivide(t *testing.T) { expected Option }{ { - caseName: "ContentLength:5/5", + caseName: "ContentLength and Concurrency is same value", concurrency: 5, expected: Option{ URL: url, @@ -48,7 +48,7 @@ func TestDivide(t *testing.T) { }, }, { - caseName: "ContentLength:5/1", + caseName: "One Thread", concurrency: 1, expected: Option{ URL: url, @@ -60,7 +60,20 @@ func TestDivide(t *testing.T) { }, }, { - caseName: "ContentLength:5/10", + caseName: "Remainder:ContentLength%Concurrency!=0", + concurrency: 2, + expected: Option{ + URL: url, + ContentLength: contentLength, + Concurrency: 2, + Units: []Unit{ + {TempFileName: "0_test.iso", RangeStart: 0, RangeEnd: 1}, + {TempFileName: "1_test.iso", RangeStart: 2, RangeEnd: 4}, + }, + }, + }, + { + caseName: "Concurrency exceed the contentLength.", concurrency: 10, expected: Option{ URL: url, From 0f1c350d1e115b7fe54a3e6777cb991d5817467e Mon Sep 17 00:00:00 2001 From: imura81gt Date: Sun, 22 Dec 2019 22:08:31 +0900 Subject: [PATCH 10/16] [rget]rename function --- kadai3/imura81gt/rget/rget.go | 20 ++++++++++++++++---- kadai3/imura81gt/rget/rget_test.go | 5 +---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/kadai3/imura81gt/rget/rget.go b/kadai3/imura81gt/rget/rget.go index 974b904..06d1b97 100644 --- a/kadai3/imura81gt/rget/rget.go +++ b/kadai3/imura81gt/rget/rget.go @@ -40,7 +40,7 @@ type Units []Unit func Run(option Option) { fmt.Printf("%+v\n", option) - err := option.contentLength() + err := option.checkingHeaders() if err != nil { fmt.Errorf("%s", err) } @@ -66,8 +66,7 @@ func Run(option Option) { } -func (o *Option) contentLength() error { - //resp, err := http.Head(url) +func (o *Option) checkingHeaders() error { resp, err := http.Head(o.URL) if err != nil { return err @@ -77,17 +76,30 @@ func (o *Option) contentLength() error { err := fmt.Errorf("%s URL cannot support Ranges Requests", o.URL) return err } + if resp.Header["Accept-Ranges"][0] == "none" { err := fmt.Errorf("%s cannot support Ranges Requests", o.URL) return err } + if resp.ContentLength == 0 { err := fmt.Errorf("%s size is %s", o.URL, resp.Header["Content-Length"][0]) return err } + redirectURL := resp.Request.URL.String() + if err != nil { + return err + } + o.ContentLength = resp.ContentLength - //return resp.ContentLength, nil + + // keep the redirect URL that accept Ranges Requests because some mirror sites may deny. + // TODO: redirectURL should set by Unit separately. + if o.URL != redirectURL { + o.URL = redirectURL + } + return err } diff --git a/kadai3/imura81gt/rget/rget_test.go b/kadai3/imura81gt/rget/rget_test.go index a5b3a4e..fb81414 100644 --- a/kadai3/imura81gt/rget/rget_test.go +++ b/kadai3/imura81gt/rget/rget_test.go @@ -12,10 +12,7 @@ func TestWrite(t *testing.T) { func TestRun(t *testing.T) { } -func TestContentLength(t *testing.T) { -} - -func TestAcceptRangesHeaderCheck(t *testing.T) { +func TestCheckingHeaders(t *testing.T) { } //func divide(contentLength int64, concurrency int) Units { From b6cb6555e56dcbb2c5680316e4ea0099770f03e0 Mon Sep 17 00:00:00 2001 From: imura81gt Date: Sun, 22 Dec 2019 22:09:47 +0900 Subject: [PATCH 11/16] [rget]fix error handling --- kadai3/imura81gt/rget/cmd/rget/main.go | 6 +++++- kadai3/imura81gt/rget/cmd/rget/main_test.go | 5 +++++ kadai3/imura81gt/rget/rget.go | 13 ++++++++----- 3 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 kadai3/imura81gt/rget/cmd/rget/main_test.go diff --git a/kadai3/imura81gt/rget/cmd/rget/main.go b/kadai3/imura81gt/rget/cmd/rget/main.go index 3c2c62f..720479b 100644 --- a/kadai3/imura81gt/rget/cmd/rget/main.go +++ b/kadai3/imura81gt/rget/cmd/rget/main.go @@ -27,5 +27,9 @@ func main() { option.URL = urls[0] fmt.Println(option) - rget.Run(option) + err := rget.Run(option) + if err != nil { + fmt.Fprintf(os.Stderr, "err: %s", err) + os.Exit(1) + } } diff --git a/kadai3/imura81gt/rget/cmd/rget/main_test.go b/kadai3/imura81gt/rget/cmd/rget/main_test.go new file mode 100644 index 0000000..097f27a --- /dev/null +++ b/kadai3/imura81gt/rget/cmd/rget/main_test.go @@ -0,0 +1,5 @@ +package main + +import "testing" + +func TestMain(t *testing.T) {} diff --git a/kadai3/imura81gt/rget/rget.go b/kadai3/imura81gt/rget/rget.go index 06d1b97..22de07e 100644 --- a/kadai3/imura81gt/rget/rget.go +++ b/kadai3/imura81gt/rget/rget.go @@ -38,32 +38,33 @@ func (u *Unit) Write(data []byte) (int, error) { type Units []Unit -func Run(option Option) { +func Run(option Option) error { fmt.Printf("%+v\n", option) err := option.checkingHeaders() if err != nil { - fmt.Errorf("%s", err) + return fmt.Errorf("%s", err) } option.divide() tmpDir, err := ioutil.TempDir("", "rget") if err != nil { - fmt.Errorf("%s", err) + return fmt.Errorf("%s", err) } defer os.RemoveAll(tmpDir) fmt.Println(tmpDir) err = option.parallelDownload(tmpDir) if err != nil { - fmt.Errorf("%s", err) + return fmt.Errorf("%s", err) } err = option.combine(tmpDir) if err != nil { - fmt.Errorf("%s", err) + return fmt.Errorf("%s", err) } + return nil } func (o *Option) checkingHeaders() error { @@ -173,6 +174,8 @@ func (o *Option) downloadWithContext( req.Header.Set("Range", byteRange) client := http.DefaultClient + // TODO: should check resp.StatusCode. + // client.Do cannot seems to return the err when statusCode is 50x etc. resp, err := client.Do(req) if err != nil { fmt.Printf("client err: %s", err) From 277514a20ada7c173f425ce204618e13b3895b6b Mon Sep 17 00:00:00 2001 From: imura81gt Date: Tue, 24 Dec 2019 00:34:48 +0900 Subject: [PATCH 12/16] [rget] add test: TestCheckingHeaders --- kadai3/imura81gt/rget/rget.go | 11 +++--- kadai3/imura81gt/rget/rget_test.go | 55 ++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/kadai3/imura81gt/rget/rget.go b/kadai3/imura81gt/rget/rget.go index 22de07e..e1f9219 100644 --- a/kadai3/imura81gt/rget/rget.go +++ b/kadai3/imura81gt/rget/rget.go @@ -74,24 +74,21 @@ func (o *Option) checkingHeaders() error { } if resp.Header.Get("Accept-Ranges") == "" { - err := fmt.Errorf("%s URL cannot support Ranges Requests", o.URL) + err := fmt.Errorf("%s : %s cannot support Ranges Requests", o.URL, resp.Request.URL.String()) return err } if resp.Header["Accept-Ranges"][0] == "none" { - err := fmt.Errorf("%s cannot support Ranges Requests", o.URL) + err := fmt.Errorf("%s : %s cannot support Ranges Requests", o.URL, resp.Request.URL.String()) return err } - if resp.ContentLength == 0 { - err := fmt.Errorf("%s size is %s", o.URL, resp.Header["Content-Length"][0]) + if resp.Header["Content-Length"] == nil { + err := fmt.Errorf("%s size is nil", o.URL) return err } redirectURL := resp.Request.URL.String() - if err != nil { - return err - } o.ContentLength = resp.ContentLength diff --git a/kadai3/imura81gt/rget/rget_test.go b/kadai3/imura81gt/rget/rget_test.go index fb81414..c5639eb 100644 --- a/kadai3/imura81gt/rget/rget_test.go +++ b/kadai3/imura81gt/rget/rget_test.go @@ -1,6 +1,10 @@ package rget import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -13,6 +17,46 @@ func TestRun(t *testing.T) { } func TestCheckingHeaders(t *testing.T) { + testCases := []struct { + caseName string + acceptRanges string + body string + isErr bool + expected string + }{ + {caseName: "acceptRanges:none", acceptRanges: "none", body: "1", isErr: true, expected: "cannot support Ranges Requests"}, + {caseName: "acceptRanges:(empty)", acceptRanges: "", body: "1", isErr: true, expected: "cannot support Ranges Requests"}, + {caseName: "acceptRanges:bytes", acceptRanges: "bytes", body: "1", isErr: false}, + {caseName: "acceptRanges:bytes but content is empty", acceptRanges: "bytes", body: "", isErr: true, expected: "size is nil"}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.caseName, func(t *testing.T) { + t.Parallel() + + ts := SetupHhttpServer(t, tc.acceptRanges, tc.body) + defer ts.Close() + + // resp, _ := http.Get(ts.URL) + // t.Logf("resp: %+v", resp) + + o := Option{URL: ts.URL} + err := o.checkingHeaders() + t.Logf("err: %+v", err) + + if tc.isErr && tc.body != "" && err == nil { + t.Errorf("actual: %+v\nexpected: %+v\n", err, tc.isErr) + } + if tc.isErr && tc.body != "" && !strings.Contains(err.Error(), tc.expected) { + t.Errorf("actual: %+v\nexpected: %+v\n", err, tc.expected) + } + if tc.isErr && tc.body == "" && !strings.Contains(err.Error(), tc.expected) { + t.Errorf("actual: %+v\nexpected: %+v\n", err, tc.isErr) + } + }) + } + } //func divide(contentLength int64, concurrency int) Units { @@ -123,3 +167,14 @@ func TestDownloadWithContext(t *testing.T) { func TestCombine(t *testing.T) { } + +func SetupHhttpServer(t *testing.T, ac string, body string) *httptest.Server { + t.Helper() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Accept-Ranges", ac) + if body != "" { + fmt.Fprintln(w, body) + } + })) + return ts +} From 3dd553895c6a73e9b8a54becda10c5f3222d5b26 Mon Sep 17 00:00:00 2001 From: imura81gt Date: Wed, 25 Dec 2019 23:50:27 +0900 Subject: [PATCH 13/16] [rget] add test:TestParallelDownload / refactor test:TestCheckingHeaders --- kadai3/imura81gt/rget/rget.go | 2 +- kadai3/imura81gt/rget/rget_test.go | 119 +++++++++++++++++++++++++---- 2 files changed, 105 insertions(+), 16 deletions(-) diff --git a/kadai3/imura81gt/rget/rget.go b/kadai3/imura81gt/rget/rget.go index e1f9219..ac9cac2 100644 --- a/kadai3/imura81gt/rget/rget.go +++ b/kadai3/imura81gt/rget/rget.go @@ -83,7 +83,7 @@ func (o *Option) checkingHeaders() error { return err } - if resp.Header["Content-Length"] == nil { + if resp.ContentLength == 0 { err := fmt.Errorf("%s size is nil", o.URL) return err } diff --git a/kadai3/imura81gt/rget/rget_test.go b/kadai3/imura81gt/rget/rget_test.go index c5639eb..5533408 100644 --- a/kadai3/imura81gt/rget/rget_test.go +++ b/kadai3/imura81gt/rget/rget_test.go @@ -2,10 +2,14 @@ package rget import ( "fmt" + "io/ioutil" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" + "time" "github.com/google/go-cmp/cmp" ) @@ -35,24 +39,25 @@ func TestCheckingHeaders(t *testing.T) { t.Run(tc.caseName, func(t *testing.T) { t.Parallel() - ts := SetupHhttpServer(t, tc.acceptRanges, tc.body) + ts := SetupHTTPServer(t, tc.acceptRanges, tc.body) defer ts.Close() // resp, _ := http.Get(ts.URL) // t.Logf("resp: %+v", resp) - o := Option{URL: ts.URL} - err := o.checkingHeaders() - t.Logf("err: %+v", err) - - if tc.isErr && tc.body != "" && err == nil { - t.Errorf("actual: %+v\nexpected: %+v\n", err, tc.isErr) + resp, err := http.Head(ts.URL) + if err != nil { + t.Fatal(err) } - if tc.isErr && tc.body != "" && !strings.Contains(err.Error(), tc.expected) { - t.Errorf("actual: %+v\nexpected: %+v\n", err, tc.expected) + t.Logf("ContentLength: %+v,Accept-Ranges: %+v", resp.ContentLength, resp.Header.Get("Accept-Ranges")) + + o := Option{URL: ts.URL} + exerr := o.checkingHeaders() + if tc.isErr && exerr == nil { + t.Errorf("tc.isErr %+v but err is %+v", tc.isErr, exerr) } - if tc.isErr && tc.body == "" && !strings.Contains(err.Error(), tc.expected) { - t.Errorf("actual: %+v\nexpected: %+v\n", err, tc.isErr) + if tc.isErr && exerr != nil && !strings.Contains(exerr.Error(), tc.expected) { + t.Errorf("actual: %+v\nexpected: %+v\n", exerr, tc.isErr) } }) } @@ -160,6 +165,85 @@ func TestDivide(t *testing.T) { } func TestParallelDownload(t *testing.T) { + + type Expected []struct { + TempFileName string + Text string + } + + testCases := []struct { + caseName string + acceptRanges string + body string + option Option + expected Expected + }{ + { + caseName: "acceptRanges:bytes", acceptRanges: "bytes", body: "12345", + option: Option{ + Units: []Unit{ + {TempFileName: "0_test.iso", RangeStart: 0, RangeEnd: 0}, + {TempFileName: "1_test.iso", RangeStart: 1, RangeEnd: 1}, + {TempFileName: "2_test.iso", RangeStart: 2, RangeEnd: 2}, + {TempFileName: "3_test.iso", RangeStart: 3, RangeEnd: 3}, + {TempFileName: "4_test.iso", RangeStart: 4, RangeEnd: 4}, + }, + }, + expected: Expected{ + {TempFileName: "0_test.iso", Text: "1"}, + {TempFileName: "1_test.iso", Text: "2"}, + {TempFileName: "2_test.iso", Text: "3"}, + {TempFileName: "3_test.iso", Text: "4"}, + {TempFileName: "4_test.iso", Text: "5"}, + }, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.caseName, func(t *testing.T) { + t.Parallel() + + prefix := "rget_test" + tmpDir, err := ioutil.TempDir("", prefix) + if err != nil { + t.Fatal(err) + } + + ts := SetupHTTPServer(t, tc.acceptRanges, tc.body) + defer ts.Close() + + tc.option.URL = ts.URL + resp, err := http.Head(tc.option.URL) + if err != nil { + t.Fatal(err) + } + + tc.option.ContentLength = resp.ContentLength + + err = tc.option.parallelDownload(tmpDir) + if err != nil { + t.Fatal(err) + } + + for _, ex := range tc.expected { + f, err := os.Open(filepath.Join(tmpDir, ex.TempFileName)) + if err != nil { + t.Fatal(err) + } + + actual, err := ioutil.ReadAll(f) + if ex.Text != string(actual) { + t.Errorf("actual: %+v\nexpected: %+v\n", string(actual), ex.Text) + } + + } + // t.Log(tmpDir) + os.RemoveAll(tmpDir) + + }) + } + } func TestDownloadWithContext(t *testing.T) { @@ -168,13 +252,18 @@ func TestDownloadWithContext(t *testing.T) { func TestCombine(t *testing.T) { } -func SetupHhttpServer(t *testing.T, ac string, body string) *httptest.Server { +func SetupHTTPServer(t *testing.T, ac string, body string) *httptest.Server { t.Helper() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Accept-Ranges", ac) - if body != "" { - fmt.Fprintln(w, body) + // the unsupported server for Range request + if ac != "bytes" { + w.Header().Set("Accept-Ranges", ac) + fmt.Fprint(w, body) + } else { + // the supported server for Range request + http.ServeContent(w, r, "", time.Unix(0, 0), strings.NewReader(body)) } + })) return ts } From 2ff92aa83e311d17b582cede3caa29f97b83695c Mon Sep 17 00:00:00 2001 From: imura81gt Date: Thu, 26 Dec 2019 00:03:05 +0900 Subject: [PATCH 14/16] [rget] add testCase for TestParallelDownload --- kadai3/imura81gt/rget/rget_test.go | 40 +++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/kadai3/imura81gt/rget/rget_test.go b/kadai3/imura81gt/rget/rget_test.go index 5533408..fe86a72 100644 --- a/kadai3/imura81gt/rget/rget_test.go +++ b/kadai3/imura81gt/rget/rget_test.go @@ -179,7 +179,7 @@ func TestParallelDownload(t *testing.T) { expected Expected }{ { - caseName: "acceptRanges:bytes", acceptRanges: "bytes", body: "12345", + caseName: "acceptRanges:bytes per 1byte", acceptRanges: "bytes", body: "12345", option: Option{ Units: []Unit{ {TempFileName: "0_test.iso", RangeStart: 0, RangeEnd: 0}, @@ -197,6 +197,44 @@ func TestParallelDownload(t *testing.T) { {TempFileName: "4_test.iso", Text: "5"}, }, }, + { + caseName: "acceptRanges:bytes per 2bytes", acceptRanges: "bytes", body: "0123456789", + option: Option{ + Units: []Unit{ + {TempFileName: "0_test.iso", RangeStart: 0, RangeEnd: 1}, + {TempFileName: "1_test.iso", RangeStart: 2, RangeEnd: 3}, + {TempFileName: "2_test.iso", RangeStart: 4, RangeEnd: 5}, + {TempFileName: "3_test.iso", RangeStart: 6, RangeEnd: 7}, + {TempFileName: "4_test.iso", RangeStart: 8, RangeEnd: 9}, + }, + }, + expected: Expected{ + {TempFileName: "0_test.iso", Text: "01"}, + {TempFileName: "1_test.iso", Text: "23"}, + {TempFileName: "2_test.iso", Text: "45"}, + {TempFileName: "3_test.iso", Text: "67"}, + {TempFileName: "4_test.iso", Text: "89"}, + }, + }, + { + caseName: "acceptRanges:bytes per 2bytes+1", acceptRanges: "bytes", body: "01234567890", + option: Option{ + Units: []Unit{ + {TempFileName: "0_test.iso", RangeStart: 0, RangeEnd: 1}, + {TempFileName: "1_test.iso", RangeStart: 2, RangeEnd: 3}, + {TempFileName: "2_test.iso", RangeStart: 4, RangeEnd: 5}, + {TempFileName: "3_test.iso", RangeStart: 6, RangeEnd: 7}, + {TempFileName: "4_test.iso", RangeStart: 8, RangeEnd: 10}, + }, + }, + expected: Expected{ + {TempFileName: "0_test.iso", Text: "01"}, + {TempFileName: "1_test.iso", Text: "23"}, + {TempFileName: "2_test.iso", Text: "45"}, + {TempFileName: "3_test.iso", Text: "67"}, + {TempFileName: "4_test.iso", Text: "890"}, + }, + }, } for _, tc := range testCases { From 14a1c9b260091d26adf8c49eaadfbe0cacdcf388 Mon Sep 17 00:00:00 2001 From: imura81gt Date: Thu, 26 Dec 2019 00:46:29 +0900 Subject: [PATCH 15/16] [rget] add testcase for TestCheckingHeaders --- kadai3/imura81gt/rget/rget_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kadai3/imura81gt/rget/rget_test.go b/kadai3/imura81gt/rget/rget_test.go index fe86a72..3f62798 100644 --- a/kadai3/imura81gt/rget/rget_test.go +++ b/kadai3/imura81gt/rget/rget_test.go @@ -62,6 +62,15 @@ func TestCheckingHeaders(t *testing.T) { }) } + t.Run("Invalid URL", func(t *testing.T) { + oEmpty := Option{URL: "s3://example.com"} + exerr := oEmpty.checkingHeaders() + t.Log(exerr) + if exerr == nil { + t.Error("invalid URL but err is nil") + } + }) + } //func divide(contentLength int64, concurrency int) Units { From a22a0167083fb5064a2aff66c223065300a099a7 Mon Sep 17 00:00:00 2001 From: imura81gt Date: Thu, 26 Dec 2019 22:57:14 +0900 Subject: [PATCH 16/16] [rget] fix to specify save dir --- kadai3/imura81gt/rget/rget.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kadai3/imura81gt/rget/rget.go b/kadai3/imura81gt/rget/rget.go index ac9cac2..4fc49ae 100644 --- a/kadai3/imura81gt/rget/rget.go +++ b/kadai3/imura81gt/rget/rget.go @@ -208,7 +208,7 @@ func (o *Option) downloadWithContext( } func (o *Option) combine(dir string) error { - w, err := os.Create(path.Base(o.URL)) + w, err := os.Create(filepath.Join(o.OutputDir, path.Base(o.URL))) if err != nil { return fmt.Errorf("Error: %v", err) }