From 4170fecd046d08b13ff7619f954b78d5296e9e16 Mon Sep 17 00:00:00 2001 From: suaaa7 Date: Sun, 1 Sep 2019 00:53:56 +0900 Subject: [PATCH 1/4] Add pkg/download --- kadai3-2/pei/pkg/download/download.go | 171 ++++++++++++++++++++++++++ kadai3-2/pei/pkg/download/file.go | 20 +++ 2 files changed, 191 insertions(+) create mode 100644 kadai3-2/pei/pkg/download/download.go create mode 100644 kadai3-2/pei/pkg/download/file.go diff --git a/kadai3-2/pei/pkg/download/download.go b/kadai3-2/pei/pkg/download/download.go new file mode 100644 index 0000000..d0e8d64 --- /dev/null +++ b/kadai3-2/pei/pkg/download/download.go @@ -0,0 +1,171 @@ +package download + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + + "golang.org/x/sync/errgroup" +) + +// Downloader Interface +type Downloader interface { + Do() (string, error) +} + +// NonRangeDownloader has info for downloading +type NonRangeDownloader struct { + url, outputPath string +} + +// RangeDownloader has info for split downloading +type RangeDownloader struct { + splitNum int + ranges []*Range + url, outputPath string +} + +// Range has rangestart and rangeend +type Range struct { + start int64 + end int64 +} + +// NewDownloader creates Downloader +func NewDownloader(splitNum int, url, outputPath string) (Downloader, error) { + dir, fileName := parseDirAndFileName(outputPath) + if fileName == "" { + outputPath = filepath.Join(dir, parseFileName(url)) + } + + res, err := http.Head(url) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.Header.Get("Accept-Ranges") != "bytes" { + return &NonRangeDownloader{url: url, outputPath: outputPath}, nil + } + + contentLength := res.ContentLength + unit := contentLength / int64(splitNum) + ranges := make([]*Range, splitNum) + + for i := range ranges { + var start, end int64 + if i != 0 { + start = int64(i)*unit + 1 + } + end = int64(i+1) * unit + if i == splitNum-1 { + end = contentLength + } + + ranges[i] = &Range{start: start, end: end} + } + + return &RangeDownloader{ + splitNum: splitNum, + ranges: ranges, + url: url, + outputPath: outputPath, + }, nil +} + +// Do download +func (d *NonRangeDownloader) Do() (string, error) { + req, err := http.NewRequest(http.MethodGet, d.url, nil) + if err != nil { + return "", err + } + + client := http.DefaultClient + res, err := client.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + + return d.outputPath, saveResponseBody(d.outputPath, res) +} + +// Do split download +func (d *RangeDownloader) Do() (string, error) { + eg, ctx := errgroup.WithContext(context.TODO()) + + for i := range d.ranges { + i := i + eg.Go(func() error { + return d.do(ctx, i) + }) + } + + if err := eg.Wait(); err != nil { + return "", err + } + + return d.outputPath, d.mergeFiles() +} + +func (d *RangeDownloader) do(ctx context.Context, idx int) error { + req, err := http.NewRequest(http.MethodGet, d.url, nil) + if err != nil { + return err + } + req = req.WithContext(ctx) + + ran := d.ranges[idx] + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", ran.start, ran.end)) + + client := http.DefaultClient + res, err := client.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + + tmpFileName := fmt.Sprintf("%s.%d", d.outputPath, idx) + return saveResponseBody(tmpFileName, res) +} + +func (d *RangeDownloader) mergeFiles() error { + file, err := os.Create(d.outputPath) + if err != nil { + return err + } + defer file.Close() + + for i := range d.ranges { + tmpFileName := fmt.Sprintf("%s.%d", d.outputPath, i) + tmpFile, err := os.Open(tmpFileName) + if err != nil { + return err + } + + io.Copy(file, tmpFile) + tmpFile.Close() + if err := os.Remove(tmpFileName); err != nil { + return err + } + } + + return nil +} + +func saveResponseBody(fileName string, response *http.Response) error { + file, err := os.Create(fileName) + if err != nil { + return err + } + defer file.Close() + + if _, err := io.Copy(file, response.Body); err != nil { + return err + } + + return nil +} diff --git a/kadai3-2/pei/pkg/download/file.go b/kadai3-2/pei/pkg/download/file.go new file mode 100644 index 0000000..ccde7e1 --- /dev/null +++ b/kadai3-2/pei/pkg/download/file.go @@ -0,0 +1,20 @@ +package download + +import ( + "path/filepath" + "strings" +) + +func parseDirAndFileName(path string) (dir, file string) { + lastSlashIndex := strings.LastIndex(path, "/") + dir = path[:lastSlashIndex+1] + if len(dir) == len(path) { + return dir, "" + } + + return dir, path[(len(dir) + 1):] +} + +func parseFileName(url string) string { + return filepath.Base(url) +} From 33a9fba7aef3e402b39670a823075d6266d9501c Mon Sep 17 00:00:00 2001 From: suaaa7 Date: Sun, 1 Sep 2019 01:15:55 +0900 Subject: [PATCH 2/4] Add main.go --- kadai3-2/pei/main.go | 78 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 kadai3-2/pei/main.go diff --git a/kadai3-2/pei/main.go b/kadai3-2/pei/main.go new file mode 100644 index 0000000..92dd96e --- /dev/null +++ b/kadai3-2/pei/main.go @@ -0,0 +1,78 @@ +package main + +import ( + "flag" + "fmt" + "os" + "reflect" + + "github.com/gopherdojo/dojo6/kadai3-2/pei/pkg/download" +) + +const ( + exitCodeOk = 0 + exitCodeError = 1 + + splitNum = 4 +) + +type cliArgs struct { + url, outputPath string +} + +func (ca *cliArgs) validate() error { + if ca.url == "" { + return fmt.Errorf("No URL") + } + + if ca.outputPath == "" { + ca.outputPath = "./" + } + + return nil +} + +func main() { + os.Exit(Run()) +} + +// Run runs download +func Run() int { + ca := parseArgs() + if err := ca.validate(); err != nil { + fmt.Fprintln(os.Stderr, "Args error: ", err) + return exitCodeError + } + + downloader, err := download.NewDownloader(splitNum, ca.url, ca.outputPath) + if err != nil { + fmt.Fprintln(os.Stderr, "Create downloader error: ", err) + return exitCodeError + } + + outputPath, err := downloader.Do() + if err != nil { + fmt.Fprintln(os.Stderr, "Download error: ", err) + return exitCodeError + } + + var downloadType string + if reflect.TypeOf(downloader) == reflect.TypeOf(&download.RangeDownloader{}) { + downloadType = "Split Download" + } else { + downloadType = "Download" + } + fmt.Println("Download Type: ", downloadType) + fmt.Println("Download completed. Output: ", outputPath) + + return exitCodeOk +} + +func parseArgs() *cliArgs { + var ca cliArgs + flag.StringVar(&ca.outputPath, "o", "", "output path") + flag.Parse() + ca.url = flag.Arg(0) + + return &ca +} From b029b022b435fb2a673e693da5050a53568ecae9 Mon Sep 17 00:00:00 2001 From: suaaa7 Date: Sun, 1 Sep 2019 01:20:24 +0900 Subject: [PATCH 3/4] Add Makefile --- kadai3-2/pei/Makefile | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 kadai3-2/pei/Makefile diff --git a/kadai3-2/pei/Makefile b/kadai3-2/pei/Makefile new file mode 100644 index 0000000..9aed06f --- /dev/null +++ b/kadai3-2/pei/Makefile @@ -0,0 +1,17 @@ +ROOT=github.com/gopherdojo/dojo6/kadai3-2/pei +BIN=split-download +MAIN=main.go +TEST=... + +.PHONY: build +build: ${MAIN} + go build -o ${BIN} ${GOPATH}/src/${ROOT}/$? + +.PHONY: test +test: + go test -v -cover ${ROOT}/${TEST} + +.PHONY: clean +clean: + rm ${BIN} + go clean From bee2d02833e5d958dbfd269d78a18893a0944d79 Mon Sep 17 00:00:00 2001 From: suaaa7 Date: Sun, 1 Sep 2019 01:23:02 +0900 Subject: [PATCH 4/4] Add README --- kadai3-2/pei/README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 kadai3-2/pei/README.md diff --git a/kadai3-2/pei/README.md b/kadai3-2/pei/README.md new file mode 100644 index 0000000..b2df4f4 --- /dev/null +++ b/kadai3-2/pei/README.md @@ -0,0 +1,33 @@ +# split-download + +## Build + +``` +$ make build +``` + +## Usage + +``` +$ ./split-loadload -o [OutputPath] [URL] +``` + +## Development + +### Build + +``` +$ make build +``` + +### Test + +``` +$ make test +``` + +### Clean + +``` +$ make clean +```