diff --git a/kadai3-1/Mizushima/.gitignore b/kadai3-1/Mizushima/.gitignore new file mode 100644 index 00000000..c5e82d74 --- /dev/null +++ b/kadai3-1/Mizushima/.gitignore @@ -0,0 +1 @@ +bin \ No newline at end of file diff --git a/kadai3-2/Mizushima/.gitignore b/kadai3-2/Mizushima/.gitignore new file mode 100644 index 00000000..4083730c --- /dev/null +++ b/kadai3-2/Mizushima/.gitignore @@ -0,0 +1,5 @@ +bin/* +../../.devcontainer/ +../../Dockerfile +../../app/ +../../docker-compose.yml diff --git a/kadai3-2/Mizushima/Makefile b/kadai3-2/Mizushima/Makefile new file mode 100644 index 00000000..a22e4431 --- /dev/null +++ b/kadai3-2/Mizushima/Makefile @@ -0,0 +1,26 @@ +BINARY_NAME := bin/paraDW +GOCMD=go +GOBUILD=$(GOCMD) build +GOCLEAN=$(GOCMD) clean +GOTEST=$(GOCMD) test +GOGET=$(GOCMD) get +GOFMT=$(GOCMD) fmt + +all: build test + +build: + $(GOBUILD) -o $(BINARY_NAME) -v + +test: build + cd download; $(GOTEST) -v -cover + cd getheader; $(GOTEST) -v -cover + cd listen; $(GOTEST) -v -cover + cd request; $(GOTEST) -v -cover + +clean: + $(GOCLEAN) + +format: + $(GOFMT) ./... + +.PHONY: test clean format diff --git a/kadai3-2/Mizushima/README.md b/kadai3-2/Mizushima/README.md new file mode 100644 index 00000000..00c0e8dd --- /dev/null +++ b/kadai3-2/Mizushima/README.md @@ -0,0 +1,83 @@ +## 課題3-2 分割ダウンローダを作ろう +- 分割ダウンロードを行う +- Rangeアクセスを用いる +- いくつかのゴルーチンでダウンロードしてマージする +- エラー処理を工夫する + - golang.org/x/sync/errgourpパッケージなどを使ってみる +- キャンセルが発生した場合の実装を行う + + + +### コマンドラインオプション + + | ショートオプション | ロングオプション | 説明 | デフォルト | + | --------- | --------- | --------- | --------- | + | -h | --help | 使い方を表示して終了 | - | + | -p \ | --procs \ | プロセス数を指定 | お使いのPCのコア数 | + | -o \ | --output \ | ダウンロードしたファイルをどこのディレクトリに保存するか指定する | カレントディレクトリ | + | -t \ | --timeout \ | サーバーへのリクエストを止める時間を秒数で指定 | 120 | + + +### インストール方法 +```bash +go get github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima +``` + +### 使い方 +1. 実行ファイル作成 +```bash +$ make build +``` +2. URLを指定してダウンロード実行 +```bash +$ ./bin/paraDW [option] URL( URL URL ...) +``` + +※ URLは複数指定できます +※ ダウンロード先がRangeアクセスに対応していれば、go routineを使った並行ダウンロードを行い、そうでなければ1プロセスのダウンロードを行います + +### テストの方法 +- バイナリビルド & テスト +```bash +$ make +``` +- テスト後の処理(掃除) +```bash +$ make clean +``` + +### ディレクトリ構成 +```bash +. +├── bin +│ └── paraDW # "make build" command required. +├── download +│ ├── download.go +│ ├── download_test.go +│ ├── go.mod +│ └── go.sum +├── getheader +│ ├── geHeader_test.go +│ ├── getHeader.go +│ └── go.mod +├── .gitignore +├── go.mod +├── go.sum +├── listen +│ ├── listen.go +│ └── listen_test.go +├── main.go +├── Makefile +├── README.md +├── request +│ ├── go.mod +│ ├── request.go +│ └── request_test.go +└── testdata + ├── 003 + └── z4d4kWk.jpg +``` + +### 参考にしたもの +[pget](https://qiita.com/codehex/items/d0a500ac387d39a34401) (goroutineを使ったダウンロード処理、コマンドラインオプションの処理等々) +https://github.com/gopherdojo/dojo3/pull/50 (ctrl+cを押したときのキャンセル処理など) \ No newline at end of file diff --git a/kadai3-2/Mizushima/download/download.go b/kadai3-2/Mizushima/download/download.go new file mode 100644 index 00000000..772c9d1d --- /dev/null +++ b/kadai3-2/Mizushima/download/download.go @@ -0,0 +1,158 @@ +// download package implements parallel download and non-parallel +// download. +package download + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "net/url" + "os" + "strconv" + + "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request" + "golang.org/x/sync/errgroup" +) + +// PDownloader is user-defined struct for the download process. +// It's not limited to parallel downloads. +type PDownloader struct { + url *url.URL // URL for the download + output *os.File // Where to save the downloaded file + fileSize uint // size of the downloaded file + part uint // Number of divided bytes + procs uint // Number of parallel download process +} + +// newPDownloader is constructor for PDownloader. +func newPDownloader(url *url.URL, output *os.File, fileSize uint, part uint, procs uint) *PDownloader { + return &PDownloader{ + url: url, + output: output, + fileSize: fileSize, + part: part, + procs: procs, + } +} + +// Downloader gets elements of PDownloader, the download is parallel or not, temprary +// directory name and context.Context, and drives DownloadFile method if isPara is false +// or PDownload if isPara is true. +// +func Downloader(url *url.URL, + output *os.File, fileSize uint, part uint, procs uint, isPara bool, + tmpDirName string, ctx context.Context) error { + pd := newPDownloader(url, output, fileSize, part, procs) + if !isPara { + fmt.Printf("%s do not accept range access: downloading by single process\n", url) + err := pd.DownloadFile(ctx) + if err != nil { + return err + } + } else { + grp, ctx := errgroup.WithContext(ctx) + if err := pd.PDownload(grp, tmpDirName, procs, ctx); err != nil { + return err + } + + if err := grp.Wait(); err != nil { + return err + } + } + return nil +} + +// DownloadFile drives a non-parallel download +func (pd *PDownloader) DownloadFile(ctx context.Context) (err error) { + + resp, err := request.Request(ctx, "GET", pd.url.String(), "", "") + if err != nil { + return + } + defer func() { + err = resp.Body.Close() + }() + + _, err = io.Copy(pd.output, resp.Body) + if err != nil { + return + } + + return nil +} + +// PDownload drives parallel download. downloaded file is in temporary +// directory named tmpDirName. +func (pd *PDownloader) PDownload(grp *errgroup.Group, + tmpDirName string, procs uint, ctx context.Context) error { + var start, end, idx uint + + for idx = uint(0); idx < procs; idx++ { + if idx == 0 { + start = 0 + } else { + start = idx*pd.part + 1 + } + + // if idx is the end + if idx == pd.procs-1 { + end = pd.fileSize + } else { + end = (idx + 1) * pd.part + } + + // idxを代入し直す + // https://qiita.com/harhogefoo/items/7ccb4e353a4a01cfa773 + idx := idx + // fmt.Printf("start: %d, end: %d, pd.part: %d\n", start, end, pd.part) + bytes := fmt.Sprintf("bytes=%d-%d", start, end) + + grp.Go(func() error { + fmt.Printf("grp.Go: tmpDirName: %s, bytes %s, idx: %d\n", tmpDirName, bytes, idx) + return pd.ReqToMakeCopy(tmpDirName, bytes, idx, ctx) + }) + } + return nil +} + +// ReqToMakeCopy sends a "GET" request with "Range" field with "bytes" range. +// And gets response and make a copy to a temprary file in temprary directory from response body. +// +func (pd *PDownloader) ReqToMakeCopy(tmpDirName, bytes string, idx uint, ctx context.Context) (err error) { + // fmt.Printf("ReqToMakeCopy: tmpDirName: %s, bytes %s, idx: %d\n", tmpDirName, bytes, idx) + resp, err := request.Request(ctx, "GET", pd.url.String(), "Range", bytes) + if err != nil { + return err + } + + tmpOut, err := os.Create(tmpDirName + "/" + strconv.Itoa(int(idx))) + if err != nil { + return err + } + // fmt.Printf("tmpOut.Name(): %s\n", tmpOut.Name()) + defer func() { + err = tmpOut.Close() + }() + + // b := make([]byte, 1000) + // resp.Body.Read(b) + // fmt.Printf("resp.body: %s\n", string(b)) + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + // fmt.Printf("err: %s\n", err) + if err != io.EOF && err != io.ErrUnexpectedEOF { + return err + } + } + + // fmt.Printf("response body: length: %d\n", len(body)) + + length, err := tmpOut.Write(body) + if err != nil { + return err + } + fmt.Printf("%d/%d was downloaded len=%d\n", idx, pd.procs, length) + return nil +} diff --git a/kadai3-2/Mizushima/download/download_test.go b/kadai3-2/Mizushima/download/download_test.go new file mode 100644 index 00000000..9a99526e --- /dev/null +++ b/kadai3-2/Mizushima/download/download_test.go @@ -0,0 +1,321 @@ +package download_test + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/ioutil" + "log" + "net/http" + "net/http/httptest" + "net/url" + "os" + "reflect" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download" +) + +var testdataPathMap = map[int][]string{ + 0: {"../testdata/003", mustGetSize("../testdata/003")}, + 1: {"../testdata/z4d4kWk.jpg", mustGetSize("../testdata/z4d4kWk.jpg")}, + // 2 : "../documents/http.request.txt", +} + +func TestDownloader_SingleProcess(t *testing.T) { + t.Helper() + + ts, clean := newTestServer(t, nonRangeAccessHandler, 0) + defer clean() + + // get a url.URL object + urlObj := getURLObject(t, ts.URL) + + // get a file for output + output, clean := makeTempFile(t) + defer clean() + + resp, err := http.Get(ts.URL) + if err != nil { + t.Error(err) + } + defer resp.Body.Close() + + // get the file size to be downloaded. + size := getSizeForTest(t, resp) + + // this test is non-parallel download. + part := size + procs := uint(1) + isPara := false + tmpDirName := "" + ctx := context.Background() + + t.Run("case 1", func(t *testing.T) { + err := download.Downloader(urlObj, output, size, part, procs, isPara, tmpDirName, ctx) + if err != nil { + t.Error(err) + } + + actual := new(bytes.Buffer).Bytes() + _, err = output.Read(actual) + if err != nil { + t.Error(err) + } + + expected, err := os.ReadFile(testdataPathMap[0][0]) + if err != nil { + t.Error(err) + } + + if reflect.DeepEqual(actual, expected) { + t.Errorf("expected %s, but got %s", expected, actual) + } + }) +} + +func TestDownloader_SingleProcessTimeout(t *testing.T) { + t.Helper() + + ts, clean := newTestServer(t, nonRangeAccessTooLateHandler, 0) + defer clean() + + // get a url.URL object + urlObj := getURLObject(t, ts.URL) + + // get a file for output + output, clean := makeTempFile(t) + defer clean() + + resp, err := http.Get(ts.URL) + if err != nil { + t.Error(err) + } + defer resp.Body.Close() + + // get the file size to be downloaded. + size := getSizeForTest(t, resp) + + // this test is non-parallel download. + part := size + procs := uint(1) + isPara := false + tmpDirName := "" + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + t.Run("case 1", func(t *testing.T) { + actual := download.Downloader(urlObj, output, size, part, procs, isPara, tmpDirName, ctx) + if err != nil { + t.Error(err) + } + expected := fmt.Errorf("request.Request err: Get \"%s\": %w", urlObj, context.DeadlineExceeded) + if actual.Error() != expected.Error() { + t.Errorf("expected %s, \nbut got %s", expected, actual) + } + }) +} + +func TestDownloader_ParallelProcess(t *testing.T) { + t.Helper() + + // make a server for test. + ts, clean := newTestServer(t, rangeAccessHandler, 1) + defer clean() + + // get a url.URL object + urlObj := getURLObject(t, ts.URL) + + // get a file for output + output, clean := makeTempFile(t) + defer clean() + + // get a response from test server. + resp, err := http.Get(ts.URL) + if err != nil { + t.Error(err) + } + defer resp.Body.Close() + + // get the file size to be downloaded. + size := getSizeForTest(t, resp) + + // this test is parallel download. + procs := uint(runtime.NumCPU()) + part := size / procs + isPara := true + tmpDirName := "test" + ctx := context.Background() + + t.Run("case 1", func(t *testing.T) { + if err := os.Mkdir(tmpDirName, 0775); err != nil { + t.Error(err) + } + err := download.Downloader(urlObj, output, size, part, procs, isPara, tmpDirName, ctx) + if err != nil { + t.Errorf("err: %w", err) + } + defer func() { + err := os.RemoveAll(tmpDirName) + if err != nil { + t.Error(err) + } + }() + }) +} + +func newTestServer(t *testing.T, + handler func(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int), + testDataKey int) (*httptest.Server, func()) { + + t.Helper() + + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + handler(t, w, r, testDataKey) + }, + )) + + return ts, func() { ts.Close() } +} + +func nonRangeAccessHandler(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int) { + t.Helper() + + body, err := os.ReadFile(testdataPathMap[testDataKey][0]) + if err != nil { + t.Fatal(err) + } + w.Header().Set("Content-Length", testdataPathMap[testDataKey][1]) + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, body) +} + +func nonRangeAccessTooLateHandler(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int) { + t.Helper() + + body, err := os.ReadFile(testdataPathMap[testDataKey][0]) + if err != nil { + t.Fatal(err) + } + + time.Sleep(3 * time.Second) + w.Header().Set("Content-Length", testdataPathMap[testDataKey][1]) + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, body) +} + +func rangeAccessHandler(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int) { + t.Helper() + + w.Header().Set("Content-Length", testdataPathMap[testDataKey][1]) + w.Header().Set("Access-Range", "bytes") + + rangeHeader := r.Header.Get("Range") + + body := retBody(t, rangeHeader, testdataPathMap[testDataKey][0]) + w.WriteHeader(http.StatusPartialContent) + fmt.Fprint(w, body) +} + +func retBody(t *testing.T, rangeHeader string, testDataPath string) []byte { + t.Helper() + + b, err := os.ReadFile(testDataPath) + if err != nil { + t.Fatal(err) + } + + if rangeHeader == "" { + return b + } + + rangeVals := strings.Split(rangeHeader, "=") + if rangeVals[0] != "bytes" { + t.Fatal(errors.New("err : Range header expected \"bytes\"")) + } + + rangeBytes := strings.Split(rangeVals[1], "-") + start, err := strconv.Atoi(rangeBytes[0]) + if err != nil { + t.Fatal(err) + } + + end, err := strconv.Atoi(rangeBytes[1]) + if err != nil { + t.Fatal(err) + } + + // fmt.Printf("length of b: %d\n", len(b)) + // fmt.Printf("length of b[start:end+1]: %d\n", len(b[start:end+1])) + return b[start : end+1] +} + +func getURLObject(t *testing.T, urlStr string) *url.URL { + t.Helper() + + urlObj, err := url.ParseRequestURI(urlStr) + if err != nil { + t.Error(err) + } + + return urlObj +} + +func makeTempFile(t *testing.T) (*os.File, func()) { + t.Helper() + + dir, err := ioutil.TempDir("", "test_download") + if err != nil { + t.Fatal(err) + } + + out, err := os.Create(dir + "/test") + if err != nil { + t.Fatal(err) + } + + return out, + func() { + err = out.Close() + if err != nil { + t.Fatal(err) + } + err = os.RemoveAll(dir) + if err != nil { + t.Fatal(err) + } + } +} + +// GetSize returns size from response header. +func getSizeForTest(t *testing.T, r *http.Response) uint { + t.Helper() + + contLen, is := r.Header["Content-Length"] + // fmt.Println(h) + if !is { + t.Errorf("cannot find Content-Length header") + } + + ret, err := strconv.ParseUint(contLen[0], 10, 32) + if err != nil { + t.Error(err) + } + return uint(ret) +} + +func mustGetSize(path string) string { + + fileinfo, err := os.Stat(path) + if err != nil { + log.Fatal(err) + } + + return strconv.Itoa(int(fileinfo.Size())) +} diff --git a/kadai3-2/Mizushima/download/go.mod b/kadai3-2/Mizushima/download/go.mod new file mode 100644 index 00000000..240ce7d4 --- /dev/null +++ b/kadai3-2/Mizushima/download/go.mod @@ -0,0 +1,10 @@ +module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download + +go 1.16 + +replace github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request => ../request + +require ( + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request v0.0.0-00010101000000-000000000000 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c +) diff --git a/kadai3-2/Mizushima/download/go.sum b/kadai3-2/Mizushima/download/go.sum new file mode 100644 index 00000000..5c00efd3 --- /dev/null +++ b/kadai3-2/Mizushima/download/go.sum @@ -0,0 +1,2 @@ +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/kadai3-2/Mizushima/getheader/geHeader_test.go b/kadai3-2/Mizushima/getheader/geHeader_test.go new file mode 100644 index 00000000..4ba88d76 --- /dev/null +++ b/kadai3-2/Mizushima/getheader/geHeader_test.go @@ -0,0 +1,139 @@ +package getheader_test + +import ( + "errors" + "net/http" + "os" + "reflect" + "testing" + + "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader" +) + +var heads1 = []string{ + "Content-Length", + "Accept-Ranges", + "Content-Type", + "Access-Control-Allow-Methods", +} + +var heads2 = []string{ + "Accept-Ranges", + "Content-Type", + "Access-Control-Allow-Methods", +} + +var vals1 = [][]string{{"146515"}, {"bytes"}, {"image/jpeg"}, {"GET", "OPTIONS"}} + +var vals2 = [][]string{{"bytes"}, {"image/jpeg"}, {"GET", "OPTIONS"}} + +func Test_ResHeader(t *testing.T) { + t.Helper() + + cases := []struct { + name string + input string + heads []string + vals [][]string + expected []string + }{ + { + name: "case 1", + input: "Content-Length", + heads: heads1, + vals: vals1, + expected: []string{"146515"}, + }, + { + name: "case 2", + input: "Accept-Ranges", + heads: heads2, + vals: vals2, + expected: []string{"bytes"}, + }, + } + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + resp, err := makeResponse(t, c.heads, c.vals) + if err != nil { + t.Error(err) + } + actual, err := getheader.ResHeader(os.Stdout, resp, c.input) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(actual, c.expected) { + t.Errorf("expected %s, but got %s", c.expected, actual) + } + }) + } +} + +func Test_GetSize(t *testing.T) { + + cases := []struct { + name string + heads []string + vals [][]string + expected uint + }{ + { + name: "case 1", + heads: heads1, + vals: vals1, + expected: uint(146515), + }, + { + name: "case 2", + heads: heads2, + vals: vals2, + expected: 0, + }, + } + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + resp, err := makeResponse(t, c.heads, c.vals) + + if err != nil { + t.Error(err) + } + actual, err := getheader.GetSize(resp) + if err != nil { + if actual != 0 || c.expected != 0 { + t.Error(err) + } else { + if err.Error() != "cannot find Content-Length header" { + t.Errorf("expected error: cannot find Content-Length header, but %w", err) + } + } + } + if actual != c.expected { + t.Errorf("expected %d, but got %d", c.expected, actual) + } + }) + } +} + +func makeResponse(t *testing.T, heads []string, vals [][]string) (*http.Response, error) { + t.Helper() + + var resp = make(map[string][]string) + + if len(heads) != len(vals) { + return nil, errors.New("expected the length of heads and vals sre same") + } + + for i := 0; i < len(heads); i++ { + if _, ok := resp[heads[i]]; ok { + return nil, errors.New("Duplicate elements in heads") + } + + resp[heads[i]] = vals[i] + } + + return &http.Response{Header: http.Header(resp)}, nil +} diff --git a/kadai3-2/Mizushima/getheader/getHeader.go b/kadai3-2/Mizushima/getheader/getHeader.go new file mode 100644 index 00000000..1165d5f5 --- /dev/null +++ b/kadai3-2/Mizushima/getheader/getHeader.go @@ -0,0 +1,36 @@ +// getheader package implements to read a response header of http +package getheader + +import ( + "fmt" + "io" + "net/http" + "os" + "strconv" +) + +// ResHeader returns the value of the specified header, and writes http response header on io.Writer. +func ResHeader(w io.Writer, r *http.Response, header string) ([]string, error) { + h, is := r.Header[header] + // fmt.Println(h) + if !is { + return nil, fmt.Errorf("cannot find %s header", header) + } + if _, err := fmt.Fprintf(w, "Header[%s] = %s\n", header, h); err != nil { + return nil, err + } + return h, nil +} + +// GetSize returns size from response header. +func GetSize(resp *http.Response) (uint, error) { + contLen, err := ResHeader(os.Stdout, resp, "Content-Length") + if err != nil { + return 0, err + } + ret, err := strconv.ParseUint(contLen[0], 10, 32) + if err != nil { + return 0, err + } + return uint(ret), nil +} diff --git a/kadai3-2/Mizushima/getheader/go.mod b/kadai3-2/Mizushima/getheader/go.mod new file mode 100644 index 00000000..52086171 --- /dev/null +++ b/kadai3-2/Mizushima/getheader/go.mod @@ -0,0 +1,3 @@ +module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader + +go 1.16 diff --git a/kadai3-2/Mizushima/go.mod b/kadai3-2/Mizushima/go.mod new file mode 100644 index 00000000..580c93bd --- /dev/null +++ b/kadai3-2/Mizushima/go.mod @@ -0,0 +1,18 @@ +module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima + +go 1.16 + +replace ( + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download => ./download + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader => ./getheader + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/listen => ./listen + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request => ./request +) + +require ( + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download v0.0.0-00010101000000-000000000000 + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader v0.0.0-00010101000000-000000000000 + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request v0.0.0-00010101000000-000000000000 + github.com/jessevdk/go-flags v1.5.0 + github.com/pkg/errors v0.9.1 +) diff --git a/kadai3-2/Mizushima/go.sum b/kadai3-2/Mizushima/go.sum new file mode 100644 index 00000000..282a40a4 --- /dev/null +++ b/kadai3-2/Mizushima/go.sum @@ -0,0 +1,8 @@ +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/kadai3-2/Mizushima/listen/listen.go b/kadai3-2/Mizushima/listen/listen.go new file mode 100644 index 00000000..200f6588 --- /dev/null +++ b/kadai3-2/Mizushima/listen/listen.go @@ -0,0 +1,34 @@ +package listen + +import ( + "context" + "fmt" + "io" + "log" + "os" + "os/signal" + "syscall" +) + +var osExit = os.Exit + +// Listen returns a context for keyboad(ctrl + c) interrupt. +func Listen(ctx context.Context, w io.Writer, f func()) (context.Context, func()) { + ctx, cancel := context.WithCancel(ctx) + + ch := make(chan os.Signal, 2) + signal.Notify(ch, os.Interrupt, syscall.SIGTERM) + go func() { + <-ch + _, err := fmt.Fprintln(w, "\n^Csignal : interrupt.") + if err != nil { + cancel() + log.Fatalf("err: listen.Listen: %s\n", err) + } + cancel() + f() + osExit(0) + }() + + return ctx, cancel +} diff --git a/kadai3-2/Mizushima/listen/listen_test.go b/kadai3-2/Mizushima/listen/listen_test.go new file mode 100644 index 00000000..24991b8c --- /dev/null +++ b/kadai3-2/Mizushima/listen/listen_test.go @@ -0,0 +1,40 @@ +package listen + +import ( + "bytes" + "context" + "os" + "testing" + "time" +) + +func Test_Listen(t *testing.T) { + t.Helper() + + cleanFn := func() {} + + doneCh := make(chan struct{}) + osExit = func(code int) { doneCh <- struct{}{} } + + output := new(bytes.Buffer) + _, cancel := Listen(context.Background(), output, cleanFn) + defer cancel() + + proc, err := os.FindProcess(os.Getpid()) + if err != nil { + t.Fatalf("err: %s", err) + } + + err = proc.Signal(os.Interrupt) + if err != nil { + t.Fatalf("err: %s", err) + } + + select { + case <-doneCh: + return + case <-time.After(100 * time.Millisecond): + t.Fatal("timeout") + } + +} diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go new file mode 100644 index 00000000..1bf14fe3 --- /dev/null +++ b/kadai3-2/Mizushima/main.go @@ -0,0 +1,242 @@ +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "log" + "net/url" + "os" + "path/filepath" + "runtime" + "strconv" + "time" + + flags "github.com/jessevdk/go-flags" + + "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download" + "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader" + "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/listen" + "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request" +) + +// struct for options +type Options struct { + Help bool `short:"h" long:"help"` + Procs uint `short:"p" long:"procs"` + Output string `short:"o" long:"output" default:"./"` + Tm int `short:"t" long:"timeout" default:"120"` +} + +// parse options +func (opts *Options) parse(argv []string) ([]string, error) { + p := flags.NewParser(opts, flags.PrintErrors) + args, err := p.ParseArgs(argv) + if err != nil { + _, err2 := os.Stderr.Write(opts.usage()) + if err2 != nil { + return nil, fmt.Errorf("%s: invalid command line options: cannot print usage: %s", err, err2) + } + return nil, fmt.Errorf("%w: invalid command line options", err) + } + + return args, nil +} + +// usage prints a description of avilable options +func (opts Options) usage() []byte { + buf := bytes.Buffer{} + + fmt.Fprintln(&buf, + `Usage: paraDW [options] URL (URL2, URL3, ...) + + Options: + -h, --help print usage and exit + -p, --procs the number of split to download (default: the number of CPU cores) + -o, --output path of the file downloaded (default: current directory) + -t, --timeout Time limit of return of http response in seconds (default: 120) + `, + ) + + return buf.Bytes() +} + +func main() { + + // parse options + var opts Options + argv := os.Args[1:] + if len(argv) == 0 { + if _, err := os.Stdout.Write(opts.usage()); err != nil { + log.Fatalf("err: %s: %s\n", errors.New("no options"), err) + } + log.Fatalf("err: %s\n", errors.New("no options")) + } + + urlsStr, err := opts.parse(argv) + if err != nil { + log.Fatalf("err: %s\n", err) + } + + var urls []*url.URL + for _, u := range urlsStr { + url, err := url.ParseRequestURI(u) + if err != nil { + log.Fatalf("err: url.ParseRequestURI: %s\n", err) + } + urls = append(urls, url) + } + + fmt.Printf("timeout: %d\n", opts.Tm) + + if opts.Help { + if _, err := os.Stdout.Write(opts.usage()); err != nil { + log.Fatalf("err: cannot print usage: %s", err) + } + log.Fatal(errors.New("print usage")) + } + + // if procs was inputted, set the number of runtime.NumCPU() to opts.Procs. + if opts.Procs == 0 { + opts.Procs = uint(runtime.NumCPU()) + } + + // if opts.Output inputted and the end of opts.Output is not '/', + // add '/'. + if len(opts.Output) > 0 && opts.Output[len(opts.Output)-1] != '/' { + opts.Output += "/" + } + + // download from each url in urls + for i, urlObj := range urls { + downloadFromUrl(i, opts, urlObj) + } +} + +// downloadFromUrl does the download processing from url object. +func downloadFromUrl(i int, opts Options, urlObj *url.URL) { + + // make a timeout context from a empty context + ctxTimeout, cancelTimeout := context.WithTimeout(context.Background(), time.Duration(opts.Tm)*time.Second) + defer cancelTimeout() + + // send "HEAD" request, and gets response. + resp, err := request.Request(ctxTimeout, "HEAD", urlObj.String(), "", "") + if err != nil { + log.Fatalf("err: %s\n", err) + } + + // get the size from the response header. + fileSize, err := getheader.GetSize(resp) + if err != nil { + log.Fatalf("err: getheader.GetSize: %s\n", err) + } + if err = resp.Body.Close(); err != nil { + log.Fatalf("err: %s", err) + } + + // get how many bytes to download at a time + partial := fileSize / opts.Procs + + outputPath := opts.Output + filepath.Base(urlObj.String()) + // if there is the same file in opts.Output, delete that file in advance. + if isExists(outputPath) { + err := os.Remove(outputPath) + if err != nil { + log.Fatalf("err: isExists: os.Remove: %s\n", err) + } + } + + // make a file for download + out, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755) + if err != nil { + log.Fatalf("err: os.Create: %s\n", err) + } + defer func() { + if err := out.Close(); err != nil { + log.Fatalf("err: %s", err) + } + }() + + // make a temporary directory for parallel download + tmpDirName := opts.Output + strconv.Itoa(i) + err = os.Mkdir(tmpDirName, 0777) + if err != nil { + if err3 := out.Close(); err3 != nil { + log.Fatalf("err: %s", err3) + } + if err2 := os.Remove(opts.Output + filepath.Base(urlObj.String())); err2 != nil { + log.Fatalf("err: os.Mkdir: %s\nerr: os.Remove: %s\n", err, err2) + } + log.Fatalf("err: os.Mkdir: %s\n", err) + } + + clean := func() { + if err := out.Close(); err != nil { + log.Fatalf("err: out.Close: %s\n", err) + } + // delete the tmporary directory + if err := os.RemoveAll(tmpDirName); err != nil { + log.Fatalf("err: RemoveAll: %s\n", err) + } + if err := os.Remove(opts.Output + filepath.Base(urlObj.String())); err != nil { + log.Fatalf("err: os.Remove: %s\n", err) + } + } + ctx, cancel := listen.Listen(ctxTimeout, os.Stdout, clean) + defer cancel() + + var isPara bool = true + _, err = getheader.ResHeader(os.Stdout, resp, "Accept-Ranges") + if err != nil && err.Error() == "cannot find Accept-Ranges header" { + isPara = false + } else if err != nil { + clean() + log.Fatalf("err: getheader.ResHeader: %s\n", err) + } + + // drive a download process + err = download.Downloader(urlObj, out, fileSize, partial, opts.Procs, isPara, tmpDirName, ctx) + if err != nil { + log.Fatalf("err: %s\n", err) + } + + fmt.Printf("download complete: %s\n", urlObj.String()) + + // Merge the temporary files into "out", when parallel download executed. + if isPara { + err = MergeFiles(tmpDirName, opts.Procs, fileSize, out) + if err != nil { + log.Fatalf("err: MergeFiles: %s\n", err) + } + } + + // delete the tmporary directory only + if err := os.RemoveAll(tmpDirName); err != nil { + log.Fatalf("err: RemoveAll: %s\n", err) + } +} + +// MergeFiles merges temporary files made for parallel download into "output". +func MergeFiles(tmpDirName string, procs, fileSize uint, output *os.File) error { + for i := uint(0); i < procs; i++ { + + body, err := os.ReadFile(tmpDirName + "/" + strconv.Itoa(int(i))) + if err != nil { + return err + } + + if _, err = fmt.Fprint(output, string(body)); err != nil { + return err + } + fmt.Printf("target file: %s, len=%d written\n", output.Name(), len(string(body))) + } + return nil +} + +// isExists returns the file exists or not, in 'path'. +func isExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/kadai3-2/Mizushima/request/go.mod b/kadai3-2/Mizushima/request/go.mod new file mode 100644 index 00000000..3a9f90c5 --- /dev/null +++ b/kadai3-2/Mizushima/request/go.mod @@ -0,0 +1,3 @@ +module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request + +go 1.16 diff --git a/kadai3-2/Mizushima/request/request.go b/kadai3-2/Mizushima/request/request.go new file mode 100644 index 00000000..865c0bcc --- /dev/null +++ b/kadai3-2/Mizushima/request/request.go @@ -0,0 +1,26 @@ +package request + +import ( + "context" + "fmt" + "net/http" +) + +// Request throws a request and returns a response object from url and a error. +func Request(ctx context.Context, method string, urlStr string, setH string, setV string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, method, urlStr, nil) + if err != nil { + return nil, err + } + + if len(setH) != 0 { + req.Header.Set(setH, setV) + } + + client := new(http.Client) + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("request.Request err: %s", err) + } + return resp, nil +} diff --git a/kadai3-2/Mizushima/request/request_test.go b/kadai3-2/Mizushima/request/request_test.go new file mode 100644 index 00000000..8ed2bfd2 --- /dev/null +++ b/kadai3-2/Mizushima/request/request_test.go @@ -0,0 +1,225 @@ +package request_test + +import ( + "context" + "errors" + "fmt" + "log" + "net/http" + "net/http/httptest" + "net/http/httputil" + "os" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request" +) + +var testdataPathMap = map[int][]string{ + 0: {"../testdata/003", mustGetSize("../testdata/003")}, + 1: {"../testdata/z4d4kWk.jpg", mustGetSize("../testdata/z4d4kWk.jpg")}, +} + +func Test_RequestStandard(t *testing.T) { + t.Helper() + + cases := map[string]struct { + key int // key for testdataPathMap + handler func(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int) + expected *http.Response + }{ + "case 1": { + key: 0, + handler: nonRangeAccessHandler, + expected: &http.Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: map[string][]string{ + "Content-Length": {testdataPathMap[0][1]}, + "Date": {mustTimeLayout(t, time.Now())}, + }, + }, + }, + "case 2": { + key: 1, + handler: rangeAccessHandler, + expected: &http.Response{ + Status: "206 Partial Content", + StatusCode: 206, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: map[string][]string{ + "Access-Range": {"bytes"}, + "Content-Length": {testdataPathMap[1][1]}, + "Date": {mustTimeLayout(t, time.Now())}, + }, + }, + }, + } + + for name, c := range cases { + c := c + t.Run(name, func(t *testing.T) { + ts, clean := newTestServer(t, c.handler, c.key) + defer clean() + actual, err := request.Request(context.Background(), "GET", ts.URL, "", "") + if err != nil { + t.Fatal(err) + } + // fmt.Println("actual:", actual.Header) + if !reflect.DeepEqual(actual.Header, c.expected.Header) { + dumped_expected, err := httputil.DumpResponse(c.expected, false) + if err != nil { + t.Fatal(err) + } + dumped_actual, err := httputil.DumpResponse(actual, false) + if err != nil { + t.Fatal(err) + } + t.Errorf("expected,\n%vbut got,\n%v", string(dumped_expected), string(dumped_actual)) + } + }) + } +} + +func Test_RequestTimeout(t *testing.T) { + t.Helper() + + name := "case timeout" + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + t.Run(name, func(t *testing.T) { + ts, clean := newTestServer(t, nonRangeAccessTooLateHandler, 1) + defer clean() + + expected := fmt.Errorf("request.Request err: Get \"%s\": %w", ts.URL, context.DeadlineExceeded) + _, err := request.Request(ctx, "GET", ts.URL, "", "") + + // fmt.Println("actual:\n", actual) + if err.Error() != expected.Error() { + t.Errorf("expected %s, but got %s", expected.Error(), err.Error()) + } + }) +} + +func newTestServer(t *testing.T, + handler func(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int), + testDataKey int) (*httptest.Server, func()) { + + t.Helper() + + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + handler(t, w, r, testDataKey) + }, + )) + + return ts, func() { ts.Close() } +} + +func nonRangeAccessHandler(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int) { + t.Helper() + + body, err := os.ReadFile(testdataPathMap[testDataKey][0]) + if err != nil { + t.Fatal(err) + } + w.Header().Set("Content-Length", testdataPathMap[testDataKey][1]) + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, body) +} + +func nonRangeAccessTooLateHandler(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int) { + t.Helper() + + body, err := os.ReadFile(testdataPathMap[testDataKey][0]) + if err != nil { + t.Fatal(err) + } + + time.Sleep(3 * time.Second) + w.Header().Set("Content-Length", testdataPathMap[testDataKey][1]) + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, body) +} + +func rangeAccessHandler(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int) { + t.Helper() + + w.Header().Set("Content-Length", testdataPathMap[testDataKey][1]) + w.Header().Set("Access-Range", "bytes") + + rangeHeader := r.Header.Get("Range") + + body := retBody(t, rangeHeader, testdataPathMap[testDataKey][0]) + w.WriteHeader(http.StatusPartialContent) + fmt.Fprint(w, body) +} + +func retBody(t *testing.T, rangeHeader string, testDataPath string) []byte { + t.Helper() + + b, err := os.ReadFile(testDataPath) + if err != nil { + t.Fatal(err) + } + + if rangeHeader == "" { + return b + } + + rangeVals := strings.Split(rangeHeader, "=") + if rangeVals[0] != "bytes" { + t.Fatal(errors.New("err : Range header expected \"bytes\"")) + } + + rangeBytes := strings.Split(rangeVals[1], "-") + start, err := strconv.Atoi(rangeBytes[0]) + if err != nil { + t.Fatal(err) + } + + end, err := strconv.Atoi(rangeBytes[1]) + if err != nil { + t.Fatal(err) + } + + // fmt.Printf("length of b: %d\n", len(b)) + // fmt.Printf("length of b[start:end+1]: %d\n", len(b[start:end+1])) + return b[start : end+1] +} + +// mustGetSize returns the size of the file in "path" as a string for "Content-Length" in http header. +func mustGetSize(path string) string { + + fileinfo, err := os.Stat(path) + if err != nil { + log.Fatal(err) + } + + return strconv.Itoa(int(fileinfo.Size())) +} + +// mustTimeLayout returns the time now in format like this : "Mon, 12 Jul 2021 09:22:22 GMT" +func mustTimeLayout(t *testing.T, tm time.Time) string { + t.Helper() + + // get the gmt time + location, err := time.LoadLocation("GMT") + if err != nil { + t.Fatal(err) + } + tm = tm.In(location) + + // the layout is like this : "Mon, 12 Jul 2021 09:22:22 GMT" + return tm.Format(time.RFC1123) +} diff --git a/kadai3-2/Mizushima/testdata/003 b/kadai3-2/Mizushima/testdata/003 new file mode 100644 index 00000000..d9b00569 --- /dev/null +++ b/kadai3-2/Mizushima/testdata/003 @@ -0,0 +1 @@ +ahoahoaho \ No newline at end of file diff --git a/kadai3-2/Mizushima/testdata/empty.txt b/kadai3-2/Mizushima/testdata/empty.txt new file mode 100644 index 00000000..e69de29b diff --git a/kadai3-2/Mizushima/testdata/z4d4kWk.jpg b/kadai3-2/Mizushima/testdata/z4d4kWk.jpg new file mode 100644 index 00000000..a2cee5f2 Binary files /dev/null and b/kadai3-2/Mizushima/testdata/z4d4kWk.jpg differ