diff --git a/kadai3/ramenjuniti/gdown/Makefile b/kadai3/ramenjuniti/gdown/Makefile new file mode 100644 index 0000000..adc6bde --- /dev/null +++ b/kadai3/ramenjuniti/gdown/Makefile @@ -0,0 +1,14 @@ +BIN := gdown + +.PHONY: test +test: + go test -cover -v ./... + +.PHONY: build +build: test + go build -o $(BIN) ./cmd/gdown + +.PHONY: clean +clean: + rm $(BIN) + go clean \ No newline at end of file diff --git a/kadai3/ramenjuniti/gdown/README.md b/kadai3/ramenjuniti/gdown/README.md new file mode 100644 index 0000000..e1e313f --- /dev/null +++ b/kadai3/ramenjuniti/gdown/README.md @@ -0,0 +1,19 @@ +# gdown + +## Usage + +```bash +./gdown -p {並列数} {URL} +``` + +## Build + +```bash +make build +``` + +## Test + +```bash +make test +``` diff --git a/kadai3/ramenjuniti/gdown/cmd/gdown/main.go b/kadai3/ramenjuniti/gdown/cmd/gdown/main.go new file mode 100644 index 0000000..35dd871 --- /dev/null +++ b/kadai3/ramenjuniti/gdown/cmd/gdown/main.go @@ -0,0 +1,37 @@ +package main + +import ( + "flag" + "fmt" + "os" + "runtime" + + "github.com/gopherdojo/dojo5/kadai3/ramenjuniti/gdown" +) + +func main() { + p := flag.Int("p", runtime.NumCPU(), "並列数") + flag.Parse() + + if *p < 1 { + fmt.Fprintln(os.Stderr, "p cannot be less than 1") + os.Exit(1) + } + + url := flag.Arg(0) + if url == "" { + fmt.Fprintln(os.Stderr, "please input URL") + os.Exit(1) + } + + c, err := gdown.New(url, *p) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + if err = c.Run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/kadai3/ramenjuniti/gdown/gdown.go b/kadai3/ramenjuniti/gdown/gdown.go new file mode 100644 index 0000000..19d88b7 --- /dev/null +++ b/kadai3/ramenjuniti/gdown/gdown.go @@ -0,0 +1,173 @@ +package gdown + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + + "golang.org/x/sync/errgroup" +) + +type RangeRequest struct { + URL string + FileName string + Unit int64 + Ranges []*Range +} + +type Range struct { + start int64 + end int64 +} + +func New(rawurl string, p int) (*RangeRequest, error) { + u, err := url.Parse(rawurl) + if err != nil { + return nil, err + } + + fn := getName(u.Path) + + res, err := http.Head(rawurl) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.Header.Get("Accept-Ranges") != "bytes" { + return &RangeRequest{URL: rawurl, FileName: fn, Ranges: nil}, nil + } + + cl := res.ContentLength + unit := cl / int64(p) + ranges := make([]*Range, p) + + for i := 0; i < p; i++ { + start := int64(i) * unit + end := start + unit - 1 + if i == p-1 { + end = cl + } + + ranges[i] = &Range{start: start, end: end} + } + + return &RangeRequest{URL: rawurl, FileName: fn, Unit: unit, Ranges: ranges}, nil +} + +func (r *RangeRequest) Run() error { + if r.Ranges == nil { + req, err := http.NewRequest(http.MethodGet, r.URL, nil) + if err != nil { + return err + } + + res, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + + return save(r.FileName, res) + + } else { + g, ctx := errgroup.WithContext(context.Background()) + + for i, ran := range r.Ranges { + i, ran := i, ran + tmpn := fmt.Sprintf("%s-%d", r.FileName, i) + + if info, err := os.Stat(tmpn); err == nil { + size := info.Size() + if i == len(r.Ranges)-1 { + if size == ran.end-ran.start { + continue + } + } else if size == r.Unit { + continue + } + ran.start += size + } + + g.Go(func() error { + req, err := makeRangeReqest(ctx, r.URL, ran.start, ran.end) + if err != nil { + return err + } + + res, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + + return save(tmpn, res) + }) + } + + if err := g.Wait(); err != nil { + return err + } + + return merge(r.FileName, len(r.Ranges)) + } +} + +func makeRangeReqest(ctx context.Context, url string, start, end int64) (*http.Request, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + + return req, nil +} + +func getName(url string) string { + us := strings.Split(url, "/") + return us[len(us)-1] +} + +func save(fn string, res *http.Response) error { + f, err := os.OpenFile(fn, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + return err + } + defer f.Close() + + if _, err := io.Copy(f, res.Body); err != nil { + return err + } + + return nil +} + +func merge(fn string, p int) error { + f, err := os.Create(fn) + if err != nil { + return err + } + defer f.Close() + + for i := 0; i < p; i++ { + tmpn := fmt.Sprintf("%s-%d", fn, i) + tmp, err := os.Open(tmpn) + if err != nil { + return err + } + + io.Copy(f, tmp) + tmp.Close() + + if err := os.Remove(tmpn); err != nil { + return err + } + } + + return nil +} diff --git a/kadai3/ramenjuniti/gdown/go.mod b/kadai3/ramenjuniti/gdown/go.mod new file mode 100644 index 0000000..0384cf8 --- /dev/null +++ b/kadai3/ramenjuniti/gdown/go.mod @@ -0,0 +1,5 @@ +module github.com/gopherdojo/dojo5/kadai3/ramenjuniti/gdown + +go 1.12 + +require golang.org/x/sync v0.0.0-20190423024810-112230192c58 diff --git a/kadai3/ramenjuniti/gdown/go.sum b/kadai3/ramenjuniti/gdown/go.sum new file mode 100644 index 0000000..6eae930 --- /dev/null +++ b/kadai3/ramenjuniti/gdown/go.sum @@ -0,0 +1,2 @@ +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=