diff --git a/kadai3-2/nagaa052/.gitignore b/kadai3-2/nagaa052/.gitignore new file mode 100644 index 0000000..567609b --- /dev/null +++ b/kadai3-2/nagaa052/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/kadai3-2/nagaa052/Makefile b/kadai3-2/nagaa052/Makefile new file mode 100644 index 0000000..4c2045c --- /dev/null +++ b/kadai3-2/nagaa052/Makefile @@ -0,0 +1,29 @@ +NAME := vget + +GO ?= go +BUILD_DIR=./build +BINARY ?= $(BUILD_DIR)/$(NAME) + +.PHONY: all +all: clean test build + +.PHONY: test +test: + $(GO) test -v -race ./... + +.PHONY: test_integration +test_integration: + $(GO) test -v -tags=integration ./... + +.PHONY: test_cover +test_cover: + $(GO) test -v -cover ./... + +.PHONY: clean +clean: + $(GO) clean + rm -f $(BINARY) + +.PHONY: build +build: + $(GO) build -o $(BINARY) -v diff --git a/kadai3-2/nagaa052/cmd/vget/main.go b/kadai3-2/nagaa052/cmd/vget/main.go new file mode 100644 index 0000000..5d7bec3 --- /dev/null +++ b/kadai3-2/nagaa052/cmd/vget/main.go @@ -0,0 +1,43 @@ +package main + +import ( + "flag" + "fmt" + "io" + "os" + "time" + + vget "github.com/gopherdojo/dojo5/kadai3-2/nagaa052" +) + +var outStream io.Writer = os.Stdout +var errStream io.Writer = os.Stderr + +func main() { + var timeout int + flag.IntVar(&timeout, "t", 30, "Timeout Seconds") + flag.Usage = usage + flag.Parse() + + args := flag.Args() + opt := vget.Options{ + TimeOut: time.Duration(timeout) * time.Second, + } + + v, err := vget.New(args[0], opt, outStream, errStream) + if err != nil { + fmt.Printf("%v\n", err.Error()) + os.Exit(1) + } + os.Exit(v.Download()) +} + +func usage() { + fmt.Fprintf(os.Stderr, ` +Parallel download +Usage: + tgame [option] +Options: +`) + flag.PrintDefaults() +} diff --git a/kadai3-2/nagaa052/pkg/executor/executor.go b/kadai3-2/nagaa052/pkg/executor/executor.go new file mode 100644 index 0000000..85308e9 --- /dev/null +++ b/kadai3-2/nagaa052/pkg/executor/executor.go @@ -0,0 +1,51 @@ +package executor + +import ( + "context" + "time" + + "golang.org/x/sync/errgroup" +) + +type Payload interface { + Execute(context.Context) error +} + +type Job struct { + Payload +} + +type Executor struct { + Timeout time.Duration + Jobs []*Job +} + +func New(maxWorkers int, timeout time.Duration) *Executor { + return &Executor{ + Timeout: timeout, + Jobs: make([]*Job, 0), + } +} + +func (ex *Executor) AddPayload(payload Payload) { + ex.Jobs = append(ex.Jobs, &Job{payload}) +} + +func (ex *Executor) Start() error { + eg, ctx := errgroup.WithContext(context.Background()) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + for _, job := range ex.Jobs { + job := job + eg.Go(func() error { + return job.Execute(ctx) + }) + } + + if err := eg.Wait(); err != nil { + return err + } + + return nil +} diff --git a/kadai3-2/nagaa052/pkg/executor/executor_test.go b/kadai3-2/nagaa052/pkg/executor/executor_test.go new file mode 100644 index 0000000..af12eb9 --- /dev/null +++ b/kadai3-2/nagaa052/pkg/executor/executor_test.go @@ -0,0 +1,86 @@ +package executor_test + +import ( + "context" + "reflect" + "testing" + "time" + + executor "github.com/gopherdojo/dojo5/kadai3-2/nagaa052/pkg/executor" +) + +func TestNew(t *testing.T) { + type args struct { + maxWorkers int + timeout time.Duration + } + tests := []struct { + name string + args args + want *executor.Executor + }{ + { + name: "Success Test", + args: args{ + maxWorkers: 4, + timeout: 2 * time.Second, + }, + want: &executor.Executor{ + Timeout: 2 * time.Second, + Jobs: make([]*executor.Job, 0), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := executor.New(tt.args.maxWorkers, tt.args.timeout); !reflect.DeepEqual(got, tt.want) { + t.Errorf("New() = %v, want %v", got, tt.want) + } + }) + } +} + +type mockPayload struct{} + +func (p *mockPayload) Execute(context.Context) error { + return nil +} + +func TestExecutor_Start(t *testing.T) { + type fields struct { + Timeout time.Duration + Jobs []*executor.Job + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "Success Test", + fields: fields{ + Timeout: 2 * time.Second, + Jobs: []*executor.Job{ + &executor.Job{ + &mockPayload{}, + }, + &executor.Job{ + &mockPayload{}, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + ex := &executor.Executor{tt.fields.Timeout, tt.fields.Jobs} + if err := ex.Start(); (err != nil) != tt.wantErr { + t.Errorf("Executor.Start() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/kadai3-2/nagaa052/pkg/request/request.go b/kadai3-2/nagaa052/pkg/request/request.go new file mode 100644 index 0000000..714987d --- /dev/null +++ b/kadai3-2/nagaa052/pkg/request/request.go @@ -0,0 +1,90 @@ +package request + +import ( + "context" + "fmt" + "io" + "net/http" + "os" +) + +type Range struct{} + +func (r *Range) Download(ctx context.Context, url string, from, to int64, outFile string) error { + client := http.DefaultClient + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + req = req.WithContext(ctx) + + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", from, to)) + + ch := make(chan struct{}) + errCh := make(chan error) + + go func() { + resp, err := client.Do(req) + if err != nil { + errCh <- err + return + } + defer resp.Body.Close() + output, err := os.OpenFile(outFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + errCh <- err + } + defer output.Close() + + io.Copy(output, resp.Body) + + ch <- struct{}{} + }() + + select { + case err := <-errCh: + return err + case <-ch: + return nil + } +} + +func (r *Range) GetContentLength(ctx context.Context, url string) (int64, error) { + client := http.DefaultClient + req, err := http.NewRequest("HEAD", url, nil) + if err != nil { + return 0, err + } + req = req.WithContext(ctx) + + ch := make(chan int64) + errCh := make(chan error) + + go func() { + resp, err := client.Do(req) + if err != nil { + errCh <- err + return + } + defer resp.Body.Close() + if resp.Header.Get("Accept-Ranges") != "bytes" { + errCh <- fmt.Errorf("not supported range access: %s", url) + return + } + + if resp.ContentLength <= 0 { + fmt.Printf("%v", resp.ContentLength) + errCh <- fmt.Errorf("not supported range access") + return + } + + ch <- resp.ContentLength + }() + + select { + case err := <-errCh: + return 0, err + case size := <-ch: + return size, nil + } +} diff --git a/kadai3-2/nagaa052/proc.go b/kadai3-2/nagaa052/proc.go new file mode 100644 index 0000000..88fb6b5 --- /dev/null +++ b/kadai3-2/nagaa052/proc.go @@ -0,0 +1,74 @@ +package vget + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/gopherdojo/dojo5/kadai3-2/nagaa052/pkg/executor" + "github.com/gopherdojo/dojo5/kadai3-2/nagaa052/pkg/request" +) + +var _ executor.Payload = &Proc{} + +type Proc struct { + URL string + OutFile string + Index int + From int64 + To int64 +} + +func NewProc(url, outFile string, from, to int64, index int) *Proc { + return &Proc{ + URL: url, + OutFile: outFile, + Index: index, + From: from, + To: to, + } +} + +func (p *Proc) Execute(ctx context.Context) error { + r := request.Range{} + return r.Download(ctx, p.URL, p.From, p.To, p.OutFile) +} + +func GetProcPath(dir, filename string, index int) string { + return fmt.Sprintf("%s/%s.%d", dir, filename, index) +} + +func IxExistProcFile(dir, filename string, index int, fileSize int64) bool { + filePath := GetProcPath(dir, filename, index) + if info, err := os.Stat(filePath); err == nil { + if info.Size() == fileSize { + return true + } + } + + return false +} + +func MargeProcFiles(dir, filename string, outFile string, procsCount int) error { + fh, err := os.Create(outFile) + if err != nil { + return err + } + defer fh.Close() + + for i := 0; i < procsCount; i++ { + procFile := GetProcPath(dir, filename, i) + subfp, err := os.Open(procFile) + if err != nil { + return err + } + defer subfp.Close() + + io.Copy(fh, subfp) + if err := os.Remove(procFile); err != nil { + return err + } + } + return nil +} diff --git a/kadai3-2/nagaa052/vget.go b/kadai3-2/nagaa052/vget.go new file mode 100644 index 0000000..1470010 --- /dev/null +++ b/kadai3-2/nagaa052/vget.go @@ -0,0 +1,140 @@ +package vget + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/gopherdojo/dojo5/kadai3-2/nagaa052/pkg/executor" + + "github.com/gopherdojo/dojo5/kadai3-2/nagaa052/pkg/request" +) + +const ( + ExitOK = iota + ExitError +) + +type Vget struct { + URL string + Options + outStream, errStream io.Writer +} + +// Options is a specifiable option. Default is DefaultOptions. +type Options struct { + Procs int + TimeOut time.Duration + DestDir string + FileName string +} + +var DefaultOptions = Options{ + Procs: runtime.NumCPU(), + TimeOut: time.Duration(60) * time.Second, + DestDir: "download", +} + +func New(url string, opt Options, outStream, errStream io.Writer) (*Vget, error) { + if url == "" { + return nil, fmt.Errorf("target url is required") + } + + if opt.Procs == 0 { + opt.Procs = DefaultOptions.Procs + } + + if opt.TimeOut == 0 { + opt.TimeOut = DefaultOptions.TimeOut + } + + if opt.DestDir == "" { + opt.DestDir = DefaultOptions.DestDir + } + + destDir, err := filepath.Abs(opt.DestDir) + if err != nil { + return nil, fmt.Errorf("Invalid directory specification") + } + opt.DestDir = destDir + + if opt.FileName == "" { + sURL := strings.Split(url, "/") + opt.FileName = sURL[len(sURL)-1] + } + + return &Vget{ + URL: url, + Options: opt, + outStream: outStream, + errStream: errStream, + }, nil +} + +func (v *Vget) Download() int { + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + r := &request.Range{} + size, err := r.GetContentLength(ctx, v.URL) + if err != nil { + fmt.Fprintf(v.errStream, "%+v\n", err) + return ExitError + } + + procs, err := v.getProcs(size) + if err != nil { + fmt.Fprintf(v.errStream, "%+v\n", err) + return ExitError + } + + if _, err := os.Stat(v.DestDir); err != nil { + if err := os.MkdirAll(v.DestDir, 0755); err != nil { + fmt.Fprintf(v.errStream, "%+v\n", err) + return ExitError + } + } + + ex := executor.New(v.Options.Procs, v.Options.TimeOut) + for _, proc := range procs { + ex.AddPayload(proc) + } + + fmt.Fprintf(v.outStream, "Start download.\n") + err = ex.Start() + if err != nil { + fmt.Fprintf(v.errStream, "%+v\n", err) + return ExitError + } + + outfile := fmt.Sprintf("%s/%s", v.DestDir, v.FileName) + err = MargeProcFiles(v.DestDir, v.FileName, outfile, v.Procs) + if err != nil { + fmt.Fprintf(v.errStream, "%+v\n", err) + return ExitError + } + + fmt.Fprintf(v.outStream, "Download finish\n") + return ExitOK +} + +func (v *Vget) getProcs(size int64) ([]*Proc, error) { + procs := []*Proc{} + + procSize := size / int64(v.Procs) + for i := 0; i < v.Procs; i++ { + if !IxExistProcFile(v.DestDir, v.FileName, i, procSize) { + from := procSize * int64(i) + to := procSize * int64(i+1) + proc := NewProc(v.URL, GetProcPath(v.DestDir, v.FileName, i), from, to, i) + procs = append(procs, proc) + } + } + return procs, nil +} diff --git a/kadai3-2/nagaa052/vget_test.go b/kadai3-2/nagaa052/vget_test.go new file mode 100644 index 0000000..7726e15 --- /dev/null +++ b/kadai3-2/nagaa052/vget_test.go @@ -0,0 +1,44 @@ +package vget_test + +import ( + "bytes" + "testing" + + vget "github.com/gopherdojo/dojo5/kadai3-2/nagaa052" +) + +func TestNew(t *testing.T) { + type args struct { + url string + opt vget.Options + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "Success Test", + args: args{ + url: "http://example.com", + opt: vget.DefaultOptions, + }, + wantErr: false, + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + outStream := &bytes.Buffer{} + errStream := &bytes.Buffer{} + _, err := vget.New(tt.args.url, tt.args.opt, outStream, errStream) + if (err != nil) != tt.wantErr { + t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +}