Downloading speed improvements

README.md

@@ -20,7 +20,7 @@ Binary file will be built at ./bin/hget, you can copy to /usr/bin or /usr/local/
 ## Usage
 
 ```
-hget [Url] [-n parallel] [-skip-tls false] //to download url, with n connections, and not skip tls certificate
+hget [-n parallel] [-skip-tls false] Url //to download url, with n connections, and not skip tls certificate
 hget tasks //get interrupted tasks
 hget resume [TaskName | URL] //to resume task
 ```

http.go

@@ -102,7 +102,8 @@ func NewHttpDownloader(url string, par int, skipTls bool) *HttpDownloader {
 }
 
 func partCalculate(par int64, len int64, url string) []Part {
-	ret := make([]Part, 0)
+	// Pre-allocate, perf tunning
+	ret := make([]Part, par)
 	for j := int64(0); j < par; j++ {
 		from := (len / par) * j
 		var to int64
@@ -119,10 +120,12 @@ func partCalculate(par int64, len int64, url string) []Part {
 			os.Exit(1)
 		}
 
-		fname := fmt.Sprintf("%s.part%d", file, j)
+		// Padding 0 before path name as filename will be sorted as string
+		fname := fmt.Sprintf("%s.part%06d", file, j)
 		path := filepath.Join(folder, fname) // ~/.hget/download-file-name/part-name
-		ret = append(ret, Part{Url: url, Path: path, RangeFrom: from, RangeTo: to})
+		ret[j] = Part{Index: j, Url: url, Path: path, RangeFrom: from, RangeTo: to}
 	}
+
 	return ret
 }
 
@@ -132,25 +135,31 @@ func (d *HttpDownloader) Do(doneChan chan bool, fileChan chan string, errorChan
 	var barpool *pb.Pool
 	var err error
 
-	if DisplayProgressBar() {
-		bars = make([]*pb.ProgressBar, 0)
-		for i, part := range d.parts {
-			newbar := pb.New64(part.RangeTo - part.RangeFrom).SetUnits(pb.U_BYTES).Prefix(color.YellowString(fmt.Sprintf("%s-%d", d.file, i)))
-			bars = append(bars, newbar)
+	for _, p := range d.parts {
+
+		if p.RangeTo <= p.RangeFrom {
+			fileChan <- p.Path
+			stateSaveChan <- Part{
+				Index:     p.Index,
+				Url:       d.url,
+				Path:      p.Path,
+				RangeFrom: p.RangeFrom,
+				RangeTo:   p.RangeTo,
+			}
+
+			continue
+		}
+
+		var bar *pb.ProgressBar
+
+		if DisplayProgressBar() {
+			bar = pb.New64(p.RangeTo - p.RangeFrom).SetUnits(pb.U_BYTES).Prefix(color.YellowString(fmt.Sprintf("%s-%d", d.file, p.Index)))
+			bars = append(bars, bar)
 		}
-		barpool, err = pb.StartPool(bars...)
-		FatalCheck(err)
-	}
 
-	for i, p := range d.parts {
 		ws.Add(1)
-		go func(d *HttpDownloader, loop int64, part Part) {
+		go func(d *HttpDownloader, bar *pb.ProgressBar, part Part) {
 			defer ws.Done()
-			var bar *pb.ProgressBar
-
-			if DisplayProgressBar() {
-				bar = bars[loop]
-			}
 
 			var ranges string
 			if part.RangeTo != d.len {
@@ -197,29 +206,42 @@ func (d *HttpDownloader) Do(doneChan chan bool, fileChan chan string, errorChan
 				writer = io.MultiWriter(f)
 			}
 
-			//make copy interruptable by copy 100 bytes each loop
 			current := int64(0)
-			for {
-				select {
-				case <-interruptChan:
-					stateSaveChan <- Part{Url: d.url, Path: part.Path, RangeFrom: current + part.RangeFrom, RangeTo: part.RangeTo}
-					return
-				default:
-					written, err := io.CopyN(writer, resp.Body, 100)
-					current += written
-					if err != nil {
-						if err != io.EOF {
-							errorChan <- err
-						}
-						bar.Finish()
-						fileChan <- part.Path
-						return
-					}
-				}
+			finishDownloadChan := make(chan bool)
+
+			go func() {
+				written, _ := io.Copy(writer, resp.Body)
+				current += written
+				fileChan <- part.Path
+				finishDownloadChan <- true
+			}()
+
+			select {
+			case <-interruptChan:
+				// interrupt download by forcefully close the input stream
+				resp.Body.Close()
+				<-finishDownloadChan
+			case <-finishDownloadChan:
+			}
+
+			stateSaveChan <- Part{
+				Index:     part.Index,
+				Url:       d.url,
+				Path:      part.Path,
+				RangeFrom: current + part.RangeFrom,
+				RangeTo:   part.RangeTo,
 			}
-		}(d, int64(i), p)
+
+			if DisplayProgressBar() {
+				bar.Update()
+				bar.Finish()
+			}
+		}(d, bar, p)
 	}
 
+	barpool, err = pb.StartPool(bars...)
+	FatalCheck(err)
+
 	ws.Wait()
 	doneChan <- true
 	barpool.Stop()

http_test.go

@@ -1,9 +1,9 @@
 package main
 
 import (
-	"testing"
 	"os/user"
 	"path/filepath"
+	"testing"
 )
 
 func TestPartCalculate(t *testing.T) {
@@ -13,15 +13,22 @@ func TestPartCalculate(t *testing.T) {
 	if len(parts) != 10 {
 		t.Fatalf("parts length should be 10")
 	}
+
 	if parts[0].Url != "http://foo.bar/file" {
 		t.Fatalf("part url was wrong")
 	}
+
 	usr, _ := user.Current()
-	dir := filepath.Join(usr.HomeDir, dataFolder, "file/file.part0")
-	if parts[0].Path != dir {
+	dir := filepath.Join(usr.HomeDir, dataFolder, "file/file.part000001")
+	if parts[1].Path != dir {
 		t.Fatalf("part path was wrong")
 	}
+
 	if parts[0].RangeFrom != 0 && parts[0].RangeTo != 10 {
 		t.Fatalf("part range was wrong")
 	}
+
+	if parts[1].Index != 1 {
+		t.Fatal("part index was wrong")
+	}
 }

joiner.go

@@ -1,8 +1,8 @@
 package main
 
 import (
-	"gopkg.in/cheggaaa/pb.v1"
 	"github.com/fatih/color"
+	"gopkg.in/cheggaaa/pb.v1"
 	"io"
 	"os"
 	"sort"

main.go

@@ -14,11 +14,12 @@ var displayProgress = true
 func main() {
 	var err error
 
-	conn    := flag.Int("n", runtime.NumCPU(), "connection")
+	conn := flag.Int("n", runtime.NumCPU(), "connection")
 	skiptls := flag.Bool("skip-tls", true, "skip verify certificate for https")
 
 	flag.Parse()
 	args := flag.Args()
+
 	if len(args) < 1 {
 		Errorln("url is required")
 		usage()
@@ -61,7 +62,6 @@ func main() {
 
 func Execute(url string, state *State, conn int, skiptls bool) {
 	//otherwise is hget <URL> command
-	var err error
 
 	signal_chan := make(chan os.Signal, 1)
 	signal.Notify(signal_chan,
@@ -120,7 +120,7 @@ func Execute(url string, state *State, conn int, skiptls bool) {
 					return
 				}
 			} else {
-				err = JoinFile(files, filepath.Base(url))
+				err := JoinFile(files, filepath.Base(url))
 				FatalCheck(err)
 				err = os.RemoveAll(FolderOf(url))
 				FatalCheck(err)
@@ -132,7 +132,7 @@ func Execute(url string, state *State, conn int, skiptls bool) {
 
 func usage() {
 	Printf(`Usage:
-hget [URL] [-n connection] [-skip-tls true]
+hget [-n connection] [-skip-tls true] URL
 hget tasks
 hget resume [TaskName]
 `)

state.go

@@ -16,6 +16,7 @@ type State struct {
 }
 
 type Part struct {
+	Index     int64
 	Url       string
 	Path      string
 	RangeFrom int64