Merge pull request #183 from carolynvs/install-fallback-location

Retry failed release downloads from test location

Author: carolynvs

dvm-helper/dockerversion/dockerversion.go

@@ -2,10 +2,14 @@ package dockerversion
 
 import (
 	"fmt"
+	"log"
+	"net/http"
+	"path/filepath"
 	"sort"
 	"strings"
 
 	"github.com/Masterminds/semver"
+	"github.com/howtowhale/dvm/dvm-helper/internal/downloader"
 	"github.com/pkg/errors"
 )
 
@@ -37,12 +41,9 @@ func Parse(value string) Version {
 	return v
 }
 
-func (version Version) BuildDownloadURL(mirror string) (url string, archived bool, checksumed bool, err error) {
+func (version Version) buildDownloadURL(mirror string, forcePrerelease bool) (url string, archived bool, checksumed bool, err error) {
 	var releaseSlug, versionSlug, extSlug string
 
-	archivedReleaseCutoff, _ := semver.NewVersion("1.11.0-rc1")
-	dockerStoreCutoff, _ := semver.NewVersion("17.06.0-ce")
-
 	var edgeVersion Version
 	if version.IsEdge() {
 		edgeVersion, err = findLatestEdgeVersion(mirror)
@@ -52,7 +53,7 @@ func (version Version) BuildDownloadURL(mirror string) (url string, archived boo
 	}
 
 	// Docker Store Download
-	if version.IsEdge() || !version.semver.LessThan(dockerStoreCutoff) {
+	if version.shouldBeInDockerStore() {
 		archived = true
 		checksumed = false
 		extSlug = archiveFileExt
@@ -62,7 +63,7 @@ func (version Version) BuildDownloadURL(mirror string) (url string, archived boo
 		if version.IsEdge() {
 			releaseSlug = "edge"
 			versionSlug = edgeVersion.String()
-		} else if version.IsPrerelease() {
+		} else if version.IsPrerelease() || forcePrerelease {
 			releaseSlug = "test"
 			versionSlug = version.String()
 		} else {
@@ -74,7 +75,7 @@ func (version Version) BuildDownloadURL(mirror string) (url string, archived boo
 			mirror, mobyOS, releaseSlug, dockerArch, versionSlug, extSlug)
 		return
 	} else { // Original Download
-		archived = !version.semver.LessThan(archivedReleaseCutoff)
+		archived = version.shouldBeArchived()
 		checksumed = true
 		versionSlug = version.String()
 		if archived {
@@ -97,6 +98,70 @@ func (version Version) BuildDownloadURL(mirror string) (url string, archived boo
 	}
 }
 
+// Download a Docker release.
+// version - the desired version.
+// mirrorURL - optional alternate download location.
+// binaryPath - full path to where the Docker client binary should be saved.
+func (version Version) Download(mirrorURL string, binaryPath string, l *log.Logger) error {
+	err := version.download(false, mirrorURL, binaryPath, l)
+	if err != nil && !version.IsPrerelease() && version.shouldBeInDockerStore() {
+		// Docker initially publishes non-rc version versions to the test location
+		// and then later republishes to the stable location
+		// Retry stable versions against test to find "unstable" stable versions. :-)
+		l.Printf("Could not find a stable release for %s, checking for a test release\n", version)
+		retryErr := version.download(true, mirrorURL, binaryPath, l)
+		return errors.Wrapf(retryErr, "Attempted to fallback to downloading from the prerelease location after downloading from the stable location failed: %s", err.Error())
+	}
+	return err
+}
+
+func (version Version) download(forcePrerelease bool, mirrorURL string, binaryPath string, l *log.Logger) error {
+	url, archived, checksumed, err := version.buildDownloadURL(mirrorURL, forcePrerelease)
+	if err != nil {
+		return errors.Wrapf(err, "Unable to determine the download URL for %s", version)
+	}
+
+	l.Printf("Checking if %s can be found at %s", version, url)
+	head, err := http.Head(url)
+	if err != nil {
+		return errors.Wrapf(err, "Unable to determine if %s is a valid version", version)
+	}
+	if head.StatusCode >= 400 {
+		return errors.Errorf("Version %s not found (%v) - try `dvm ls-remote` to browse available versions", version, head.StatusCode)
+	}
+
+	d := downloader.New(l)
+	binaryName := filepath.Base(binaryPath)
+
+	if archived {
+		archivedFile := filepath.Join("docker", binaryName)
+		if checksumed {
+			return d.DownloadArchivedFileWithChecksum(url, archivedFile, binaryPath)
+		}
+		return d.DownloadArchivedFile(url, archivedFile, binaryPath)
+	}
+
+	if checksumed {
+		return d.DownloadFileWithChecksum(url, binaryPath)
+	}
+	return d.DownloadFile(url, binaryPath)
+}
+
+func (version Version) shouldBeInDockerStore() bool {
+	if version.IsEdge() {
+		return true
+	}
+
+	dockerStoreCutoff, _ := semver.NewVersion("17.06.0-ce")
+
+	return version.semver != nil && !version.semver.LessThan(dockerStoreCutoff)
+}
+
+func (version Version) shouldBeArchived() bool {
+	archivedReleaseCutoff, _ := semver.NewVersion("1.11.0-rc1")
+	return version.semver != nil && !version.semver.LessThan(archivedReleaseCutoff)
+}
+
 func (version Version) IsPrerelease() bool {
 	if version.semver == nil {
 		return false

dvm-helper/dockerversion/dockerversion_test.go

@@ -2,9 +2,13 @@ package dockerversion
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net/http"
+	"path/filepath"
 	"testing"
 
+	"log"
+
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 )
@@ -172,7 +176,7 @@ func TestVersion_BuildDownloadURL(t *testing.T) {
 
 	for version, testcase := range testcases {
 		t.Run(version.String(), func(t *testing.T) {
-			gotURL, gotArchived, gotChecksumed, err := version.BuildDownloadURL("")
+			gotURL, gotArchived, gotChecksumed, err := version.buildDownloadURL("", false)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -201,26 +205,12 @@ func TestVersion_BuildDownloadURL(t *testing.T) {
 
 func TestVersion_DownloadEdgeRelease(t *testing.T) {
 	version := Parse("edge")
+	tempDir, _ := ioutil.TempDir("", "dvmtest")
+	destPath := filepath.Join(tempDir, "docker")
 
-	url, archived, checksumed, err := version.BuildDownloadURL("")
+	l := log.New(ioutil.Discard, "", log.LstdFlags)
+	err := version.Download("", destPath, l)
 	if err != nil {
 		t.Fatalf("%#v", err)
 	}
-
-	if !archived {
-		t.Fatal("Expected the edge release to be archived.")
-	}
-
-	if checksumed {
-		t.Fatal("Expected the edge release to NOT be checksumed.")
-	}
-
-	response, err := http.DefaultClient.Get(url)
-	if err != nil {
-		t.Fatalf("%#v", errors.Wrapf(err, "Unable to download release from %s", response))
-	}
-
-	if response.StatusCode != 200 {
-		t.Fatalf("Unexpected status code (%d) when downloading %s", response.StatusCode, url)
-	}
 }

dvm-helper/dvm-helper.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"log"
 	neturl "net/url"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -404,7 +404,7 @@ func list(pattern string) {
 func install(version dockerversion.Version) {
 	versionDir := getVersionDir(version)
 
-	if version.IsEdge() && pathExists(versionDir) {
+	if version.IsEdge() {
 		// Always install latest of edge build
 		err := os.RemoveAll(versionDir)
 		if err != nil {
@@ -428,29 +428,20 @@ func install(version dockerversion.Version) {
 }
 
 func downloadRelease(version dockerversion.Version) {
-	url, archived, checksumed, err := version.BuildDownloadURL(mirrorURL)
+	destPath := filepath.Join(getVersionDir(version), getBinaryName())
+	err := version.Download(mirrorURL, destPath, getDebugLogger())
 	if err != nil {
-		die("Unable to determine the download URL for %s", err, retCodeRuntimeError, version)
+		die("", err, retCodeRuntimeError)
 	}
 
-	binaryName := getBinaryName()
-	binaryPath := filepath.Join(getVersionDir(version), binaryName)
-	if archived {
-		archivedFile := path.Join("docker", binaryName)
-		if checksumed {
-			downloadArchivedFileWithChecksum(url, archivedFile, binaryPath)
-		} else {
-			downloadArchivedFile(url, archivedFile, binaryPath)
-		}
-	} else {
-		if checksumed {
-			downloadFileWithChecksum(url, binaryPath)
-		} else {
-			downloadFile(url, binaryPath)
-		}
+	writeDebug("Downloaded Docker %s to %s", version, destPath)
+}
 
+func getDebugLogger() *log.Logger {
+	if debug {
+		return log.New(color.Output, "", log.LstdFlags)
 	}
-	writeDebug("Downloaded Docker %s to %s.", version, binaryPath)
+	return log.New(ioutil.Discard, "", log.LstdFlags)
 }
 
 func uninstall(version dockerversion.Version) {

dvm-helper/dvm-helper.nix.go

@@ -5,18 +5,28 @@ package main
 import (
 	"os"
 	"path/filepath"
+
+	"github.com/howtowhale/dvm/dvm-helper/internal/downloader"
 )
 
 const binaryFileExt string = ""
 
 func upgradeSelf(version string) {
+	d := downloader.New(getDebugLogger())
+
 	binaryURL := buildDvmReleaseURL(version, dvmOS, dvmArch, "dvm-helper")
 	binaryPath := filepath.Join(dvmDir, "dvm-helper", "dvm-helper")
-	downloadFileWithChecksum(binaryURL, binaryPath)
+	err := d.DownloadFileWithChecksum(binaryURL, binaryPath)
+	if err != nil {
+		die("", err, retCodeRuntimeError)
+	}
 
 	scriptURL := buildDvmReleaseURL(version, "dvm.sh")
 	scriptPath := filepath.Join(dvmDir, "dvm.sh")
-	downloadFile(scriptURL, scriptPath)
+	err = d.DownloadFile(scriptURL, scriptPath)
+	if err != nil {
+		die("", err, retCodeRuntimeError)
+	}
 }
 
 func getCleanPathRegex() string {

dvm-helper/dvm-helper.windows.go

@@ -6,24 +6,37 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
+
+	"github.com/howtowhale/dvm/dvm-helper/internal/downloader"
 )
-import "strings"
 
 const dvmOS string = "Windows"
 const binaryFileExt string = ".exe"
 
 func upgradeSelf(version string) {
+	d := downloader.New(getDebugLogger())
+
 	binaryURL := buildDvmReleaseURL(version, dvmOS, dvmArch, "dvm-helper.exe")
 	binaryPath := filepath.Join(dvmDir, ".tmp", "dvm-helper.exe")
-	downloadFileWithChecksum(binaryURL, binaryPath)
+	err := d.DownloadFileWithChecksum(binaryURL, binaryPath)
+	if err != nil {
+		die("", err, retCodeRuntimeError)
+	}
 
 	psScriptURL := buildDvmReleaseURL(version, "dvm.ps1")
 	psScriptPath := filepath.Join(dvmDir, "dvm.ps1")
-	downloadFile(psScriptURL, psScriptPath)
+	err = d.DownloadFile(psScriptURL, psScriptPath)
+	if err != nil {
+		die("", err, retCodeRuntimeError)
+	}
 
 	cmdScriptURL := buildDvmReleaseURL(version, "dvm.cmd")
 	cmdScriptPath := filepath.Join(dvmDir, "dvm.cmd")
-	downloadFile(cmdScriptURL, cmdScriptPath)
+	err = d.DownloadFile(cmdScriptURL, cmdScriptPath)
+	if err != nil {
+		die("", err, retCodeRuntimeError)
+	}
 
 	writeUpgradeScript()
 }

dvm-helper/dvm-helper_test.go

@@ -161,3 +161,35 @@ func TestInstallPrereleases(t *testing.T) {
 	assert.NotEmpty(t, output, "Should have captured stdout")
 	assert.Contains(t, output, "Now using Docker 1.12.5-rc1", "Should have installed a prerelease version")
 }
+
+// install a version from the test location that is missing the -rc suffix
+func TestInstallNonPrereleaseTestRelease(t *testing.T) {
+	_, github := createMockDVM(nil)
+	defer github.Close()
+
+	outputCapture := &bytes.Buffer{}
+	color.Output = outputCapture
+
+	dvm := makeCliApp()
+	dvm.Run([]string{"dvm-helper", "--debug", "install", "17.10.0-ce"})
+
+	output := outputCapture.String()
+	assert.NotEmpty(t, output, "Should have captured stdout")
+	assert.Contains(t, output, "Now using Docker 17.10.0-ce", "Should have installed a test version")
+}
+
+// install something that used to be a test release and is now considered stable
+func TestInstallStabilizedTestRelease(t *testing.T) {
+	_, github := createMockDVM(nil)
+	defer github.Close()
+
+	outputCapture := &bytes.Buffer{}
+	color.Output = outputCapture
+
+	dvm := makeCliApp()
+	dvm.Run([]string{"dvm-helper", "--debug", "install", "17.09.0-ce"})
+
+	output := outputCapture.String()
+	assert.NotEmpty(t, output, "Should have captured stdout")
+	assert.Contains(t, output, "Now using Docker 17.09.0-ce", "Should have installed a stable version")
+}