Add handling for multiple urls in fetch

* change Download struct to list of urls
* add URLs to Fetch struct
* add new handler for Fetch Type "download_list" in Fetcher
* Type "download" and "default" will use list with single url
* Fetch will iterate through the urls and try them one by one
  until one is successful

Signed-off-by: Fabian 'xx4h' Sylvester <xx4h@xx4h.de>

Author: xx4h

internal/fetch/download.go

@@ -17,47 +17,70 @@ import (
 
 // Download handles direct binary releases
 type Download struct {
-	url     string
+	urls    []string
 	headers map[string]string
 }
 
 // Fetch gets the package and returns location of downloaded file
 func (d Download) Fetch(ctx context.Context, dist, v string, mapper mapping.Mapper) (string, error) {
 	logger := zerolog.Ctx(ctx).With().Str("func", "Download.Fetch").Logger()
 
-	args := tpl.New(v, mapper)
-
-	url, err := args.Render(d.url)
-	if err != nil {
-		return "", err
-	}
-
-	logger.Debug().Msgf("fetching version %q for arch %q and OS %q at %s", v, runtime.GOARCH, runtime.GOOS, url)
-
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return "", err
-	}
-
-	for k, v := range d.headers {
-		req.Header.Add(k, v)
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("unable to download binary at %s: %s", url, resp.Status)
+	var resp *http.Response
+
+	for i, u := range d.urls {
+
+		args := tpl.New(v, mapper)
+
+		url, err := args.Render(u)
+		if err != nil {
+			if len(d.urls)-1 > i {
+				continue
+			} else {
+				return "", err
+			}
+		}
+
+		logger.Debug().Msgf("fetching version %q for arch %q and OS %q at %s", v, runtime.GOARCH, runtime.GOOS, url)
+
+		req, err := http.NewRequest("GET", url, nil)
+		if err != nil {
+			if len(d.urls)-1 > i {
+				continue
+			} else {
+				return "", err
+			}
+		}
+
+		for k, v := range d.headers {
+			req.Header.Add(k, v)
+		}
+
+		resp, err = http.DefaultClient.Do(req)
+		if err != nil {
+			if len(d.urls)-1 > i {
+				continue
+			} else {
+				return "", err
+			}
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode != http.StatusOK {
+			if len(d.urls)-1 > i {
+				logger.Debug().Msgf("unable to download binary at %s: %s, %d urls left to try...", url, resp.Status, len(d.urls)-1-i)
+				continue
+			} else {
+				return "", fmt.Errorf("unable to download binary at %s: %s", url, resp.Status)
+			}
+		}
+		// if we reach this point, download was successful, let's move on
+		break
 	}
 
 	tmpfile, err := os.CreateTemp("", v)
 	if err != nil {
 		logger.Fatal().Err(err)
 	}
-
 	defer tmpfile.Close()
 
 	bar := progressbar.DefaultBytes(

internal/fetch/fetch.go

@@ -16,9 +16,10 @@ type Fetcher interface {
 
 // Fetch contains fetch configuration
 type Fetch struct {
-	Type     string `yaml:"type"`
-	URL      string `yaml:"url"`
-	TokenEnv string `yaml:"token_env"`
+	Type     string   `yaml:"type"`
+	URL      string   `yaml:"url"`
+	URLs     []string `yaml:"urls"`
+	TokenEnv string   `yaml:"token_env"`
 }
 
 // Factory returns instances that comply to Fetcher interface
@@ -37,10 +38,13 @@ func (r Fetch) Factory() (Fetcher, error) {
 			}
 			headers["PRIVATE-TOKEN"] = token
 		}
-
 		return Download{
-			url:     r.URL,
+			urls:    []string{r.URL},
 			headers: headers,
 		}, nil
+	case "download_list":
+		return Download{
+			urls: r.URLs,
+		}, nil
 	}
 }