handle some broken redirects to pre-signed S3 URLs (#438)

* handle some broken redirects to pre-signed S3 URLs

* making this more generic

* more concise

Author: drewda

request/http.go

@@ -26,6 +26,13 @@ func (r *Http) SetSecret(secret dmfr.Secret) error {
 	return nil
 }
 
+func removeDefaultPortFromHost(req *http.Request) {
+	if (req.URL.Scheme == "https" && strings.HasSuffix(req.URL.Host, ":443")) ||
+		(req.URL.Scheme == "http" && strings.HasSuffix(req.URL.Host, ":80")) {
+		req.Host = strings.Split(req.URL.Host, ":")[0]
+	}
+}
+
 func (r Http) Download(ctx context.Context, ustr string) (io.ReadCloser, int, error) {
 	return r.DownloadAuth(ctx, ustr, dmfr.FeedAuthorization{})
 }
@@ -68,7 +75,16 @@ func (r Http) DownloadAuth(ctx context.Context, ustr string, auth dmfr.FeedAutho
 	// Make HTTP request
 	req.Header.Set("User-Agent", fmt.Sprintf("transitland/%s", tl.Version.Tag))
 
-	client := &http.Client{}
+	// Remove default ports from host header if explicitly specified as it
+	// may break pre-signed S3 URLs or other systems that rely on the host header
+	removeDefaultPortFromHost(req)
+
+	client := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			removeDefaultPortFromHost(req)
+			return nil
+		},
+	}
 	resp, err := client.Do(req)
 	if err != nil {
 		// return error directly